diff --git a/docs/modules/storage/pages/addendum/tools.adoc b/docs/modules/storage/pages/addendum/tools.adoc index ef86cccb..6ad6fe85 100644 --- a/docs/modules/storage/pages/addendum/tools.adoc +++ b/docs/modules/storage/pages/addendum/tools.adoc @@ -64,6 +64,7 @@ This is useful for: * Changing the number of storage channels * Moving storage from local disk to a cloud storage target or vice-versa +* Converting === Prerequisites @@ -132,3 +133,21 @@ To move from local file system to a cloud storage target, configure the target ` ==== For more information see the https://github.com/eclipse-store/store/tree/main/storage/embedded-tools/storage-converter[readme file]. + + +=== Converting binary data + +To convert the binary representation of persisted objects BinaryConverter implementations can be specified by the implementations full class name. + +.To use the supplied the BinaryConverterBitmapLevel2 converter: +[source, bash] +---- +java -jar storage-embedded-tools-storage-converter-{maven-version}.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 +---- + +If more than one BinaryConverter shall be applied the -c option, including the converters must be applied in quotation marks: +[source, bash] +---- +... "-c binaryConverter1, binaryConverter2" +---- + diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java index c1f730e5..b3d1e272 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java @@ -17,6 +17,8 @@ import org.eclipse.serializer.memory.XMemory; import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; import org.eclipse.serializer.persistence.binary.types.Binary; +import org.eclipse.serializer.persistence.binary.types.BinaryLegacyTypeHandlerSupplier; +import org.eclipse.serializer.persistence.types.PersistenceLegacyTypeHandler; import org.eclipse.serializer.persistence.types.PersistenceLoadHandler; import org.eclipse.serializer.persistence.types.PersistenceReferenceLoader; import org.eclipse.serializer.persistence.types.PersistenceStoreHandler; @@ -29,8 +31,12 @@ * It extends {@link AbstractBinaryHandlerCustom} to support binary-based I/O operations * specific to {@link BitmapLevel2}. */ -public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom -{ +public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom implements BinaryLegacyTypeHandlerSupplier +{ + private static final int BITMAP_LEVEL2_VERSION_SIZE = Integer.BYTES; + private static final long BITMAP_LIST_OFFSET = BITMAP_LEVEL2_VERSION_SIZE; + private static final int BITMAP_LEVEL2_BINARY_VERSION = 2; + public static BinaryHandlerBitmapLevel2 New() { return new BinaryHandlerBitmapLevel2(); @@ -46,6 +52,7 @@ public static BinaryHandlerBitmapLevel2 New() super( BitmapLevel2.class, CustomFields( + CustomField(int.class, "version"), bytes("data") ) ); @@ -59,12 +66,12 @@ public static BinaryHandlerBitmapLevel2 New() @Override public BitmapLevel2 create(final Binary data, final PersistenceLoadHandler handler) { - final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()); + final long contentLength = Binary.toBinaryListContentByteLength(data.getBinaryListTotalByteLength(BITMAP_LEVEL2_VERSION_SIZE)); final int fullLength = BitmapLevel2.getTotalLengthFromPersistentLength(XTypes.to_int(contentLength)); final long level2Address = XMemory.allocate(fullLength); final long persistentAddress = BitmapLevel2.toPersistentDataAddress(level2Address); - data.copyToAddress(0, persistentAddress, contentLength); + data.copyToAddress(Binary.toBinaryListElementsOffset(BITMAP_LIST_OFFSET), persistentAddress, contentLength); BitmapLevel2.initializeFromData(level2Address, fullLength); // required to prevent JVM crashes caused by misinterpreted off-heap data. @@ -82,17 +89,18 @@ public void store( ) { // if the parent level3 segment decided to store a level2 segment, it must be stored in any case. - + instance.ensureCompressed(); final long persistentAddress = BitmapLevel2.toPersistentDataAddress(instance.level2Address); final int persistentLength = BitmapLevel2.getPersistentLengthFromTotalLength(instance.totalLength()); data.storeEntityHeader( - Binary.toBinaryListTotalByteLength(persistentLength), - this.typeId(), + Binary.toBinaryListTotalByteLength(persistentLength) + BITMAP_LEVEL2_VERSION_SIZE, + this.typeId(), objectId ); - - data.copyFromAddress(0, persistentAddress, persistentLength); + data.store_int(BITMAP_LEVEL2_BINARY_VERSION); + data.storeListHeader(BITMAP_LIST_OFFSET, persistentLength, persistentLength); + data.copyFromAddress(Binary.toBinaryListElementsOffset(BITMAP_LIST_OFFSET), persistentAddress, persistentLength); } @Override @@ -110,5 +118,11 @@ public void iterateLoadableReferences(final Binary data, final PersistenceRefere { // no-op } + + @Override + public PersistenceLegacyTypeHandler getLegacyTypeHandler() + { + return BinaryLegacyTypeHandlerBitmapLevel2.New(); + } } diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java new file mode 100644 index 00000000..d1264057 --- /dev/null +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java @@ -0,0 +1,92 @@ +package org.eclipse.store.gigamap.types; + +/*- + * #%L + * EclipseStore GigaMap + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; +import org.eclipse.serializer.persistence.binary.types.Binary; +import org.eclipse.serializer.persistence.binary.types.BinaryLegacyTypeHandler; +import org.eclipse.serializer.persistence.types.PersistenceLoadHandler; +import org.eclipse.serializer.persistence.types.PersistenceReferenceLoader; +import org.eclipse.serializer.typing.XTypes; + + +/** + * BinaryHandlerBitmapLevel2 is responsible for managing serialization, deserialization, + * and state updates for objects of type {@link BitmapLevel2} within a persistence framework. + * It extends {@link AbstractBinaryHandlerCustom} to support binary-based I/O operations + * specific to {@link BitmapLevel2}. + */ +public class BinaryLegacyTypeHandlerBitmapLevel2 extends BinaryLegacyTypeHandler.AbstractCustom +{ + public static BinaryLegacyTypeHandlerBitmapLevel2 New() + { + return new BinaryLegacyTypeHandlerBitmapLevel2(); + } + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + BinaryLegacyTypeHandlerBitmapLevel2() + { + super( + BitmapLevel2.class, + CustomFields( + bytes("data") + ) + ); + } + + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public BitmapLevel2 create(final Binary data, final PersistenceLoadHandler handler) + { + final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()); + final int fullLength = BitmapLevel2.getTotalLengthFromPersistentLength(XTypes.to_int(contentLength)); + + final long level2Address = XMemory.allocate(fullLength); + final long persistentAddress = BitmapLevel2.toPersistentDataAddress(level2Address); + data.copyToAddress(0, persistentAddress, contentLength); + BitmapLevel2.initializeFromData(level2Address, fullLength); + + // required to prevent JVM crashes caused by misinterpreted off-heap data. + BitmapLevel2.validateLevel2SegmentType(level2Address); + + return new BitmapLevel2(false, level2Address); + } + + @Override + public void updateState( + final Binary data , + final BitmapLevel2 instance, + final PersistenceLoadHandler handler + ) + { + // there are no references to be set, hence nothing to do here + } + + @Override + public void iterateLoadableReferences(final Binary data, final PersistenceReferenceLoader iterator) + { + // no-op + } + +} diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index e8ce149e..2a8bdc81 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -8,7 +8,7 @@ It is possible to define a different channel count for the target than the sourc ## Building the storage converter standalone jar -To build the converter standalone jar using Maven you need to active the profile **converter-standalone**. +To build the converter standalone jar using Maven you need to activate the profile **converter-standalone**. ```console mvn -Pconverter-standalone clean package @@ -22,7 +22,7 @@ mvn -Pconverter-standalone clean package To configure the input and output storage an [external configuration](https://docs.eclipsestore.io/manual/storage/configuration/index.html#external-configuration) file for each storage is required. ```console -java -jar storage-embedded-tools-storage-converter-4.0.0-beta1.jar sourceCongig.xml targetConfig.xml +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar sourceConfig.ini targetConfig.ini ``` ### StorageConverter.java @@ -35,4 +35,18 @@ org.eclipse.store.storage.embedded.tools.storage.converter.StorageConverter.Stor which gives you some more control on the storage's configurations. +### Converting binary data +To convert the binary representation of persisted objects BinaryConverter implementations can be specified by the implementations full class name. + +using the BinaryConverterBitmapLevel2 converter: + +```console +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 +``` + +If more than one BinaryConverter shall be applied the -c option, including the converters must be applied in quotation marks: + +```console +... "-c binaryConverter1, binaryConverter2" +``` diff --git a/storage/embedded-tools/storage-converter/src/main/java/module-info.java b/storage/embedded-tools/storage-converter/src/main/java/module-info.java index d62ee551..57313a86 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/module-info.java +++ b/storage/embedded-tools/storage-converter/src/main/java/module-info.java @@ -16,4 +16,5 @@ exports org.eclipse.store.storage.embedded.tools.storage.converter; requires transitive org.eclipse.store.storage.embedded.configuration; + requires org.eclipse.serializer.base; } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java new file mode 100644 index 00000000..5ac77305 --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java @@ -0,0 +1,60 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.ByteBuffer; + +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; + +/** + * The Binary Converter converters the input binary data to a new binary format. + */ +public interface BinaryConverter +{ + /** + * Convert the input binary data. + * + * @param bufferIn binary input data. + * @return ByteBuffer containing the converted binary data. + */ + ByteBuffer convert(ByteBuffer bufferIn); + + /** + * Returns a new typeDefinition if the converted + * binary data requires an update of the type definition. + * The new TypeDefinition will be added to the output + * storage type-dictionary. + * + * @return a PersistenceTypeDefinition + */ + public PersistenceTypeDefinition getTypeDefinition(); + + /** + * Returns true if the handler requires the type dictionary to be updated. + * + * @return true if the converter updates type dictionary. + */ + boolean requiresTypeDictionaryUpdate(); + + /** + * Returns true if the converter can process + * the provided type. + * + * @param e PersistenceTypeDescription to check if the type can be processed by the converter. + * @return true if converter is can process the types' data. + */ + boolean matches(PersistenceTypeDescription e); +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java new file mode 100644 index 00000000..0b1fc48d --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java @@ -0,0 +1,133 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.ByteBuffer; +import java.util.List; + +import org.eclipse.serializer.collections.types.XGettingSequence; +import org.eclipse.serializer.collections.types.XImmutableSequence; +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescriptionMember; +import org.eclipse.serializer.util.logging.Logging; +import org.slf4j.Logger; + +/** + * Binary Converter for the initial released version of + * the org.eclipse.store.gigamap.types.BitmapLevel2 binary data format. + * This converter adds the missing list header and a new integer "version" field + * to differ between the original and the converted data. + */ +public class BinaryConverterBitmapLevel2 implements BinaryConverter +{ + private final static Logger logger = Logging.getLogger(BinaryConverter.class); + + private static final int BINARY_SIZE_ADJUSTMENT = 20; + private static final int LIST_SIZE_ADJUSTMENT = 24; + private static final int OUTPUT_BINARY_FORMAT_VERSION = 2; + + private final static String clazz = "org.eclipse.store.gigamap.types.BitmapLevel2"; + private final static List fields = List.of("data"); + + private final static XImmutableSequence newTypeDefinitionMembers = AbstractBinaryHandlerCustom.CustomFields( + AbstractBinaryHandlerCustom.CustomField(int.class, "version"), + AbstractBinaryHandlerCustom.bytes("data")); + + private final long newTypeId; + private final PersistenceTypeDefinition newTypeDefinition; + + public BinaryConverterBitmapLevel2(ConverterTypeDictionary converterTypeDictionary) + { + this.newTypeId = converterTypeDictionary.incrementAndGetMaxTypeID(); + this.newTypeDefinition = converterTypeDictionary.createTypeDictionaryEntry( + this.newTypeId, + clazz, + BinaryConverterBitmapLevel2.newTypeDefinitionMembers); + } + + @Override + public PersistenceTypeDefinition getTypeDefinition() + { + return this.newTypeDefinition; + } + + @Override + public boolean requiresTypeDictionaryUpdate() + { + return true; + } + + @Override + public ByteBuffer convert(final ByteBuffer bufferIn) + { + int pos = bufferIn.position(); + int size = bufferIn.limit() - bufferIn.position(); + + long binarySize = bufferIn.getLong(pos); + long binaryTid = bufferIn.getLong(pos + 8); + long binaryOid = bufferIn.getLong(pos + 16); + + long newSize = size + BINARY_SIZE_ADJUSTMENT; + long newListSize = size - LIST_SIZE_ADJUSTMENT; + + ByteBuffer converted = XMemory.allocateDirectNative(newSize); + + logger.debug("Converting object: binaryTid {}, binaryOid {} , binarySize {} to binaryTid {}, binaryOid {} , binarySize {}", + binaryTid, binaryOid, binarySize, + this.newTypeId, binaryOid, newSize); + + converted.putLong(newSize); // 0 + converted.putLong(this.newTypeId); // 8 + converted.putLong(binaryOid); //16 + + converted.putInt(OUTPUT_BINARY_FORMAT_VERSION); //24 + + converted.putLong(newListSize); //28 + converted.putLong(newListSize);//36 + + converted.put(44, bufferIn, pos + LIST_SIZE_ADJUSTMENT, size - LIST_SIZE_ADJUSTMENT); //44 + converted.limit(44 + size - LIST_SIZE_ADJUSTMENT); + converted.position(0); + + return converted; + } + + @Override + public boolean matches(final PersistenceTypeDescription e) + { + if(clazz.equals(e.typeName())) + { + //existing type definition != new type definition + if(PersistenceTypeDescription.equalDescription(e, this.newTypeDefinition)) + { + return false; + } + + //must contain "data" field + XGettingSequence members = e.allMembers(); + for(int i = 0; i < e.allMembers().size(); i++) + { + if(!BinaryConverterBitmapLevel2.fields.contains(members.at(i).identifier())) return false; + } + + return true; + } + return false; + } +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java new file mode 100644 index 00000000..56a2026f --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java @@ -0,0 +1,69 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.HashMap; + +public class BinaryConverterSelector +{ + private final HashMap converters; + private final ConverterTypeDictionary converterTypeDictionary; + + public BinaryConverterSelector(final ConverterTypeDictionary typeDictionary) + { + this.converters = new HashMap<>(); + this.converterTypeDictionary = typeDictionary; + } + + public void initConverter(final String binaryConverterClassName) + { + MethodHandles.Lookup publicLookup = MethodHandles.publicLookup(); + MethodType mt = MethodType.methodType(void.class, ConverterTypeDictionary.class); + + try + { + Class clazz = Class.forName(binaryConverterClassName); + MethodHandle ch = publicLookup.findConstructor(clazz, mt); + + BinaryConverter converter = (BinaryConverter)ch.invoke(this.converterTypeDictionary); + this.match(converter); + if(converter.requiresTypeDictionaryUpdate()) + { + this.converterTypeDictionary.add(converter.getTypeDefinition()); + } + + } + catch(Throwable e) + { + throw new RuntimeException("Failed to initialize converter: " + binaryConverterClassName, e); + } + } + + private void match(final BinaryConverter converter) + { + this.converterTypeDictionary.entries().forEach( + e -> { if(converter.matches(e)) { + this.converters.put(e.typeId(), converter); + }}); + } + + public BinaryConverter get(final long tid) + { + return this.converters.get(tid); + } +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java new file mode 100644 index 00000000..2d95ce27 --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java @@ -0,0 +1,125 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.chars.VarString; +import org.eclipse.serializer.collections.BulkList; +import org.eclipse.serializer.collections.EqHashEnum; +import org.eclipse.serializer.collections.types.XImmutableSequence; +import org.eclipse.serializer.persistence.binary.types.BinaryFieldLengthResolver; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionCreator; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescriptionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryAssembler; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryParser; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryParser.Default; +import org.eclipse.serializer.persistence.types.PersistenceTypeNameMapper; +import org.eclipse.serializer.persistence.types.PersistenceTypeResolver; +import org.eclipse.serializer.reflect.ClassLoaderProvider; +import org.eclipse.serializer.util.logging.Logging; +import org.slf4j.Logger; + +public class ConverterTypeDictionary +{ + private final static Logger logger = Logging.getLogger(BinaryConverter.class); + private static PersistenceTypeDefinitionCreator typeDefinitionCreator; + private static PersistenceTypeDictionaryAssembler persistenceTypeDictionaryAssembler; + + private final BulkList dictionaryEntries; + private long maxTypeId; + + + public ConverterTypeDictionary(final String sourceTypeDictionary) + { + Default parser = PersistenceTypeDictionaryParser.New( + PersistenceTypeResolver.New(ClassLoaderProvider.System()), + new BinaryFieldLengthResolver.Default(), + PersistenceTypeNameMapper.New()); + + typeDefinitionCreator = PersistenceTypeDefinitionCreator.New(); + persistenceTypeDictionaryAssembler = PersistenceTypeDictionaryAssembler.New(); + + this.dictionaryEntries = BulkList.New(parser.parseTypeDictionaryEntries(sourceTypeDictionary)); + this.maxTypeId = this.initMaxTypeID(); + } + + public BulkList entries() + { + return this.dictionaryEntries; + } + + public long incrementAndGetMaxTypeID() + { + return ++this.maxTypeId; + } + + public PersistenceTypeDefinition createTypeDictionaryEntry( + final long newTypeId, + final String clazz, + final XImmutableSequence typeDefinitionMembers) + { + if(newTypeId == 0) + { + throw new RuntimeException("TypeID not initialized!"); + } + + EqHashEnum members = EqHashEnum.New(PersistenceTypeDescriptionMember.identityHashEqualator()); + for( PersistenceTypeDefinitionMember m : typeDefinitionMembers) + { + members.add(m); + } + + PersistenceTypeDefinition typeDefinition = typeDefinitionCreator.createTypeDefinition( + newTypeId, clazz, clazz, null, members, members); + + VarString typeDictionaryString = VarString.New(); + persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, typeDefinition); + + logger.debug("Assembled type dictionary entry: {}", typeDictionaryString); + + return typeDefinition; + } + + private long initMaxTypeID() + { + long maxId = 0; + + for(PersistenceTypeDescription entry : this.dictionaryEntries) + { + if(entry.typeId() > maxId) maxId = entry.typeId(); + } + + return maxId; + } + + public void add(final PersistenceTypeDefinition typeDefinition) + { + this.entries().add(typeDefinition); + } + + @Override + public String toString() + { + VarString typeDictionaryString = VarString.New(); + for(PersistenceTypeDescription description : this.dictionaryEntries) + { + persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, description); + } + + return typeDictionaryString.toString(); + } +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java index dd1b8965..a24f66ea 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java @@ -15,6 +15,9 @@ */ import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.eclipse.store.storage.embedded.configuration.types.EmbeddedStorageConfiguration; import org.eclipse.store.storage.types.StorageConfiguration; @@ -36,8 +39,13 @@ public class MainUtilStorageConverter "Convert a storage into a new one. The source and the new target storage \n" + "must be specified in storage config files provided as program arguments: \n" + "\n" - + "MainUtilStorageConverter sourceConfig.xml targetConfig.xml" + + "MainUtilStorageConverter sourceConfig.ini targetConfig.ini" + "\n" + + "Optional: provide a list of BinaryConverter implementations as ONE sting include the -c:" + + " \n" + + "\"-c binaryConverter1, binaryConverter2\"" + + "It is import to have the \"-c\" inside the quotation marks, " + + "the converters must be specified using the full class name" ; public static void main(final String[] args) @@ -53,19 +61,49 @@ public static void main(final String[] args) final StorageConfiguration targetConfig = EmbeddedStorageConfiguration.load(dstConfigFile) .createEmbeddedStorageFoundation().getConfiguration(); + String[] binaryConverters = {}; + if(args.length > 2) + { + for(int i = 2; i < args.length; i++) + { + if(args[i].startsWith("-c")) + { + binaryConverters = parseBinaryConverters(args, i); + } + } + } System.out.println("Source storage configuration: " + srcConfigFile); System.out.println("Target storage configuration: " + dstConfigFile); + System.out.println("Binary format converters: " + Arrays.toString(binaryConverters)); - final StorageConverter storageConverter = new StorageConverter(sourceConfig, targetConfig); + final StorageConverter storageConverter = new StorageConverter(sourceConfig, targetConfig, binaryConverters); storageConverter.start(); System.out.println("Storage conversion finished!"); } + private static String[] parseBinaryConverters(String[] args, int startIndex) + { + List binaryConverters = new ArrayList<>(); + + for(int i = startIndex + 1; i < args.length; i++) + { + if(args[i].startsWith("-")) + { + return binaryConverters.toArray(new String[0]); + } + else + { + binaryConverters.add(args[i]); + } + } + return binaryConverters.toArray(new String[0]); + } + private static void verifyArguments(final String[] args) { - if(args.length == 2) + if(args.length >= 2) { if(new File(args[0]).canRead()) { @@ -91,5 +129,4 @@ private static void verifyArguments(final String[] args) System.out.println(HELP); System.exit(-1); } - } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java index 5e69c16b..4f4e056b 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java @@ -55,6 +55,9 @@ public class StorageConverter private ByteBuffer bufferIn; private final StorageConverterTarget target; + + private final BinaryConverterSelector binaryConverterSelector; + private final ConverterTypeDictionary converterTypeDictionary; /** * Helper class describing a single entity in the current processed file @@ -88,14 +91,48 @@ public StorageConverter ( final StorageConfiguration targetStorageConfiguration ) { - this.srcFileProvider = sourceStorageConfiguration.fileProvider(); - this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); + this.srcFileProvider = sourceStorageConfiguration.fileProvider(); + this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); - this.srcInventories = this.createChannelInventories(); - this.processedIds = new HashSet<>(); - this.currentFileEntities = new HashMap<>(); + this.converterTypeDictionary = new ConverterTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.binaryConverterSelector = new BinaryConverterSelector(this.converterTypeDictionary); + + this.srcInventories = this.createChannelInventories(); + this.processedIds = new HashSet<>(); + this.currentFileEntities = new HashMap<>(); - this.target = new StorageConverterTarget(targetStorageConfiguration); + this.target = new StorageConverterTarget(targetStorageConfiguration); + } + + /** + * Converts a EmbeddedStorage into another one. + * + * @param sourceStorageConfiguration configuration for storage to be converted. + * @param targetStorageConfiguration configuration of the target storage. + * @param binaryConverters list of BinaryConvert class names that should be applied. + */ + public StorageConverter ( + final StorageConfiguration sourceStorageConfiguration, + final StorageConfiguration targetStorageConfiguration, + final String[] binaryConverters + ) + { + this.srcFileProvider = sourceStorageConfiguration.fileProvider(); + this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); + + this.converterTypeDictionary = new ConverterTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.binaryConverterSelector = new BinaryConverterSelector(this.converterTypeDictionary); + + for(String converter : binaryConverters) { + this.binaryConverterSelector.initConverter(converter); + } + + + this.srcInventories = this.createChannelInventories(); + this.processedIds = new HashSet<>(); + this.currentFileEntities = new HashMap<>(); + + this.target = new StorageConverterTarget(targetStorageConfiguration); } @@ -115,8 +152,7 @@ public void start() private void copyTypeDictionary() { - this.target - .storeTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.target.storeTypeDictionary(this.converterTypeDictionary.toString()); } private void close() @@ -131,7 +167,18 @@ private void transferEntity(final long oid, final FileEntity entity) this.bufferIn.limit((int) (entity.offset + entity.length)); this.bufferIn.position((int) entity.offset); - this.target.transferBytes(this.bufferIn, oid); + long tid = XMemory.get_long(XMemory.getDirectByteBufferAddress(this.bufferIn) + entity.offset + 8); + + BinaryConverter converter = this.binaryConverterSelector.get(tid); + if(converter!=null) + { + ByteBuffer converted = converter.convert(this.bufferIn); + this.target.transferBytes(converted, oid); + } + else + { + this.target.transferBytes(this.bufferIn, oid); + } this.processedIds.add(oid); }