From 3de79b099cae410e5c42606fac57edc8690d6432 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:27:59 +0100 Subject: [PATCH 01/13] Fixing erroneous BitmapLevel2 Binary Format changed BitmapLevel2 binary format added BinaryConverter to storage converter --- .../types/BinaryHandlerBitmapLevel2.java | 16 ++- .../storage-converter/README.md | 7 + .../src/main/java/module-info.java | 1 + .../storage/converter/BinaryConverter.java | 60 +++++++++ .../BinaryConverterBitmapLevel2.java | 121 ++++++++++++++++++ .../converter/BinaryConverterSelector.java | 63 +++++++++ .../converter/ConverterTypeDictionary.java | 120 +++++++++++++++++ .../converter/MainUtilStorageConverter.java | 36 +++++- .../storage/converter/StorageConverter.java | 65 ++++++++-- 9 files changed, 471 insertions(+), 18 deletions(-) create mode 100644 storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java create mode 100644 storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java create mode 100644 storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java create mode 100644 storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java index c1f730e5..1bf56cf0 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java @@ -46,6 +46,7 @@ public static BinaryHandlerBitmapLevel2 New() super( BitmapLevel2.class, CustomFields( + CustomField(int.class, "version"), bytes("data") ) ); @@ -59,12 +60,12 @@ public static BinaryHandlerBitmapLevel2 New() @Override public BitmapLevel2 create(final Binary data, final PersistenceLoadHandler handler) { - final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()); + final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()) - 20; final int fullLength = BitmapLevel2.getTotalLengthFromPersistentLength(XTypes.to_int(contentLength)); final long level2Address = XMemory.allocate(fullLength); final long persistentAddress = BitmapLevel2.toPersistentDataAddress(level2Address); - data.copyToAddress(0, persistentAddress, contentLength); + data.copyToAddress(Binary.toBinaryListElementsOffset(4), persistentAddress, contentLength); BitmapLevel2.initializeFromData(level2Address, fullLength); // required to prevent JVM crashes caused by misinterpreted off-heap data. @@ -82,17 +83,18 @@ public void store( ) { // if the parent level3 segment decided to store a level2 segment, it must be stored in any case. - + instance.ensureCompressed(); final long persistentAddress = BitmapLevel2.toPersistentDataAddress(instance.level2Address); final int persistentLength = BitmapLevel2.getPersistentLengthFromTotalLength(instance.totalLength()); data.storeEntityHeader( - Binary.toBinaryListTotalByteLength(persistentLength), - this.typeId(), + Binary.toBinaryListTotalByteLength(persistentLength) + 20, + this.typeId(), objectId ); - - data.copyFromAddress(0, persistentAddress, persistentLength); + data.store_int(2); + data.storeListHeader(4, persistentLength, persistentLength); + data.copyFromAddress(Binary.toBinaryListElementsOffset(4), persistentAddress, persistentLength); } @Override diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index 4d3bf767..edf537e5 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -35,4 +35,11 @@ org.eclipse.store.storage.embedded.tools.storage.converter.StorageConverter.Stor which gives you some more control on the storage's configurations. +### converting binary data + +To convert the binary representation of peristeted objects BinaryConverter implemenations can be specified: + +```console +java -jar storage-embedded-tools-storage-converter-3.1.0.jar sourceCongig.xml targetConfig.xml -c BinaryConverterAImpl.class BinaryConverterBImpl.class +``` diff --git a/storage/embedded-tools/storage-converter/src/main/java/module-info.java b/storage/embedded-tools/storage-converter/src/main/java/module-info.java index d62ee551..57313a86 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/module-info.java +++ b/storage/embedded-tools/storage-converter/src/main/java/module-info.java @@ -16,4 +16,5 @@ exports org.eclipse.store.storage.embedded.tools.storage.converter; requires transitive org.eclipse.store.storage.embedded.configuration; + requires org.eclipse.serializer.base; } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java new file mode 100644 index 00000000..5ac77305 --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverter.java @@ -0,0 +1,60 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.ByteBuffer; + +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; + +/** + * The Binary Converter converters the input binary data to a new binary format. + */ +public interface BinaryConverter +{ + /** + * Convert the input binary data. + * + * @param bufferIn binary input data. + * @return ByteBuffer containing the converted binary data. + */ + ByteBuffer convert(ByteBuffer bufferIn); + + /** + * Returns a new typeDefinition if the converted + * binary data requires an update of the type definition. + * The new TypeDefinition will be added to the output + * storage type-dictionary. + * + * @return a PersistenceTypeDefinition + */ + public PersistenceTypeDefinition getTypeDefinition(); + + /** + * Returns true if the handler requires the type dictionary to be updated. + * + * @return true if the converter updates type dictionary. + */ + boolean requiresTypeDictionaryUpdate(); + + /** + * Returns true if the converter can process + * the provided type. + * + * @param e PersistenceTypeDescription to check if the type can be processed by the converter. + * @return true if converter is can process the types' data. + */ + boolean matches(PersistenceTypeDescription e); +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java new file mode 100644 index 00000000..acb2e694 --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java @@ -0,0 +1,121 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.ByteBuffer; +import java.util.List; + +import org.eclipse.serializer.collections.types.XGettingSequence; +import org.eclipse.serializer.collections.types.XImmutableSequence; +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescriptionMember; +import org.eclipse.serializer.util.logging.Logging; +import org.slf4j.Logger; + +public class BinaryConverterBitmapLevel2 implements BinaryConverter +{ + private final static Logger logger = Logging.getLogger(BinaryConverter.class); + + private static final int BINARY_SIZE_ADJUSTMENT = 20; + private static final int LIST_SIZE_ADJUSTMENT = 24; + private static final int OUTPUT_BINARY_FORMAT_VERSION = 2; + + private final static String clazz = "org.eclipse.store.gigamap.types.BitmapLevel2"; + private final static List fields = List.of("data"); + + private final static XImmutableSequence newTypeDefinitionMembers = AbstractBinaryHandlerCustom.CustomFields( + AbstractBinaryHandlerCustom.CustomField(int.class, "version"), + AbstractBinaryHandlerCustom.bytes("data")); + + private final long newTypeId; + private final PersistenceTypeDefinition newTypeDefinition; + + public BinaryConverterBitmapLevel2(ConverterTypeDictionary converterTypeDictionary) { + this.newTypeId = converterTypeDictionary.incrementAndGetMaxTypeID(); + this.newTypeDefinition = converterTypeDictionary.createTypeDictionaryEntry( + this.newTypeId, + clazz, + BinaryConverterBitmapLevel2.newTypeDefinitionMembers); + } + + @Override + public PersistenceTypeDefinition getTypeDefinition() { + return this.newTypeDefinition; + } + + @Override + public boolean requiresTypeDictionaryUpdate() { + return true; + } + + @Override + public ByteBuffer convert(final ByteBuffer bufferIn) { + + int pos = bufferIn.position(); + int size = bufferIn.limit() - bufferIn.position(); + + long binarySize = bufferIn.getLong(pos); + long binaryTid = bufferIn.getLong(pos + 8); + long binaryOid = bufferIn.getLong(pos + 16); + + long newSize = size + BINARY_SIZE_ADJUSTMENT; + long newListSize = size - LIST_SIZE_ADJUSTMENT; + + ByteBuffer converted = XMemory.allocateDirectNative(newSize); + + logger.debug("Converting object: binaryTid {}, binaryOid {} , binarySize {} to binaryTid {}, binaryOid {} , binarySize {}", + binaryTid, binaryOid, binarySize, + this.newTypeId, binaryOid, newSize); + + converted.putLong(newSize); // 0 + converted.putLong(this.newTypeId); // 8 + converted.putLong(binaryOid); //16 + + converted.putInt(OUTPUT_BINARY_FORMAT_VERSION); //24 + + converted.putLong(newListSize); //28 + converted.putLong(newListSize);//36 + + converted.put(44, bufferIn, pos + LIST_SIZE_ADJUSTMENT, size - LIST_SIZE_ADJUSTMENT); //44 + converted.limit(44 + size - LIST_SIZE_ADJUSTMENT); + converted.position(0); + + return converted; + } + + @Override + public boolean matches(final PersistenceTypeDescription e) { + if(clazz.equals(e.typeName())) + { + //existing type definition != new type definition + if(PersistenceTypeDescription.equalDescription(e, this.newTypeDefinition)) { + return false; + } + + //must contain "data" field + XGettingSequence members = e.allMembers(); + for(int i = 0; i < e.allMembers().size(); i++) { + if(!BinaryConverterBitmapLevel2.fields.contains(members.at(i).identifier())) return false; + } + + return true; + } + return false; + } +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java new file mode 100644 index 00000000..92f90583 --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java @@ -0,0 +1,63 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.HashMap; + +public class BinaryConverterSelector +{ + private final HashMap converters; + private final ConverterTypeDictionary converterTypeDictionary; + + public BinaryConverterSelector(final ConverterTypeDictionary typeDictionary) { + this.converters = new HashMap<>(); + this.converterTypeDictionary = typeDictionary; + } + + public void initConverter(final String binaryConverterClassName) { + + MethodHandles.Lookup publicLookup = MethodHandles.publicLookup(); + MethodType mt = MethodType.methodType(void.class, ConverterTypeDictionary.class); + + try { + Class clazz = Class.forName(binaryConverterClassName); + MethodHandle ch = publicLookup.findConstructor(clazz, mt); + + BinaryConverter converter = (BinaryConverter)ch.invoke(this.converterTypeDictionary); + this.match(converter); + if(converter.requiresTypeDictionaryUpdate()) { + this.converterTypeDictionary.add(converter.getTypeDefinition()); + } + + } catch(Throwable e) { + throw new RuntimeException("Failed to initialize converter: " + binaryConverterClassName, e); + } + } + + private void match(final BinaryConverter converter) { + this.converterTypeDictionary.entries().forEach( + e -> { if(converter.matches(e)) { + this.converters.put(e.typeId(), converter); + }}); + } + + public BinaryConverter get(final long tid) { + return this.converters.get(tid); + } + +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java new file mode 100644 index 00000000..83f8ff5c --- /dev/null +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java @@ -0,0 +1,120 @@ +package org.eclipse.store.storage.embedded.tools.storage.converter; + +/*- + * #%L + * EclipseStore Storage Embedded Tools Storage Converter + * %% + * Copyright (C) 2023 - 2026 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.chars.VarString; +import org.eclipse.serializer.collections.BulkList; +import org.eclipse.serializer.collections.EqHashEnum; +import org.eclipse.serializer.collections.types.XImmutableSequence; +import org.eclipse.serializer.persistence.binary.types.BinaryFieldLengthResolver; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionCreator; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinitionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescription; +import org.eclipse.serializer.persistence.types.PersistenceTypeDescriptionMember; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryAssembler; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryParser; +import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryParser.Default; +import org.eclipse.serializer.persistence.types.PersistenceTypeNameMapper; +import org.eclipse.serializer.persistence.types.PersistenceTypeResolver; +import org.eclipse.serializer.reflect.ClassLoaderProvider; +import org.eclipse.serializer.util.logging.Logging; +import org.slf4j.Logger; + +public class ConverterTypeDictionary +{ + private final static Logger logger = Logging.getLogger(BinaryConverter.class); + private static PersistenceTypeDefinitionCreator typeDefinitionCreator; + private static PersistenceTypeDictionaryAssembler persistenceTypeDictionaryAssembler; + + private final BulkList dictionaryEntries; + private long maxTypeId; + + + public ConverterTypeDictionary(final String sourceTypeDictionary) { + Default parser = PersistenceTypeDictionaryParser.New( + PersistenceTypeResolver.New(ClassLoaderProvider.System()), + new BinaryFieldLengthResolver.Default(), + PersistenceTypeNameMapper.New()); + + typeDefinitionCreator = PersistenceTypeDefinitionCreator.New(); + persistenceTypeDictionaryAssembler = PersistenceTypeDictionaryAssembler.New(); + + this.dictionaryEntries = BulkList.New(parser.parseTypeDictionaryEntries(sourceTypeDictionary)); + this.maxTypeId = this.initMaxTypeID(); + } + + public BulkList entries() { + return this.dictionaryEntries; + } + + public long incrementAndGetMaxTypeID() { + return ++this.maxTypeId; + } + + public PersistenceTypeDefinition createTypeDictionaryEntry( + final long newTypeId, + final String clazz, + final XImmutableSequence typeDefinitionMembers) + { + if(newTypeId == 0) { + throw new RuntimeException("TypeID not initialized!"); + } + + EqHashEnum members = EqHashEnum.New(PersistenceTypeDescriptionMember.identityHashEqualator()); + for( PersistenceTypeDefinitionMember m : typeDefinitionMembers) { + members.add(m); + } + + PersistenceTypeDefinition typeDefinition = typeDefinitionCreator.createTypeDefinition( + newTypeId, clazz, clazz, null, members, members); + + + VarString typeDictionaryString = VarString.New(); + persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, typeDefinition); + + logger.debug("Assembled type dictionary entry: {}", typeDictionaryString); + + return typeDefinition; + } + + private long initMaxTypeID() { + + long maxId = 0; + + for(PersistenceTypeDescription entry : this.dictionaryEntries) { + if(entry.typeId() > maxId) maxId = entry.typeId(); + } + + return maxId; + } + + public void add(final PersistenceTypeDefinition typeDefinition) { + this.entries().add(typeDefinition); + + } + + @Override + public String toString() { + + VarString typeDictionaryString = VarString.New(); + for(PersistenceTypeDescription description : this.dictionaryEntries) { + persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, description); + } + + return typeDictionaryString.toString(); + } + +} diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java index dd1b8965..29314ae2 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java @@ -15,6 +15,9 @@ */ import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.eclipse.store.storage.embedded.configuration.types.EmbeddedStorageConfiguration; import org.eclipse.store.storage.types.StorageConfiguration; @@ -38,6 +41,10 @@ public class MainUtilStorageConverter + "\n" + "MainUtilStorageConverter sourceConfig.xml targetConfig.xml" + "\n" + + "Optional: provide a list of BinaryConverter implementations as ONE sting include the -c:" + + " \n" + + "\"-c binaryConverter1, binaryConverter2\"" + + "It is import to have the \"-c\" inside the quotation marks!" ; public static void main(final String[] args) @@ -53,19 +60,44 @@ public static void main(final String[] args) final StorageConfiguration targetConfig = EmbeddedStorageConfiguration.load(dstConfigFile) .createEmbeddedStorageFoundation().getConfiguration(); + String[] binaryConverters = {}; + if(args.length > 2) + { + for(int i = 2; i < args.length; i++) + { + if(args[i].startsWith("-c")) + { + binaryConverters = parseBinaryConverters(args, i); + } + } + } System.out.println("Source storage configuration: " + srcConfigFile); System.out.println("Target storage configuration: " + dstConfigFile); + System.out.println("Binary format converters: " + Arrays.toString(binaryConverters)); - final StorageConverter storageConverter = new StorageConverter(sourceConfig, targetConfig); + final StorageConverter storageConverter = new StorageConverter(sourceConfig, targetConfig, binaryConverters); storageConverter.start(); System.out.println("Storage conversion finished!"); } + private static String[] parseBinaryConverters(String[] args, int startIndex) { + + List binaryConverters = new ArrayList<>(); + + for(int i = startIndex + 1; i < args.length; i++) { + if(args[i].startsWith("-")) return binaryConverters.toArray(new String[0]); + else { + binaryConverters.add(args[i]); + } + } + return binaryConverters.toArray(new String[0]); + } + private static void verifyArguments(final String[] args) { - if(args.length == 2) + if(args.length >= 2) { if(new File(args[0]).canRead()) { diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java index 5e69c16b..4f4e056b 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/StorageConverter.java @@ -55,6 +55,9 @@ public class StorageConverter private ByteBuffer bufferIn; private final StorageConverterTarget target; + + private final BinaryConverterSelector binaryConverterSelector; + private final ConverterTypeDictionary converterTypeDictionary; /** * Helper class describing a single entity in the current processed file @@ -88,14 +91,48 @@ public StorageConverter ( final StorageConfiguration targetStorageConfiguration ) { - this.srcFileProvider = sourceStorageConfiguration.fileProvider(); - this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); + this.srcFileProvider = sourceStorageConfiguration.fileProvider(); + this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); - this.srcInventories = this.createChannelInventories(); - this.processedIds = new HashSet<>(); - this.currentFileEntities = new HashMap<>(); + this.converterTypeDictionary = new ConverterTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.binaryConverterSelector = new BinaryConverterSelector(this.converterTypeDictionary); + + this.srcInventories = this.createChannelInventories(); + this.processedIds = new HashSet<>(); + this.currentFileEntities = new HashMap<>(); - this.target = new StorageConverterTarget(targetStorageConfiguration); + this.target = new StorageConverterTarget(targetStorageConfiguration); + } + + /** + * Converts a EmbeddedStorage into another one. + * + * @param sourceStorageConfiguration configuration for storage to be converted. + * @param targetStorageConfiguration configuration of the target storage. + * @param binaryConverters list of BinaryConvert class names that should be applied. + */ + public StorageConverter ( + final StorageConfiguration sourceStorageConfiguration, + final StorageConfiguration targetStorageConfiguration, + final String[] binaryConverters + ) + { + this.srcFileProvider = sourceStorageConfiguration.fileProvider(); + this.srcChannelCount = sourceStorageConfiguration.channelCountProvider().getChannelCount(); + + this.converterTypeDictionary = new ConverterTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.binaryConverterSelector = new BinaryConverterSelector(this.converterTypeDictionary); + + for(String converter : binaryConverters) { + this.binaryConverterSelector.initConverter(converter); + } + + + this.srcInventories = this.createChannelInventories(); + this.processedIds = new HashSet<>(); + this.currentFileEntities = new HashMap<>(); + + this.target = new StorageConverterTarget(targetStorageConfiguration); } @@ -115,8 +152,7 @@ public void start() private void copyTypeDictionary() { - this.target - .storeTypeDictionary(this.srcFileProvider.provideTypeDictionaryIoHandler().loadTypeDictionary()); + this.target.storeTypeDictionary(this.converterTypeDictionary.toString()); } private void close() @@ -131,7 +167,18 @@ private void transferEntity(final long oid, final FileEntity entity) this.bufferIn.limit((int) (entity.offset + entity.length)); this.bufferIn.position((int) entity.offset); - this.target.transferBytes(this.bufferIn, oid); + long tid = XMemory.get_long(XMemory.getDirectByteBufferAddress(this.bufferIn) + entity.offset + 8); + + BinaryConverter converter = this.binaryConverterSelector.get(tid); + if(converter!=null) + { + ByteBuffer converted = converter.convert(this.bufferIn); + this.target.transferBytes(converted, oid); + } + else + { + this.target.transferBytes(this.bufferIn, oid); + } this.processedIds.add(oid); } From 5d9517451a34c4f8dd34c0a2c908c50e13c56357 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Mon, 19 Jan 2026 15:00:24 +0100 Subject: [PATCH 02/13] typos --- storage/embedded-tools/storage-converter/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index edf537e5..ffd5ee6e 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -8,7 +8,7 @@ It is possible to define a different channel count for the target than the sourc ## Building the storage converter standalone jar -To build the converter standalone jar using Maven you need to active the profile **converter-standalone**. +To build the converter standalone jar using Maven you need to activate the profile **converter-standalone**. ```console mvn -Pconverter-standalone clean package @@ -37,7 +37,7 @@ which gives you some more control on the storage's configurations. ### converting binary data -To convert the binary representation of peristeted objects BinaryConverter implemenations can be specified: +To convert the binary representation of persisted objects BinaryConverter implementations can be specified: ```console java -jar storage-embedded-tools-storage-converter-3.1.0.jar sourceCongig.xml targetConfig.xml -c BinaryConverterAImpl.class BinaryConverterBImpl.class From a6ff3d5830ac5a1c7ea8ffea5b5238247f5d5964 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Tue, 20 Jan 2026 07:36:34 +0100 Subject: [PATCH 03/13] brackets... --- .../BinaryConverterBitmapLevel2.java | 28 ++++++++++---- .../converter/BinaryConverterSelector.java | 24 +++++++----- .../converter/ConverterTypeDictionary.java | 37 +++++++++++-------- .../converter/MainUtilStorageConverter.java | 16 +++++--- 4 files changed, 66 insertions(+), 39 deletions(-) diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java index acb2e694..0b1fc48d 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterBitmapLevel2.java @@ -28,6 +28,12 @@ import org.eclipse.serializer.util.logging.Logging; import org.slf4j.Logger; +/** + * Binary Converter for the initial released version of + * the org.eclipse.store.gigamap.types.BitmapLevel2 binary data format. + * This converter adds the missing list header and a new integer "version" field + * to differ between the original and the converted data. + */ public class BinaryConverterBitmapLevel2 implements BinaryConverter { private final static Logger logger = Logging.getLogger(BinaryConverter.class); @@ -46,7 +52,8 @@ public class BinaryConverterBitmapLevel2 implements BinaryConverter private final long newTypeId; private final PersistenceTypeDefinition newTypeDefinition; - public BinaryConverterBitmapLevel2(ConverterTypeDictionary converterTypeDictionary) { + public BinaryConverterBitmapLevel2(ConverterTypeDictionary converterTypeDictionary) + { this.newTypeId = converterTypeDictionary.incrementAndGetMaxTypeID(); this.newTypeDefinition = converterTypeDictionary.createTypeDictionaryEntry( this.newTypeId, @@ -55,18 +62,20 @@ public BinaryConverterBitmapLevel2(ConverterTypeDictionary converterTypeDictiona } @Override - public PersistenceTypeDefinition getTypeDefinition() { + public PersistenceTypeDefinition getTypeDefinition() + { return this.newTypeDefinition; } @Override - public boolean requiresTypeDictionaryUpdate() { + public boolean requiresTypeDictionaryUpdate() + { return true; } @Override - public ByteBuffer convert(final ByteBuffer bufferIn) { - + public ByteBuffer convert(final ByteBuffer bufferIn) + { int pos = bufferIn.position(); int size = bufferIn.limit() - bufferIn.position(); @@ -100,17 +109,20 @@ public ByteBuffer convert(final ByteBuffer bufferIn) { } @Override - public boolean matches(final PersistenceTypeDescription e) { + public boolean matches(final PersistenceTypeDescription e) + { if(clazz.equals(e.typeName())) { //existing type definition != new type definition - if(PersistenceTypeDescription.equalDescription(e, this.newTypeDefinition)) { + if(PersistenceTypeDescription.equalDescription(e, this.newTypeDefinition)) + { return false; } //must contain "data" field XGettingSequence members = e.allMembers(); - for(int i = 0; i < e.allMembers().size(); i++) { + for(int i = 0; i < e.allMembers().size(); i++) + { if(!BinaryConverterBitmapLevel2.fields.contains(members.at(i).identifier())) return false; } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java index 92f90583..56a2026f 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/BinaryConverterSelector.java @@ -24,40 +24,46 @@ public class BinaryConverterSelector private final HashMap converters; private final ConverterTypeDictionary converterTypeDictionary; - public BinaryConverterSelector(final ConverterTypeDictionary typeDictionary) { + public BinaryConverterSelector(final ConverterTypeDictionary typeDictionary) + { this.converters = new HashMap<>(); this.converterTypeDictionary = typeDictionary; } - public void initConverter(final String binaryConverterClassName) { - + public void initConverter(final String binaryConverterClassName) + { MethodHandles.Lookup publicLookup = MethodHandles.publicLookup(); MethodType mt = MethodType.methodType(void.class, ConverterTypeDictionary.class); - try { + try + { Class clazz = Class.forName(binaryConverterClassName); MethodHandle ch = publicLookup.findConstructor(clazz, mt); BinaryConverter converter = (BinaryConverter)ch.invoke(this.converterTypeDictionary); this.match(converter); - if(converter.requiresTypeDictionaryUpdate()) { + if(converter.requiresTypeDictionaryUpdate()) + { this.converterTypeDictionary.add(converter.getTypeDefinition()); } - } catch(Throwable e) { + } + catch(Throwable e) + { throw new RuntimeException("Failed to initialize converter: " + binaryConverterClassName, e); } } - private void match(final BinaryConverter converter) { + private void match(final BinaryConverter converter) + { this.converterTypeDictionary.entries().forEach( e -> { if(converter.matches(e)) { this.converters.put(e.typeId(), converter); }}); } - public BinaryConverter get(final long tid) { + public BinaryConverter get(final long tid) + { return this.converters.get(tid); } - } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java index 83f8ff5c..2d95ce27 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/ConverterTypeDictionary.java @@ -43,7 +43,8 @@ public class ConverterTypeDictionary private long maxTypeId; - public ConverterTypeDictionary(final String sourceTypeDictionary) { + public ConverterTypeDictionary(final String sourceTypeDictionary) + { Default parser = PersistenceTypeDictionaryParser.New( PersistenceTypeResolver.New(ClassLoaderProvider.System()), new BinaryFieldLengthResolver.Default(), @@ -56,11 +57,13 @@ public ConverterTypeDictionary(final String sourceTypeDictionary) { this.maxTypeId = this.initMaxTypeID(); } - public BulkList entries() { + public BulkList entries() + { return this.dictionaryEntries; } - public long incrementAndGetMaxTypeID() { + public long incrementAndGetMaxTypeID() + { return ++this.maxTypeId; } @@ -69,19 +72,20 @@ public PersistenceTypeDefinition createTypeDictionaryEntry( final String clazz, final XImmutableSequence typeDefinitionMembers) { - if(newTypeId == 0) { + if(newTypeId == 0) + { throw new RuntimeException("TypeID not initialized!"); } EqHashEnum members = EqHashEnum.New(PersistenceTypeDescriptionMember.identityHashEqualator()); - for( PersistenceTypeDefinitionMember m : typeDefinitionMembers) { + for( PersistenceTypeDefinitionMember m : typeDefinitionMembers) + { members.add(m); } PersistenceTypeDefinition typeDefinition = typeDefinitionCreator.createTypeDefinition( newTypeId, clazz, clazz, null, members, members); - - + VarString typeDictionaryString = VarString.New(); persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, typeDefinition); @@ -90,31 +94,32 @@ public PersistenceTypeDefinition createTypeDictionaryEntry( return typeDefinition; } - private long initMaxTypeID() { - + private long initMaxTypeID() + { long maxId = 0; - for(PersistenceTypeDescription entry : this.dictionaryEntries) { + for(PersistenceTypeDescription entry : this.dictionaryEntries) + { if(entry.typeId() > maxId) maxId = entry.typeId(); } return maxId; } - public void add(final PersistenceTypeDefinition typeDefinition) { + public void add(final PersistenceTypeDefinition typeDefinition) + { this.entries().add(typeDefinition); - } @Override - public String toString() { - + public String toString() + { VarString typeDictionaryString = VarString.New(); - for(PersistenceTypeDescription description : this.dictionaryEntries) { + for(PersistenceTypeDescription description : this.dictionaryEntries) + { persistenceTypeDictionaryAssembler.assembleTypeDescription(typeDictionaryString, description); } return typeDictionaryString.toString(); } - } diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java index 29314ae2..982ae79e 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java @@ -82,13 +82,18 @@ public static void main(final String[] args) System.out.println("Storage conversion finished!"); } - private static String[] parseBinaryConverters(String[] args, int startIndex) { - + private static String[] parseBinaryConverters(String[] args, int startIndex) + { List binaryConverters = new ArrayList<>(); - for(int i = startIndex + 1; i < args.length; i++) { - if(args[i].startsWith("-")) return binaryConverters.toArray(new String[0]); - else { + for(int i = startIndex + 1; i < args.length; i++) + { + if(args[i].startsWith("-")) + { + return binaryConverters.toArray(new String[0]); + } + else + { binaryConverters.add(args[i]); } } @@ -123,5 +128,4 @@ private static void verifyArguments(final String[] args) System.out.println(HELP); System.exit(-1); } - } From 236bf78e0b0cef4352f0dc85fe2c65a6fa79c034 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:14:23 +0100 Subject: [PATCH 04/13] Updated Handler and new LegacyHandler --- .../BinaryLegacyTypeHandlerBitmapLevel2.java | 93 +++++++++++++++++++ .../store/gigamap/types/BitmapLevel2.java | 24 +++-- 2 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java new file mode 100644 index 00000000..81df8bfc --- /dev/null +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java @@ -0,0 +1,93 @@ +package org.eclipse.store.gigamap.types; + +/*- + * #%L + * EclipseStore GigaMap + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; +import org.eclipse.serializer.persistence.binary.types.Binary; +import org.eclipse.serializer.persistence.binary.types.BinaryLegacyTypeHandler; +import org.eclipse.serializer.persistence.types.PersistenceLoadHandler; +import org.eclipse.serializer.persistence.types.PersistenceReferenceLoader; +import org.eclipse.serializer.typing.XTypes; + + +/** + * BinaryHandlerBitmapLevel2 is responsible for managing serialization, deserialization, + * and state updates for objects of type {@link BitmapLevel2} within a persistence framework. + * It extends {@link AbstractBinaryHandlerCustom} to support binary-based I/O operations + * specific to {@link BitmapLevel2}. + */ +public class BinaryLegacyTypeHandlerBitmapLevel2 extends BinaryLegacyTypeHandler.AbstractCustom +{ + public static BinaryLegacyTypeHandlerBitmapLevel2 New() + { + return new BinaryLegacyTypeHandlerBitmapLevel2(); + } + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + BinaryLegacyTypeHandlerBitmapLevel2() + { + super( + BitmapLevel2.class, + CustomFields( + CustomField(int.class, "version"), + bytes("data") + ) + ); + } + + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public BitmapLevel2 create(final Binary data, final PersistenceLoadHandler handler) + { + final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()); + final int fullLength = BitmapLevel2.getTotalLengthFromPersistentLength(XTypes.to_int(contentLength)); + + final long level2Address = XMemory.allocate(fullLength); + final long persistentAddress = BitmapLevel2.toPersistentDataAddress(level2Address); + data.copyToAddress(0, persistentAddress, contentLength); + BitmapLevel2.initializeFromData(level2Address, fullLength); + + // required to prevent JVM crashes caused by misinterpreted off-heap data. + BitmapLevel2.validateLevel2SegmentType(level2Address); + + return new BitmapLevel2(false, level2Address); + } + + @Override + public void updateState( + final Binary data , + final BitmapLevel2 instance, + final PersistenceLoadHandler handler + ) + { + // there are no references to be set, hence nothing to do here + } + + @Override + public void iterateLoadableReferences(final Binary data, final PersistenceReferenceLoader iterator) + { + // no-op + } + +} diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java index d757a69c..b099fd1a 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java @@ -1,5 +1,15 @@ package org.eclipse.store.gigamap.types; +import java.util.function.Consumer; + +import org.eclipse.serializer.chars.VarString; +import org.eclipse.serializer.math.XMath; +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandler; +import org.eclipse.serializer.persistence.types.Storer; +import org.eclipse.serializer.persistence.types.Unpersistable; +import org.eclipse.serializer.typing.XTypes; + /*- * #%L * EclipseStore GigaMap @@ -15,15 +25,6 @@ */ import org.eclipse.store.gigamap.exceptions.BitmapLevel2Exception; -import org.eclipse.serializer.chars.VarString; -import org.eclipse.serializer.math.XMath; -import org.eclipse.serializer.memory.XMemory; -import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandler; -import org.eclipse.serializer.persistence.types.Storer; -import org.eclipse.serializer.persistence.types.Unpersistable; -import org.eclipse.serializer.typing.XTypes; - -import java.util.function.Consumer; /** @@ -130,6 +131,11 @@ static BinaryTypeHandler provideTypeHandler() return BinaryHandlerBitmapLevel2.New(); } + static BinaryTypeHandler provideLegacyTypeHandler() + { + return BinaryLegacyTypeHandlerBitmapLevel2.New(); + } + private static void setEntryHeader(final long level1EntryAddress, final byte entryType) { XMemory.set_byte(level1EntryAddress + ENTRY_OFFSET_HEADER, entryType); From 2c52fae53ffe04d0f80a71fb46668d0afff857f2 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Fri, 13 Feb 2026 12:32:01 +0100 Subject: [PATCH 05/13] Update BinaryLegacyTypeHandlerBitmapLevel2.java --- .../store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java | 1 - 1 file changed, 1 deletion(-) diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java index 81df8bfc..d1264057 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryLegacyTypeHandlerBitmapLevel2.java @@ -46,7 +46,6 @@ public static BinaryLegacyTypeHandlerBitmapLevel2 New() super( BitmapLevel2.class, CustomFields( - CustomField(int.class, "version"), bytes("data") ) ); From f798208040b46423c5a50787d5fb55ffc20544b4 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:37:30 +0100 Subject: [PATCH 06/13] Supporting gigmap handler initialization for LegacyTypeHandler --- .../store/gigamap/types/BinaryHandlerBitmapLevel2.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java index 1bf56cf0..3e87dabc 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java @@ -17,6 +17,8 @@ import org.eclipse.serializer.memory.XMemory; import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; import org.eclipse.serializer.persistence.binary.types.Binary; +import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandlerSupplier; +import org.eclipse.serializer.persistence.types.PersistenceLegacyTypeHandler; import org.eclipse.serializer.persistence.types.PersistenceLoadHandler; import org.eclipse.serializer.persistence.types.PersistenceReferenceLoader; import org.eclipse.serializer.persistence.types.PersistenceStoreHandler; @@ -29,7 +31,7 @@ * It extends {@link AbstractBinaryHandlerCustom} to support binary-based I/O operations * specific to {@link BitmapLevel2}. */ -public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom +public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom implements BinaryTypeHandlerSupplier { public static BinaryHandlerBitmapLevel2 New() { @@ -112,5 +114,11 @@ public void iterateLoadableReferences(final Binary data, final PersistenceRefere { // no-op } + + @Override + public PersistenceLegacyTypeHandler getLegacyTypeHandler() + { + return BinaryLegacyTypeHandlerBitmapLevel2.New(); + } } From c33cc66ddf06feab9b316a9dcf29deda38e5926f Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Tue, 17 Feb 2026 11:22:17 +0100 Subject: [PATCH 07/13] Update BinaryHandlerBitmapLevel2.java --- .../store/gigamap/types/BinaryHandlerBitmapLevel2.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java index 3e87dabc..ef4f4ab8 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java @@ -17,7 +17,7 @@ import org.eclipse.serializer.memory.XMemory; import org.eclipse.serializer.persistence.binary.types.AbstractBinaryHandlerCustom; import org.eclipse.serializer.persistence.binary.types.Binary; -import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandlerSupplier; +import org.eclipse.serializer.persistence.binary.types.BinaryLegacyTypeHandlerSupplier; import org.eclipse.serializer.persistence.types.PersistenceLegacyTypeHandler; import org.eclipse.serializer.persistence.types.PersistenceLoadHandler; import org.eclipse.serializer.persistence.types.PersistenceReferenceLoader; @@ -31,7 +31,7 @@ * It extends {@link AbstractBinaryHandlerCustom} to support binary-based I/O operations * specific to {@link BitmapLevel2}. */ -public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom implements BinaryTypeHandlerSupplier +public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom implements BinaryLegacyTypeHandlerSupplier { public static BinaryHandlerBitmapLevel2 New() { From a3f9dc4773ea32b304635a831f8142412e739423 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 18 Feb 2026 11:47:36 +0100 Subject: [PATCH 08/13] Update README.md --- .../embedded-tools/storage-converter/README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index 06d2bdea..b6fe33f9 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -22,7 +22,7 @@ mvn -Pconverter-standalone clean package To configure the input and output storage an [external configuration](https://docs.eclipsestore.io/manual/storage/configuration/index.html#external-configuration) file for each storage is required. ```console -java -jar storage-embedded-tools-storage-converter-4.0.0-beta1.jar sourceCongig.xml targetConfig.xml +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar sourceCongig.ini targetConfig.ini ``` ### StorageConverter.java @@ -35,11 +35,18 @@ org.eclipse.store.storage.embedded.tools.storage.converter.StorageConverter.Stor which gives you some more control on the storage's configurations. -### converting binary data +### Converting binary data -To convert the binary representation of persisted objects BinaryConverter implementations can be specified: +To convert the binary representation of persisted objects BinaryConverter implementations can be specified by the implementations full class name. + +using the BinaryConverterBitmapLevel2 converter: ```console -java -jar storage-embedded-tools-storage-converter-3.1.0.jar sourceCongig.xml targetConfig.xml -c BinaryConverterAImpl.class BinaryConverterBImpl.class +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 ``` +If more than one BinaryConverter shall be applied the -c option, including the converters must be applied in quotation marks: + +```console +... "-c binaryConverter1, binaryConverter2" +``` From c21073e74cfebc7c29d274d2cccbe0bf67ab2315 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 18 Feb 2026 12:04:09 +0100 Subject: [PATCH 09/13] added docu for Converting binary data --- .../modules/storage/pages/addendum/tools.adoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/modules/storage/pages/addendum/tools.adoc b/docs/modules/storage/pages/addendum/tools.adoc index ef86cccb..227afc71 100644 --- a/docs/modules/storage/pages/addendum/tools.adoc +++ b/docs/modules/storage/pages/addendum/tools.adoc @@ -64,6 +64,7 @@ This is useful for: * Changing the number of storage channels * Moving storage from local disk to a cloud storage target or vice-versa +* Converting === Prerequisites @@ -132,3 +133,21 @@ To move from local file system to a cloud storage target, configure the target ` ==== For more information see the https://github.com/eclipse-store/store/tree/main/storage/embedded-tools/storage-converter[readme file]. + + +=== Converting binary data + +To convert the binary representation of persisted objects BinaryConverter implementations can be specified by the implementations full class name. + +.To use the supplied the BinaryConverterBitmapLevel2 converter: +[source, bash] +---- +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 +---- + +If more than one BinaryConverter shall be applied the -c option, including the converters must be applied in quotation marks: +[source, bash] +---- +... "-c binaryConverter1, binaryConverter2" +---- + From 2186d743edc4457b1ecb59bf8149ab163fa7011c Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 18 Feb 2026 12:10:33 +0100 Subject: [PATCH 10/13] docu improvements using {maven-version} converter help text --- docs/modules/storage/pages/addendum/tools.adoc | 2 +- .../tools/storage/converter/MainUtilStorageConverter.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/modules/storage/pages/addendum/tools.adoc b/docs/modules/storage/pages/addendum/tools.adoc index 227afc71..6ad6fe85 100644 --- a/docs/modules/storage/pages/addendum/tools.adoc +++ b/docs/modules/storage/pages/addendum/tools.adoc @@ -142,7 +142,7 @@ To convert the binary representation of persisted objects BinaryConverter implem .To use the supplied the BinaryConverterBitmapLevel2 converter: [source, bash] ---- -java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 +java -jar storage-embedded-tools-storage-converter-{maven-version}.jar src.ini dst.ini -c org.eclipse.store.storage.embedded.tools.storage.converter.BinaryConverterBitmapLevel2 ---- If more than one BinaryConverter shall be applied the -c option, including the converters must be applied in quotation marks: diff --git a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java index 982ae79e..a24f66ea 100644 --- a/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java +++ b/storage/embedded-tools/storage-converter/src/main/java/org/eclipse/store/storage/embedded/tools/storage/converter/MainUtilStorageConverter.java @@ -39,12 +39,13 @@ public class MainUtilStorageConverter "Convert a storage into a new one. The source and the new target storage \n" + "must be specified in storage config files provided as program arguments: \n" + "\n" - + "MainUtilStorageConverter sourceConfig.xml targetConfig.xml" + + "MainUtilStorageConverter sourceConfig.ini targetConfig.ini" + "\n" + "Optional: provide a list of BinaryConverter implementations as ONE sting include the -c:" + " \n" + "\"-c binaryConverter1, binaryConverter2\"" - + "It is import to have the \"-c\" inside the quotation marks!" + + "It is import to have the \"-c\" inside the quotation marks, " + + "the converters must be specified using the full class name" ; public static void main(final String[] args) From 700630f50649258258d76b44cef5ce2dc2392914 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Thu, 19 Feb 2026 06:45:14 +0100 Subject: [PATCH 11/13] Code beautifications --- .../types/BinaryHandlerBitmapLevel2.java | 18 +++++++++++------- .../store/gigamap/types/BitmapLevel2.java | 7 +------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java index ef4f4ab8..b3d1e272 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BinaryHandlerBitmapLevel2.java @@ -32,7 +32,11 @@ * specific to {@link BitmapLevel2}. */ public class BinaryHandlerBitmapLevel2 extends AbstractBinaryHandlerCustom implements BinaryLegacyTypeHandlerSupplier -{ +{ + private static final int BITMAP_LEVEL2_VERSION_SIZE = Integer.BYTES; + private static final long BITMAP_LIST_OFFSET = BITMAP_LEVEL2_VERSION_SIZE; + private static final int BITMAP_LEVEL2_BINARY_VERSION = 2; + public static BinaryHandlerBitmapLevel2 New() { return new BinaryHandlerBitmapLevel2(); @@ -62,12 +66,12 @@ public static BinaryHandlerBitmapLevel2 New() @Override public BitmapLevel2 create(final Binary data, final PersistenceLoadHandler handler) { - final long contentLength = Binary.toBinaryListContentByteLength(data.getLoadItemAvailableContentLength()) - 20; + final long contentLength = Binary.toBinaryListContentByteLength(data.getBinaryListTotalByteLength(BITMAP_LEVEL2_VERSION_SIZE)); final int fullLength = BitmapLevel2.getTotalLengthFromPersistentLength(XTypes.to_int(contentLength)); final long level2Address = XMemory.allocate(fullLength); final long persistentAddress = BitmapLevel2.toPersistentDataAddress(level2Address); - data.copyToAddress(Binary.toBinaryListElementsOffset(4), persistentAddress, contentLength); + data.copyToAddress(Binary.toBinaryListElementsOffset(BITMAP_LIST_OFFSET), persistentAddress, contentLength); BitmapLevel2.initializeFromData(level2Address, fullLength); // required to prevent JVM crashes caused by misinterpreted off-heap data. @@ -90,13 +94,13 @@ public void store( final long persistentAddress = BitmapLevel2.toPersistentDataAddress(instance.level2Address); final int persistentLength = BitmapLevel2.getPersistentLengthFromTotalLength(instance.totalLength()); data.storeEntityHeader( - Binary.toBinaryListTotalByteLength(persistentLength) + 20, + Binary.toBinaryListTotalByteLength(persistentLength) + BITMAP_LEVEL2_VERSION_SIZE, this.typeId(), objectId ); - data.store_int(2); - data.storeListHeader(4, persistentLength, persistentLength); - data.copyFromAddress(Binary.toBinaryListElementsOffset(4), persistentAddress, persistentLength); + data.store_int(BITMAP_LEVEL2_BINARY_VERSION); + data.storeListHeader(BITMAP_LIST_OFFSET, persistentLength, persistentLength); + data.copyFromAddress(Binary.toBinaryListElementsOffset(BITMAP_LIST_OFFSET), persistentAddress, persistentLength); } @Override diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java index b099fd1a..0374b6d1 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java @@ -130,12 +130,7 @@ static BinaryTypeHandler provideTypeHandler() { return BinaryHandlerBitmapLevel2.New(); } - - static BinaryTypeHandler provideLegacyTypeHandler() - { - return BinaryLegacyTypeHandlerBitmapLevel2.New(); - } - + private static void setEntryHeader(final long level1EntryAddress, final byte entryType) { XMemory.set_byte(level1EntryAddress + ENTRY_OFFSET_HEADER, entryType); From d495787465358bd025156e8c4e9baea2852dcd3b Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:24:21 +0100 Subject: [PATCH 12/13] reverted white-space changes --- .../store/gigamap/types/BitmapLevel2.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java index 0374b6d1..d757a69c 100644 --- a/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java +++ b/gigamap/gigamap/src/main/java/org/eclipse/store/gigamap/types/BitmapLevel2.java @@ -1,15 +1,5 @@ package org.eclipse.store.gigamap.types; -import java.util.function.Consumer; - -import org.eclipse.serializer.chars.VarString; -import org.eclipse.serializer.math.XMath; -import org.eclipse.serializer.memory.XMemory; -import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandler; -import org.eclipse.serializer.persistence.types.Storer; -import org.eclipse.serializer.persistence.types.Unpersistable; -import org.eclipse.serializer.typing.XTypes; - /*- * #%L * EclipseStore GigaMap @@ -25,6 +15,15 @@ */ import org.eclipse.store.gigamap.exceptions.BitmapLevel2Exception; +import org.eclipse.serializer.chars.VarString; +import org.eclipse.serializer.math.XMath; +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.BinaryTypeHandler; +import org.eclipse.serializer.persistence.types.Storer; +import org.eclipse.serializer.persistence.types.Unpersistable; +import org.eclipse.serializer.typing.XTypes; + +import java.util.function.Consumer; /** @@ -130,7 +129,7 @@ static BinaryTypeHandler provideTypeHandler() { return BinaryHandlerBitmapLevel2.New(); } - + private static void setEntryHeader(final long level1EntryAddress, final byte entryType) { XMemory.set_byte(level1EntryAddress + ENTRY_OFFSET_HEADER, entryType); From 1d7f040b4fd13f6eda5592cecce5428bc1818404 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:19:26 +0100 Subject: [PATCH 13/13] Update README.md typo fixed --- storage/embedded-tools/storage-converter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index b6fe33f9..2a8bdc81 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -22,7 +22,7 @@ mvn -Pconverter-standalone clean package To configure the input and output storage an [external configuration](https://docs.eclipsestore.io/manual/storage/configuration/index.html#external-configuration) file for each storage is required. ```console -java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar sourceCongig.ini targetConfig.ini +java -jar storage-embedded-tools-storage-converter-4.0.0-SNAPSHOT.jar sourceConfig.ini targetConfig.ini ``` ### StorageConverter.java