diff --git a/README.md b/README.md index beb14f9..ed0b967 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Adaptive Encoding Strategies for Erasing-Based Lossless Floating-Point Compression +# Adaptive Encoding Strategies for Streaming Erasing-Based Lossless Floating-Point Compression *** Based on the erasing method of ***Elf***, we improved it to get the ***ElfStar*** method. Compared with the ***Elf*** diff --git a/src/main/java/com.github.Tranway.buff/BuffCompressor.java b/src/main/java/com.github.Tranway.buff/BuffCompressor.java new file mode 100644 index 0000000..802b1da --- /dev/null +++ b/src/main/java/com.github.Tranway.buff/BuffCompressor.java @@ -0,0 +1,286 @@ +package com.github.Tranway.buff; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.io.IOException; + +public class BuffCompressor { + private static int batchSize = 1000; + private static final int[] PRECISION_MAP = new int[]{ + 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 52, 52, 52, 64, 64, 64, 64, 64, 64, 64 + }; + private static final long[] LAST_MASK = new long[]{ + 0b1L, 0b11L, 0b111L, 0b1111L, 0b11111L, 0b111111L, 0b1111111L, 0b11111111L + }; + private final OutputBitStream out; + private long size; + private long lowerBound; + private int maxPrec; + private int decWidth; + private int intWidth; + private int wholeWidth; + private int columnCount; + + public BuffCompressor(int batchSize) { + BuffCompressor.batchSize = batchSize; + out = new OutputBitStream(new byte[100000]); + size = 0; + } + + private static int getWidthNeeded(long number) { + if (number == 0) { + return 0; + } + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; // 右移一位 + } + return bitCount; + } + + public static int getDecimalPlace(double db) { + if (db == 0.0) { + return 0; + } + String strDb = Double.toString(db); + int indexOfDecimalPoint = strDb.indexOf('.'); + int cnt = 0; + + if (indexOfDecimalPoint >= 0) { + for (int i = indexOfDecimalPoint + 1; i < strDb.length(); ++i) { + if (strDb.charAt(i) != 'E') { + cnt++; + } else { + i++; + cnt -= Integer.parseInt(strDb.substring(i)); + return Math.max(cnt, 0); + } + } + return cnt; + } else { + return 0; // 没有小数点，小数位数为0 + } + } + + public static SparseResult findMajority(byte[] nums) { + SparseResult result = new SparseResult(batchSize); + byte candidate = 0; + int count = 0; + + for (byte num : nums) { + if (count == 0) { + candidate = num; + count = 1; + } else if (num == candidate) { + count++; + } else { + count--; + } + } + + // 验证候选元素是否确实出现频率达到90%以上 + count = 0; + for (int i = 0; i < nums.length; ++i) { + int index = i / 8; // 当前行所处的byte下标 + result.bitmap[index] = (byte) (result.bitmap[index] << 1); + if (nums[i] == candidate) { + count++; + } else { + result.bitmap[index] = (byte) (result.bitmap[index] | 0b1); + result.outliers[result.outliersCnt++] = nums[i]; + } + if (i + 1 == nums.length && (i + 1) % 8 != 0) { + result.bitmap[index] = (byte) (result.bitmap[index] << (i % 8) + 1); + } + } + + if (count >= nums.length * 0.9) { + result.flag = true; + result.frequentValue = candidate; + } else { + result.flag = false; + } + return result; + } + + public byte[] getOut() { + return this.out.getBuffer(); + } + + public void compress(double[] values) { + headSample(values); + byte[][] cols = encode(values); + size += out.writeLong(lowerBound, 64); + size += out.writeInt(batchSize, 32); + size += out.writeInt(maxPrec, 32); + size += out.writeInt(intWidth, 32); + if (wholeWidth >= 64) { + wholeWidthLongCompress(values); + } else { + sparseEncode(cols); + } + close(); + } + + public void wholeWidthLongCompress(double[] values) { + for (double value : values) { + size += out.writeLong(Double.doubleToLongBits(value), 64); + } + } + + public void close() { + out.writeInt(0, 8); + } + + public long getSize() { + return size; + } + + public void headSample(double[] dbs) { + lowerBound = Long.MAX_VALUE; + long upperBound = Long.MIN_VALUE; + for (double db : dbs) { + // double -> bits + long bits = Double.doubleToLongBits(db); + long sign = bits >>> 63; + // get the exp + long expBinary = bits >>> 52 & 0x7FF; + long exp = expBinary - 1023; + // get the mantissa + long mantissa = bits & 0x000fffffffffffffL; // 0.11 1 -0.12 -1 + + // get the mantissa with implicit bit + long implicitMantissa = mantissa | (1L << 52); + + // get the precision + int prec = getDecimalPlace(db); + + // update the max prec + if (prec > maxPrec) { + maxPrec = prec; + } + + // get the integer + long integer = exp < 0 ? 0 : (implicitMantissa >>> (52 - exp)); + long integerValue = (sign == 0) ? integer : -integer; + + if (integerValue > upperBound) { + upperBound = integerValue; + } + if (integerValue < lowerBound) { + lowerBound = integerValue; + } + } + + // get the int_width + intWidth = getWidthNeeded(upperBound - lowerBound); + + // get the dec_width + decWidth = PRECISION_MAP[maxPrec]; + + // get the whole_width + wholeWidth = intWidth + decWidth + 1; + + // get the col/bytes needed + columnCount = wholeWidth / 8; + if (wholeWidth % 8 != 0) { + columnCount++; + } + } + + public byte[][] encode(double[] dbs) { + byte[][] cols = new byte[columnCount][dbs.length]; // 第一维代表列号，第二维代表行号 + + int dbCnt = 0; + for (double db : dbs) { + // double -> bits + long bits = Double.doubleToLongBits(db); + // bits -> string + + // get the sign + long sign = bits >>> 63; + + // get the exp + long expBinary = bits >>> 52 & 0x7FF; // mask for the last 11 bits + long exp = expBinary - 1023; + + // get the mantissa + long mantissa = bits & 0x000fffffffffffffL; + + // get the mantissa with implicit bit + long implicitMantissa = mantissa | (1L << 52); + + long decimal; + if (exp >= 0) { + decimal = mantissa << (12 + exp) >>> (64 - decWidth); + } else { + if (53 - decWidth >= 0) { + decimal = implicitMantissa >>> 53 - decWidth >>> (-exp - 1); + } else { + decimal = implicitMantissa << decWidth - 53 >>> (-exp - 1); + } + } + + // get the integer + long integer = exp < 0 ? 0 : (implicitMantissa >>> (52 - exp)); + long integerValue = (sign == 0) ? integer : -integer; + + // get the offset of integer + long offset = integerValue - lowerBound; + + // get the bitpack result + long bitpack = sign << (wholeWidth - 1) | (offset << decWidth) | decimal; + + // encode into cols[][] + int remain = wholeWidth % 8; + int bytesCnt = 0; + if (remain != 0) { + bytesCnt++; + cols[columnCount - bytesCnt][dbCnt] = (byte) (bitpack & LAST_MASK[remain - 1]); + bitpack = bitpack >>> remain; + } + while (bytesCnt < columnCount) { + bytesCnt++; + cols[columnCount - bytesCnt][dbCnt] = (byte) (bitpack & LAST_MASK[7]); + bitpack = bitpack >>> 8; + } + + dbCnt++; + } + return cols; + } + + public void sparseEncode(byte[][] cols) { + SparseResult result; + for (int j = 0; j < columnCount; ++j) { + // 遍历每一列，查找频繁项 + result = findMajority(cols[j]); + + // col serilize + if (result.flag) { + size += out.writeBit(true); + serialize(result); + } else { + size += out.writeBit(false); + try { + size += out.write(cols[j], batchSize * 8L); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + private void serialize(SparseResult sr) { + size += out.writeInt(sr.frequentValue, 8); + try { + size += out.write(sr.bitmap, batchSize); + } catch (IOException e) { + e.printStackTrace(); + } + for (int i = 0; i < sr.outliersCnt; i++) { + size += out.writeInt(sr.outliers[i], 8); + } + } +} diff --git a/src/main/java/com.github.Tranway.buff/BuffCompressor32.java b/src/main/java/com.github.Tranway.buff/BuffCompressor32.java new file mode 100644 index 0000000..466c8eb --- /dev/null +++ b/src/main/java/com.github.Tranway.buff/BuffCompressor32.java @@ -0,0 +1,288 @@ +package com.github.Tranway.buff; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.io.IOException; + +public class BuffCompressor32 { + private static final int batchSize = 1000; + private static final int[] PRECISION_MAP = new int[]{ + 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 52, 52, 52, 52, 52, 52 + }; + private static final int[] LAST_MASK = new int[]{ + 0b1, 0b11, 0b111, 0b1111, 0b11111, 0b111111, 0b1111111, 0b11111111 + }; + private final OutputBitStream out; + private long size; + private int lowerBound; + private int maxPrec; + private int decWidth; + private int intWidth; + private int wholeWidth; + private int columnCount; + + public BuffCompressor32() { + out = new OutputBitStream(new byte[100000]); + size = 0; + } + + private static int getWidthNeeded(int number) { + if (number == 0) { + return 0; + } + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; + } + return bitCount; + } + + public static int getDecimalPlace(float db) { + if (db == 0.0) { + return 0; + } + String strDb = Float.toString(db); + int indexOfDecimalPoint = strDb.indexOf('.'); + int cnt = 0; + + if (indexOfDecimalPoint >= 0) { + for (int i = indexOfDecimalPoint + 1; i < strDb.length(); ++i) { + if (strDb.charAt(i) != 'E') { + cnt++; + } else { + i++; + cnt -= Integer.parseInt(strDb.substring(i)); + return Math.max(cnt, 0); + } + } + return cnt; + } else { + return 0; + } + } + + public static SparseResult findMajority(byte[] nums) { + SparseResult result = new SparseResult(batchSize); + byte candidate = 0; + int count = 0; + + for (byte num : nums) { + if (count == 0) { + candidate = num; + count = 1; + } else if (num == candidate) { + count++; + } else { + count--; + } + } + + // 验证候选元素是否确实出现频率达到90%以上 + count = 0; + for (int i = 0; i < nums.length; ++i) { + int index = i / 8; // 当前行所处的byte下标 + result.bitmap[index] = (byte) (result.bitmap[index] << 1); + if (nums[i] == candidate) { + count++; + } else { + result.bitmap[index] = (byte) (result.bitmap[index] | 0b1); + result.outliers[result.outliersCnt++] = nums[i]; + } + } + + if (count >= nums.length * 0.9) { + result.flag = true; + result.frequentValue = candidate; + } else { + result.flag = false; + } + return result; + } + + public byte[] getOut() { + return this.out.getBuffer(); + } + + public void compress(float[] values) { + headSample(values); + byte[][] cols = encode(values); + size += out.writeInt(lowerBound, 32); + size += out.writeInt(batchSize, 32); + size += out.writeInt(maxPrec, 32); + size += out.writeInt(intWidth, 32); + if (wholeWidth >= 32) { + wholeWidthLongCompress(values); + } else { + sparseEncode(cols); + } + close(); + } + + public void wholeWidthLongCompress(float[] values) { + for (float value : values) { + size += out.writeInt(Float.floatToIntBits(value), 32); + } + } + + public void close() { + out.writeInt(0, 8); + } + + public long getSize() { + return size; + } + + public void headSample(float[] dbs) { + lowerBound = Integer.MAX_VALUE; + int upperBound = Integer.MIN_VALUE; + for (float db : dbs) { + // float -> bits + int bits = Float.floatToIntBits(db); + // bits -> string + + // get the sign + int sign = bits >>> 31; + + // get the exp + int expBinary = bits >>> 23 & 0xFF; + int exp = expBinary - 127; + + // get the mantissa + int mantissa = bits & 0x7FFFFF; + + // get the mantissa with implicit bit + int implicitMantissa = mantissa | (1 << 23); + + // get the precision + int prec = getDecimalPlace(db); + + // update the max prec + if (prec > maxPrec) { + maxPrec = prec; + } + + // get the integer + int integer = exp < 0 ? 0 : (implicitMantissa >>> (23 - exp)); + int integerValue = (sign == 0) ? integer : -integer; + + // update the integer bound + if (integerValue > upperBound) { + upperBound = integerValue; + } + if (integerValue < lowerBound) { + lowerBound = integerValue; + } + } + + // get the int_width + intWidth = getWidthNeeded(upperBound - lowerBound); + + // get the dec_width + decWidth = PRECISION_MAP[maxPrec]; + + // get the whole_width + wholeWidth = intWidth + decWidth + 1; + + // get the col/bytes needed + columnCount = wholeWidth / 8; + if (wholeWidth % 8 != 0) { + columnCount++; + } + } + + public byte[][] encode(float[] dbs) { + byte[][] cols = new byte[columnCount][dbs.length]; // 第一维代表列号，第二维代表行号 + + int dbCnt = 0; + for (float db : dbs) { + // float -> bits + int bits = Float.floatToIntBits(db); + // bits -> string + + // get the sign + int sign = bits >>> 31; + + // get the exp + int expBinary = bits >>> 23 & 0xFF; // mask for the last 8 bits + int exp = expBinary - 127; + + // get the mantissa + int mantissa = bits & 0x7FFFFF; + + // get the mantissa with implicit bit + int implicitMantissa = mantissa | (1 << 23); + + int decimal; + if (exp >= 0) { + decimal = mantissa << (9 + exp) >>> (32 - decWidth); + } else { + if (24 - decWidth >= 0) { + decimal = implicitMantissa >>> 24 - decWidth >>> (-exp - 1); + } else { + decimal = implicitMantissa << decWidth - 24 >>> (-exp - 1); + } + } + + // get the integer + int integer = exp < 0 ? 0 : (implicitMantissa >>> (23 - exp)); + int integerValue = (sign == 0) ? integer : -integer; + + // get the offset of integer + int offset = integerValue - lowerBound; + + // get the bitpack result + long bitpack = ((long) sign) << (wholeWidth - 1) | ((long) offset << decWidth) | decimal; + + // encode into cols[][] + int remain = wholeWidth % 8; + int bytesCnt = 0; + if (remain != 0) { + bytesCnt++; + cols[columnCount - bytesCnt][dbCnt] = (byte) (bitpack & LAST_MASK[remain - 1]); + bitpack = bitpack >>> remain; + } + while (bytesCnt < columnCount) { + bytesCnt++; + cols[columnCount - bytesCnt][dbCnt] = (byte) (bitpack & LAST_MASK[7]); + bitpack = bitpack >>> 8; + } + + dbCnt++; + } + return cols; + } + + public void sparseEncode(byte[][] cols) { + SparseResult result; + for (int j = 0; j < columnCount; ++j) { + // 遍历每一列，查找频繁项 + result = findMajority(cols[j]); + + // col serilize + if (result.flag) { + size += out.writeBit(true); + serialize(result); + } else { + size += out.writeBit(false); + try { + size += out.write(cols[j], batchSize * 8); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + private void serialize(SparseResult sr) { + size += out.writeInt(sr.frequentValue, 8); + try { + size += out.write(sr.bitmap, batchSize); + } catch (IOException e) { + e.printStackTrace(); + } + for (int i = 0; i < sr.outliersCnt; i++) { + size += out.writeInt(sr.outliers[i], 8); + } + } +} diff --git a/src/main/java/com.github.Tranway.buff/BuffDecompressor.java b/src/main/java/com.github.Tranway.buff/BuffDecompressor.java new file mode 100644 index 0000000..4f3286d --- /dev/null +++ b/src/main/java/com.github.Tranway.buff/BuffDecompressor.java @@ -0,0 +1,178 @@ +package com.github.Tranway.buff; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class BuffDecompressor { + private static final int[] PRECISION_MAP = new int[]{ + 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 52, 52, 52, 64, 64, 64, 64, 64, 64, 64 + }; + private static final long[] LAST_MASK = new long[]{ + 0b1L, 0b11L, 0b111L, 0b1111L, 0b11111L, 0b111111L, 0b1111111L, 0b11111111L + }; + private final InputBitStream in; + private int columnCount; + private long lowerBound; + private int batchSize; + private int maxPrec; + private int decWidth; + private int intWidth; + private int wholeWidth; + private byte[][] cols; + + public BuffDecompressor(byte[] bs) { + in = new InputBitStream(bs); + } + + public static int getWidthNeeded(long number) { + if (number == 0) { + return 0; // 约定0不需要位宽 + } + + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; // 右移一位 + } + + return bitCount; + } + + public double[] decompress() throws IOException { + lowerBound = in.readLong(64); + batchSize = in.readInt(32); + maxPrec = in.readInt(32); + intWidth = in.readInt(32); + decWidth = PRECISION_MAP[maxPrec]; + wholeWidth = decWidth + intWidth + 1; + if (wholeWidth >= 64) { + double[] result = new double[batchSize]; + for (int i = 0; i < batchSize; i++) { + result[i] = Double.longBitsToDouble(in.readLong(64)); + } + return result; + } + columnCount = wholeWidth / 8; + if (wholeWidth % 8 != 0) { + columnCount++; + } + cols = new byte[columnCount][batchSize]; + sparseDecode(); + return mergeDoubles(); + } + + public SparseResult deserialize() throws IOException { + SparseResult result = new SparseResult(batchSize); + result.setFrequentValue(in.readInt(8)); + in.read(result.bitmap, batchSize); + int count = 0; + for (byte b : result.bitmap) { + for (int i = 0; i < 8; i++) { + count += (b >> i) & 1; + } + } + for (int i = 0; i < count; i++) { + result.getOutliers()[i] = (byte) in.readInt(8); + } + + return result; + } + + public void sparseDecode() throws IOException { + for (int j = 0; j < columnCount; ++j) { + if (in.readBit() == 0) { + in.read(cols[j], batchSize * 8); + } else { + SparseResult result; + result = deserialize(); + int index, offset, vecCnt = 0; + for (int i = 0; i < batchSize; i++) { + index = i / 8; + offset = i % 8; + if ((result.bitmap[index] & (1 << (7 - offset))) == 0) { + cols[j][i] = result.frequentValue; + } else { + cols[j][i] = result.outliers[vecCnt++]; + } + } + } + } + } + + public double[] mergeDoubles() { + double[] dbs = new double[batchSize]; + for (int i = 0; i < batchSize; i++) { + // 逐行提取数据 + long bitpack = 0; + int remain = wholeWidth % 8; + if (remain == 0) { + for (int j = 0; j < columnCount; j++) { + bitpack = (bitpack << 8) | (cols[j][i] & LAST_MASK[7]); + } + } else { + for (int j = 0; j < columnCount - 1; j++) { + bitpack = (bitpack << 8) | (cols[j][i] & LAST_MASK[7]); + } + bitpack = (bitpack << remain) | (cols[columnCount - 1][i] & LAST_MASK[remain - 1]); + } + + // get the offset + long offset = (intWidth != 0) ? (bitpack << 65 - wholeWidth >>> 64 - intWidth) : 0; + + // get the integer + long integer = lowerBound + offset; + + // get the decimal + long decimal = bitpack << (64 - decWidth) >>> (64 - decWidth); + + // modified decimal [used for - exp] + long modifiedDecimal = decimal << (decWidth - getWidthNeeded(decimal)); + + // get the exp + long exp = integer != 0 ? (getWidthNeeded(Math.abs(integer)) + 1022) + : 1023 - (decWidth - getWidthNeeded(decimal) + 1); + long expValue = exp - 1023; + + // get the mantissa with implicit bit + int tmp = 53 - decWidth - getWidthNeeded(Math.abs(integer)); + + long implicitMantissa = (Math.abs(integer) << tmp + decWidth) + | (expValue < 0 ? (tmp >= 0 ? (modifiedDecimal << tmp) : (modifiedDecimal >>> Math.abs(tmp))) + : tmp >= 0 + ? (decimal << (tmp)) + : (decimal >>> Math.abs(tmp))); + + // get the mantissa + long mantissa = implicitMantissa & 0x000fffffffffffffL; + + // get the sign + long sign = bitpack >>> (wholeWidth - 1); + + // get the origin bits in IEEE754 + long bits = (sign << 63) | (exp << 52) | mantissa; + + // get the origin value + double db = Double.longBitsToDouble(bits); + + /* Here we use BigDecimal (although it works slowly) to ensure that the double can be rounded by the maxPrec safely. + otherwise, for example, letting db = Math.round(db * Math.pow(10,maxPrec))/Math.pow(10,maxPrec) + will yield a large number [db * Math.pow(db,maxPrec)] which cannot be represented exactly by IEEE754 double, + and result in the fail of lossless decompression. + + eg: + db = 36.070883827972324 + db_tmp = db * Math.pow(10,15) = 36070883827972320 + db_rounded = db_tmp / Math.pow(10,15) = 36.07088382797232 (except 36.070883827972324) + */ + BigDecimal bd = new BigDecimal(db); + db = bd.setScale(maxPrec, RoundingMode.HALF_UP).doubleValue(); + + if (db == 0 && sign == 1) db = -db; + dbs[i] = db; + } + return dbs; + } +} diff --git a/src/main/java/com.github.Tranway.buff/BuffDecompressor32.java b/src/main/java/com.github.Tranway.buff/BuffDecompressor32.java new file mode 100644 index 0000000..687db33 --- /dev/null +++ b/src/main/java/com.github.Tranway.buff/BuffDecompressor32.java @@ -0,0 +1,167 @@ +package com.github.Tranway.buff; + + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class BuffDecompressor32 { + private static final int[] PRECISION_MAP = new int[]{ + 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 52, 52, 52, 52, 52, 52 + }; + private static final int[] LAST_MASK = new int[]{ + 0b1, 0b11, 0b111, 0b1111, 0b11111, 0b111111, 0b1111111, 0b11111111 + }; + private final InputBitStream in; + private int columnCount; + private int lowerBound; + private int batchSize; + private int maxPrec; + private int decWidth; + private int intWidth; + private int wholeWidth; + private byte[][] cols; + + public BuffDecompressor32(byte[] bs) { + in = new InputBitStream(bs); + } + + public static int getWidthNeeded(int number) { + if (number == 0) { + return 0; // 约定0不需要位宽 + } + + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; // 右移一位 + } + + return bitCount; + } + + public float[] decompress() throws IOException { + lowerBound = in.readInt(32); + batchSize = in.readInt(32); + maxPrec = in.readInt(32); + intWidth = in.readInt(32); + decWidth = PRECISION_MAP[maxPrec]; + wholeWidth = decWidth + intWidth + 1; + if (wholeWidth >= 32) { + float[] result = new float[batchSize]; + for (int i = 0; i < batchSize; i++) { + result[i] = Float.intBitsToFloat(in.readInt(32)); + } + return result; + } + columnCount = wholeWidth / 8; + if (wholeWidth % 8 != 0) { + columnCount++; + } + cols = new byte[columnCount][batchSize]; + sparseDecode(); + return mergeDoubles(); + } + + public SparseResult deserialize() throws IOException { + SparseResult result = new SparseResult(batchSize); + result.setFrequentValue(in.readInt(8)); + in.read(result.bitmap, batchSize); + int count = 0; + for (byte b : result.bitmap) { + for (int i = 0; i < 8; i++) { + count += (b >> i) & 1; + } + } + for (int i = 0; i < count; i++) { + result.getOutliers()[i] = (byte) in.readInt(8); + } + + return result; + } + + public void sparseDecode() throws IOException { + for (int j = 0; j < columnCount; ++j) { + if (in.readBit() == 0) { + in.read(cols[j], batchSize * 8); + } else { + SparseResult result; + result = deserialize(); + int index, offset, vecCnt = 0; + for (int i = 0; i < batchSize; i++) { + index = i / 8; + offset = i % 8; + if ((result.bitmap[index] & (1 << (7 - offset))) == 0) { + cols[j][i] = result.frequentValue; + } else { + cols[j][i] = result.outliers[vecCnt++]; + } + } + } + } + } + + public float[] mergeDoubles() { + float[] dbs = new float[batchSize]; + for (int i = 0; i < batchSize; i++) { + long bitpack = 0; + int remain = wholeWidth % 8; + if (remain == 0) { + for (int j = 0; j < columnCount; j++) { + bitpack = (bitpack << 8) | (cols[j][i] & LAST_MASK[7]); + } + } else { + for (int j = 0; j < columnCount - 1; j++) { + bitpack = (bitpack << 8) | (cols[j][i] & LAST_MASK[7]); + } + bitpack = (bitpack << remain) | (cols[columnCount - 1][i] & LAST_MASK[remain - 1]); + } + + // get the offset + int offset = (intWidth != 0) ? (int) (bitpack << 65 - wholeWidth >>> 64 - intWidth) : 0; + + + // get the integer + int integer = lowerBound + offset; + + // get the decimal + int decimal = (int) (bitpack << (64 - decWidth) >>> (64 - decWidth)); + + // modified decimal [used for - exp] + int modifiedDecimal = decimal << (decWidth - getWidthNeeded(decimal)); + + // get the mantissa with implicit bit + int tmp = 24 - decWidth - getWidthNeeded(Math.abs(integer)); + + int implicitMantissa = (Math.abs(integer) << tmp + decWidth) + | (integer == 0 ? tmp >= 0 ? (modifiedDecimal << tmp) : (modifiedDecimal >>> Math.abs(tmp)) + : tmp >= 0 + ? (decimal << (tmp)) + : (decimal >>> Math.abs(tmp))); + + // get the mantissa + int mantissa = implicitMantissa & 0x7FFFFF; + + // get the sign + int sign = (int) (bitpack >>> (wholeWidth - 1)); + + // get the exp + int exp = integer != 0 ? (getWidthNeeded(Math.abs(integer)) + 126) + : 127 - (decWidth - getWidthNeeded(decimal) + 1); + + // get the origin bits in IEEE754 + int bits = (sign << 31) | (exp << 23) | mantissa; + + // get the origin value + float db = Float.intBitsToFloat(bits); + + BigDecimal bd = new BigDecimal(db); + db = bd.setScale(maxPrec, RoundingMode.HALF_UP).floatValue(); + if (db == 0 && sign == 1) db = -db; + dbs[i] = db; + } + return dbs; + } +} diff --git a/src/main/java/com.github.Tranway.buff/SparseResult.java b/src/main/java/com.github.Tranway.buff/SparseResult.java new file mode 100644 index 0000000..68e6a10 --- /dev/null +++ b/src/main/java/com.github.Tranway.buff/SparseResult.java @@ -0,0 +1,25 @@ +package com.github.Tranway.buff; + +public class SparseResult { + public boolean flag; + public byte frequentValue; + public byte[] bitmap; + public boolean[] isFrequentValue; + public Byte[] outliers; + public int outliersCnt = 0; + + SparseResult(int batch_size) { + flag = false; + bitmap = new byte[batch_size / 8 + 1]; + outliers = new Byte[batch_size]; + isFrequentValue = new boolean[batch_size]; + } + + public void setFrequentValue(int frequentValue) { + this.frequentValue = (byte) frequentValue; + } + + public Byte[] getOutliers() { + return outliers; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPCombination.java b/src/main/java/com/github/Cwida/alp/ALPCombination.java new file mode 100644 index 0000000..16d232d --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPCombination.java @@ -0,0 +1,26 @@ +package com.github.Cwida.alp; + +public class ALPCombination { + public byte e; // exponent指数 + public byte f; // factor因子 + public long bestCnt; // 作为最佳组合出现的次数 + + public ALPCombination(byte exponent, byte factor, long bestCnt) { + this.e = exponent; + this.f = factor; + this.bestCnt = bestCnt; + } + + /** + * 判断组合c1是否优于组合c2 + * + * @param c1 combination 1 + * @param c2 combination 2 + * @return if c1 better than c2 + */ + public static boolean compareALPCombinations(ALPCombination c1, ALPCombination c2) { + return (c1.bestCnt > c2.bestCnt) || + (c1.bestCnt == c2.bestCnt && c2.e < c1.e) || + ((c1.bestCnt == c2.bestCnt && c2.e == c1.e) && (c2.f < c1.f)); + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPCompression.java b/src/main/java/com/github/Cwida/alp/ALPCompression.java new file mode 100644 index 0000000..2578adc --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPCompression.java @@ -0,0 +1,405 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.io.IOException; +import java.util.*; + +public class ALPCompression { + static final double MAGIC_NUMBER = Math.pow(2, 51) + Math.pow(2, 52); // 对应文章中的sweet值，用于消除小数部分 + static final byte MAX_EXPONENT = 18; + static final byte EXACT_TYPE_BIT_SIZE = Double.SIZE; + private static final double[] EXP_ARR = { + 1.0, + 10.0, + 100.0, + 1000.0, + 10000.0, + 100000.0, + 1000000.0, + 10000000.0, + 100000000.0, + 1000000000.0, + 10000000000.0, + 100000000000.0, + 1000000000000.0, + 10000000000000.0, + 100000000000000.0, + 1000000000000000.0, + 10000000000000000.0, + 100000000000000000.0, + 1000000000000000000.0, + 10000000000000000000.0, + 100000000000000000000.0, + 1000000000000000000000.0, + 10000000000000000000000.0, + 100000000000000000000000.0 + }; + private static final double[] FRAC_ARR = { + 1.0, + 0.1, + 0.01, + 0.001, + 0.0001, + 0.00001, + 0.000001, + 0.0000001, + 0.00000001, + 0.000000001, + 0.0000000001, + 0.00000000001, + 0.000000000001, + 0.0000000000001, + 0.00000000000001, + 0.000000000000001, + 0.0000000000000001, + 0.00000000000000001, + 0.000000000000000001, + 0.0000000000000000001, + 0.00000000000000000001 + }; + static ALPCompressionState state; + private final OutputBitStream out; + ALPrdCompression aLPrd; + private long size; + + public void reset(){ + state.reset(); + aLPrd.reset(); + } + + public long getSize(){ + return size; + } + + public ALPCompression(int vectorSize) { + this.out = new OutputBitStream( + new byte[7000000]); + this.aLPrd = new ALPrdCompression(out, size, vectorSize); + size = 0; + ALPConstants.selfAdaption(vectorSize); + state = new ALPCompressionState(vectorSize); + } + + /** + * 用于将double转为long，来自文章中的fast rounding部分 + * + * @param db a double value + * @return n + */ + public static long doubleToLong(double db) { + + double n = db + MAGIC_NUMBER - MAGIC_NUMBER; + return (long) n; + } + + + public void compress(List inputVector, int nValues, ALPCompressionState state) { + if (state.bestKCombinations.size() > 1) { + findBestFactorAndExponent(inputVector, nValues, state); + } else { + state.vectorExponent = state.bestKCombinations.get(0).e; + state.vectorFactor = state.bestKCombinations.get(0).f; + } + + // Encoding Floating-Point to Int64 + //! We encode all the values regardless of their correctness to recover the original floating-point + //! We detect exceptions later using a predicated comparison + List tmpDecodedValues = new ArrayList<>(nValues); // Tmp array to check wether the encoded values are exceptions + for (int i = 0; i < nValues; i++) { + double db = inputVector.get(i); + double tmpEncodedValue = db * EXP_ARR[state.vectorExponent] * FRAC_ARR[state.vectorFactor]; + long encodedValue = doubleToLong(tmpEncodedValue); + state.encodedIntegers[i] = encodedValue; + + double decodedValue = encodedValue * ALPConstants.FACT_ARR[state.vectorFactor] * FRAC_ARR[state.vectorExponent]; + tmpDecodedValues.add(decodedValue); + } + + // Detecting exceptions with predicated comparison + List exceptionsPositions = new ArrayList<>(nValues); + for (int i = 0; i < nValues; i++) { + double decodedValue = tmpDecodedValues.get(i); + double actualValue = inputVector.get(i); + boolean isException = (decodedValue != actualValue) || Double.doubleToRawLongBits(actualValue)==-9223372036854775808L; // 将-0.00归为异常值 + if (isException) + exceptionsPositions.add((short) i); + } + + // Finding first non exception value + long aNonExceptionValue = 0; + for (int i = 0; i < nValues; i++) { + if (i == exceptionsPositions.size() || i != exceptionsPositions.get(i)) { + aNonExceptionValue = state.encodedIntegers[i]; + break; + } + } + + // Replacing that first non exception value on the vector exceptions + short exceptionsCount = 0; + for (short exceptionPos : exceptionsPositions) { + double actualValue = inputVector.get(exceptionPos); + state.encodedIntegers[exceptionPos] = aNonExceptionValue; + state.exceptions[exceptionsCount] = actualValue; + state.exceptionsPositions[exceptionsCount] = exceptionPos; + exceptionsCount++; + } + state.exceptionsCount = exceptionsCount; + + // Analyze FFOR + long minValue = Long.MAX_VALUE; + long maxValue = Long.MIN_VALUE; + for (int i = 0; i < nValues; i++) { + long encodedValue = state.encodedIntegers[i]; + maxValue = Math.max(maxValue, encodedValue); + minValue = Math.min(minValue, encodedValue); + } + long minMaxDiff = maxValue - minValue; + + // Subtract FOR + for (int i = 0; i < nValues; i++) { + state.encodedIntegers[i] -= minValue; + } + + int bitWidth = getWidthNeeded(minMaxDiff); // FFOR单值所需位宽 + + state.bitWidth = (short) bitWidth; + state.frameOfReference = minValue; + + size += out.writeBit(true); + size += out.writeInt(state.vectorExponent, 8); + size += out.writeInt(state.vectorFactor, 8); + size += out.writeInt(bitWidth, 16); + size += out.writeLong(state.frameOfReference, 64); + size += out.writeInt(nValues, 32); + for (int i = 0; i < nValues; i++) { + size += out.writeLong(state.encodedIntegers[i], bitWidth); + } + size += out.writeInt(state.exceptionsCount, 16); + for (int i = 0; i < state.exceptionsCount; i++) { + size += out.writeLong(Double.doubleToRawLongBits(state.exceptions[i]), 64); + size += out.writeLong(state.exceptionsPositions[i], 16); + } + } + + private static int getWidthNeeded(long number) { + if (number == 0) { + return 0; + } + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; // 右移一位 + } + return bitCount; + } + + public byte[] getOut() { + int byteCount = (int) Math.ceil(size / 8.0); + return Arrays.copyOf(out.getBuffer(), byteCount); + } + + public void close() { + try { + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void flush() { + out.flush(); + } + + /** + * ALP & ALPrd 算法压缩入口 + * + * @param rowGroup 输入行组 + */ + public void entry(List> rowGroup) { + List> vectorsSampled = new ArrayList<>(); + int idxIncrements = Math.max(1, (int) Math.ceil((double) rowGroup.size() / ALPConstants.RG_SAMPLES)); // 用于行组采样的向量下标增量 + for (int i = 0; i < rowGroup.size(); i += idxIncrements) { + vectorsSampled.add(rowGroup.get(i)); + } + // 第一级采样 + findTopKCombinations(vectorsSampled); + size += out.writeLong(rowGroup.size(), 8); + if (!state.useALP) { + // use ALPrd + for (List row : rowGroup) { // 逐行处理 + aLPrd.entry(row); + } + + size += aLPrd.getSize(); + } else { + // use ALP + for (List row : rowGroup) { // 逐行处理 + // 第二级采样，获取最佳组合 + findBestFactorAndExponent(row, row.size(), state); + // 压缩处理 + compress(row, row.size(), state); + } + } + } + + // 第一级采样 + private void findTopKCombinations(List> vectorsSampled) { + state.bestKCombinations.clear(); + Map, Integer> bestKCombinationsHash = new HashMap<>(); // 记录每个组合出现的次数 + for (List sampledVector : vectorsSampled) { + int nSamples = sampledVector.size(); + byte bestFactor = MAX_EXPONENT; + byte bestExponent = MAX_EXPONENT; + + // Initialize bestTotalBits using the worst possible total bits obtained from compression【异常值大小+正常值大小】 + long bestTotalBits = (long) nSamples * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE) + (long) nSamples * EXACT_TYPE_BIT_SIZE; + + // Try all combinations in search for the one which minimize the compression size + for (byte expIdx = MAX_EXPONENT; expIdx >= 0; expIdx--) { + for (byte factorIdx = expIdx; factorIdx >= 0; factorIdx--) { + int exceptionsCnt = 0; // 记录异常值出现次数 + int nonExceptionsCnt = 0; // 记录能够正常编码的次数 + int estimatedBitsPerValue; // 预计单值正常编码所需要的位宽 + long estimatedCompressionSize = 0; // 预计编码后所需的总位数 + long maxEncodedValue = Long.MIN_VALUE; + long minEncodedValue = Long.MAX_VALUE; + + int idxIncrements = Math.max(1, (int) Math.ceil((double) sampledVector.size() / ALPConstants.SAMPLES_PER_VECTOR)); // 用于行组采样的向量下标增量 + for (int dbIndex = 0; dbIndex < sampledVector.size(); dbIndex += idxIncrements) { + double db = sampledVector.get(dbIndex); + double tmp_encoded_value = db * EXP_ARR[expIdx] * FRAC_ARR[factorIdx]; + long encoded_value = doubleToLong(tmp_encoded_value); // 对应ALPenc + + // The cast to double is needed to prevent a signed integer overflow + double decoded_value = (double) (encoded_value) * ALPConstants.FACT_ARR[factorIdx] * FRAC_ARR[expIdx]; // 对应Pdec + if (decoded_value == db) { + nonExceptionsCnt++; + maxEncodedValue = Math.max(encoded_value, maxEncodedValue); + minEncodedValue = Math.min(encoded_value, minEncodedValue); + } else { + exceptionsCnt++; + } + } + // Skip combinations which yields to almost all exceptions + // Also skip combinations which yields integers bigger than 2^48 + if (nonExceptionsCnt < ALPConstants.SAMPLES_PER_VECTOR * 0.5 || maxEncodedValue >= 1L << 48) { + continue; + } + // Evaluate factor/exponent compression size (we optimize for FOR) + long delta = maxEncodedValue - minEncodedValue; + estimatedBitsPerValue = (int) Math.ceil(Math.log(delta + 1) / Math.log(2)); // FOR单值位宽 + estimatedCompressionSize += (long) nSamples * estimatedBitsPerValue; // 正常编码的部分 + estimatedCompressionSize += (long) exceptionsCnt * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE); // 异常值部分 + + // 更新单个向量中的最佳组合 + if ((estimatedCompressionSize < bestTotalBits) || + // We prefer bigger exponents + (estimatedCompressionSize == bestTotalBits && (bestExponent < expIdx)) || + // We prefer bigger factors + ((estimatedCompressionSize == bestTotalBits && bestExponent == expIdx) && (bestFactor < factorIdx))) { + bestTotalBits = estimatedCompressionSize; + bestExponent = expIdx; + bestFactor = factorIdx; + } + } + } + // 更新行组中的最佳组合 + if (bestTotalBits != (long) nSamples * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE) + (long) nSamples * EXACT_TYPE_BIT_SIZE) { + Map.Entry bestCombination = new AbstractMap.SimpleEntry<>(bestExponent, bestFactor); + int cnt = bestKCombinationsHash.getOrDefault(bestCombination, 0); + bestKCombinationsHash.put(bestCombination, cnt + 1); + } + } + + // Convert our hash pairs to a Combination vector to be able to sort + List bestKCombinations = new ArrayList<>(); + bestKCombinationsHash.forEach((key, value) -> bestKCombinations.add(new ALPCombination(key.getKey(), // Exponent + key.getValue(), // Factor + value // N of times it appeared (hash value) + ))); + + bestKCombinations.sort((c1, c2) -> { + if (ALPCombination.compareALPCombinations(c1, c2)) { + return -1; // 返回负值表示 c1 应排在 c2 前面 + } else { + return 1; + } + }); + + // Save k' best combinations + for (int i = 0; i < Math.min(ALPConstants.MAX_COMBINATIONS, (byte) bestKCombinations.size()); i++) { + state.bestKCombinations.add(bestKCombinations.get(i)); + } + + // 判断是否使用ALPrd + state.useALP = !bestKCombinations.isEmpty(); + } + + // 第二级采样 + private void findBestFactorAndExponent(List inputVector, int nValues, ALPCompressionState state) { + // We sample equidistant values within a vector; to do this we skip a fixed number of values + List vectorSample = new ArrayList<>(); + int idxIncrements = Math.max(1, (int) Math.ceil((double) nValues / ALPConstants.SAMPLES_PER_VECTOR)); + for (int i = 0; i < nValues; i += idxIncrements) { + vectorSample.add(inputVector.get(i)); + } + + byte bestExponent = 0; + byte bestFactor = 0; + long bestTotalBits = 0; + int worseTotalBitsCounter = 0; + int nSamples = vectorSample.size(); + + // We try each K combination in search for the one which minimize the compression size in the vector + for (int combinationIdx = 0; combinationIdx < state.bestKCombinations.size(); combinationIdx++) { + byte exponentIdx = state.bestKCombinations.get(combinationIdx).e; + byte factorIdx = state.bestKCombinations.get(combinationIdx).f; + int exceptionsCount = 0; + long estimatedCompressionSize = 0; + long maxEncodedValue = Long.MIN_VALUE; + long minEncodedValue = Long.MAX_VALUE; + + for (double db : vectorSample) { + double tmpEncodedValue = db * EXP_ARR[exponentIdx] * FRAC_ARR[factorIdx]; + long encodedValue = (long) tmpEncodedValue; + + double decodedValue = encodedValue * ALPConstants.FACT_ARR[factorIdx] * FRAC_ARR[exponentIdx]; + if (decodedValue == db) { + maxEncodedValue = Math.max(encodedValue, maxEncodedValue); + minEncodedValue = Math.min(encodedValue, minEncodedValue); + } else { + exceptionsCount++; + } + } + + long delta = Math.abs(maxEncodedValue - minEncodedValue); + int estimatedBitsPerValue = (int) Math.ceil(Math.log(delta + 1) / Math.log(2)); + estimatedCompressionSize += (long) nSamples * estimatedBitsPerValue; + estimatedCompressionSize += exceptionsCount * (ALPConstants.EXCEPTION_POSITION_SIZE * 8 + EXACT_TYPE_BIT_SIZE); + + if (combinationIdx == 0) { + bestTotalBits = estimatedCompressionSize; + bestFactor = factorIdx; + bestExponent = exponentIdx; + continue; + } + + if (estimatedCompressionSize >= bestTotalBits) { + worseTotalBitsCounter += 1; + // 贪婪提前推出【连续两个组合未优于之前的最佳组合】 + if (worseTotalBitsCounter == ALPConstants.SAMPLING_EARLY_EXIT_THRESHOLD) { + break; + } + continue; + } + + bestTotalBits = estimatedCompressionSize; + bestFactor = factorIdx; + bestExponent = exponentIdx; + worseTotalBitsCounter = 0; + } + state.vectorExponent = bestExponent; + state.vectorFactor = bestFactor; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPCompression32.java b/src/main/java/com/github/Cwida/alp/ALPCompression32.java new file mode 100644 index 0000000..40cc1af --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPCompression32.java @@ -0,0 +1,381 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.io.IOException; +import java.util.*; + +public class ALPCompression32 { + static final float MAGIC_NUMBER = 12582912.0F; //(float) (Math.pow(2, 22) + Math.pow(2, 23)); + static final byte MAX_EXPONENT = 10; + static final byte EXACT_TYPE_BIT_SIZE = Float.SIZE; + private static final float[] EXP_ARR = { + 1.0F, + 10.0F, + 100.0F, + 1000.0F, + 10000.0F, + 100000.0F, + 1000000.0F, + 10000000.0F, + 100000000.0F, + 1000000000.0F, + 10000000000.0F, + }; + private static final float[] FRAC_ARR = { + 1.0F, + 0.1F, + 0.01F, + 0.001F, + 0.0001F, + 0.00001F, + 0.000001F, + 0.0000001F, + 0.00000001F, + 0.000000001F, + 0.0000000001F + }; + static ALPCompressionState32 state; + private final OutputBitStream out; + ALPrdCompression32 aLPrd; + private long size; + + public void reset(){ + state.reset(); + aLPrd.reset(); + } + + public long getSize(){ + return size; + } + + public ALPCompression32() { + this.out = new OutputBitStream( + new byte[7000000]); + this.aLPrd = new ALPrdCompression32(out, size); + size = 0; + state = new ALPCompressionState32(); + } + + /** + * 用于将float转为long，来自文章中的fast rounding部分 + * + * @param db a float value + * @return n + */ + public static long floatToLong(float db) { + float n = db + MAGIC_NUMBER - MAGIC_NUMBER; + return (long) n; + } + + + public void compress(List inputVector, int nValues, ALPCompressionState32 state) { + if (state.bestKCombinations.size() > 1) { + findBestFactorAndExponent(inputVector, nValues, state); + } else { + state.vectorExponent = state.bestKCombinations.get(0).e; + state.vectorFactor = state.bestKCombinations.get(0).f; + } + + // Encoding Floating-Point to Int64 + //! We encode all the values regardless of their correctness to recover the original floating-point + //! We detect exceptions later using a predicated comparison + List tmpDecodedValues = new ArrayList<>(nValues); // Tmp array to check wether the encoded values are exceptions + for (int i = 0; i < nValues; i++) { + float db = inputVector.get(i); + float tmpEncodedValue = db * EXP_ARR[state.vectorExponent] * FRAC_ARR[state.vectorFactor]; + long encodedValue = floatToLong(tmpEncodedValue); + state.encodedIntegers[i] = encodedValue; + + float decodedValue = encodedValue * ALPConstants.FACT_ARR[state.vectorFactor] * FRAC_ARR[state.vectorExponent]; + tmpDecodedValues.add(decodedValue); + } + + // Detecting exceptions with predicated comparison + List exceptionsPositions = new ArrayList<>(nValues); + for (int i = 0; i < nValues; i++) { + float decodedValue = tmpDecodedValues.get(i); + float actualValue = inputVector.get(i); + boolean isException = (decodedValue != actualValue) || Float.floatToRawIntBits(actualValue)==-2147483648; // 将-0.00归为异常值 + if (isException) + exceptionsPositions.add((short) i); + } + + // Finding first non exception value + long aNonExceptionValue = 0; + for (int i = 0; i < nValues; i++) { + if (i == exceptionsPositions.size() || i != exceptionsPositions.get(i)) { + aNonExceptionValue = state.encodedIntegers[i]; + break; + } + } + + // Replacing that first non exception value on the vector exceptions + short exceptionsCount = 0; + for (short exceptionPos : exceptionsPositions) { + float actualValue = inputVector.get(exceptionPos); + state.encodedIntegers[exceptionPos] = aNonExceptionValue; + state.exceptions[exceptionsCount] = actualValue; + state.exceptionsPositions[exceptionsCount] = exceptionPos; + exceptionsCount++; + } + state.exceptionsCount = exceptionsCount; + + // Analyze FFOR + long minValue = Long.MAX_VALUE; + long maxValue = Long.MIN_VALUE; + for (int i = 0; i < nValues; i++) { + long encodedValue = state.encodedIntegers[i]; + maxValue = Math.max(maxValue, encodedValue); + minValue = Math.min(minValue, encodedValue); + } + long minMaxDiff = maxValue - minValue; + + // Subtract FOR + for (int i = 0; i < nValues; i++) { + state.encodedIntegers[i] -= minValue; + } + + int bitWidth = getWidthNeeded(minMaxDiff); // FFOR单值所需位宽 + + state.bitWidth = (short) bitWidth; + state.frameOfReference = minValue; + + size += out.writeBit(true); + size += out.writeInt(state.vectorExponent, 8); + size += out.writeInt(state.vectorFactor, 8); + size += out.writeInt(bitWidth, 16); + size += out.writeLong(state.frameOfReference, 64); + size += out.writeInt(nValues, 32); + for (int i = 0; i < nValues; i++) { + size += out.writeLong(state.encodedIntegers[i], bitWidth); + } + size += out.writeInt(state.exceptionsCount, 16); + for (int i = 0; i < state.exceptionsCount; i++) { + size += out.writeInt(Float.floatToRawIntBits(state.exceptions[i]), 32); + size += out.writeInt(state.exceptionsPositions[i], 16); + } + } + + private static int getWidthNeeded(long number) { + if (number == 0) { + return 0; + } + int bitCount = 0; + while (number > 0) { + bitCount++; + number = number >>> 1; // 右移一位 + } + return bitCount; + } + + public byte[] getOut() { + int byteCount = (int) Math.ceil(size / 8.0); + return Arrays.copyOf(out.getBuffer(), byteCount); + } + + public void close() { + try { + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void flush() { + out.flush(); + } + + /** + * ALP & ALPrd 算法压缩入口 + * + * @param rowGroup 输入行组 + */ + public void entry(List> rowGroup) { + List> vectorsSampled = new ArrayList<>(); + int idxIncrements = Math.max(1, (int) Math.ceil((double) rowGroup.size() / ALPConstants.RG_SAMPLES)); // 用于行组采样的向量下标增量 + for (int i = 0; i < rowGroup.size(); i += idxIncrements) { + vectorsSampled.add(rowGroup.get(i)); + } + // 第一级采样 + findTopKCombinations(vectorsSampled); + size += out.writeLong(rowGroup.size(), 8); + if (!state.useALP) { + // use ALPrd + for (List row : rowGroup) { // 逐行处理 + aLPrd.entry(row); + } + + size += aLPrd.getSize(); + } else { + // use ALP + for (List row : rowGroup) { // 逐行处理 + // 第二级采样，获取最佳组合 + findBestFactorAndExponent(row, row.size(), state); + // 压缩处理 + compress(row, row.size(), state); + } + } + } + + // 第一级采样 + private void findTopKCombinations(List> vectorsSampled) { + state.bestKCombinations.clear(); + Map, Integer> bestKCombinationsHash = new HashMap<>(); // 记录每个组合出现的次数 + for (List sampledVector : vectorsSampled) { + int nSamples = sampledVector.size(); + byte bestFactor = MAX_EXPONENT; + byte bestExponent = MAX_EXPONENT; + + // Initialize bestTotalBits using the worst possible total bits obtained from compression【异常值大小+正常值大小】 + long bestTotalBits = (long) nSamples * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE) + (long) nSamples * EXACT_TYPE_BIT_SIZE; + + // Try all combinations in search for the one which minimize the compression size + for (byte expIdx = MAX_EXPONENT; expIdx >= 0; expIdx--) { + for (byte factorIdx = expIdx; factorIdx >= 0; factorIdx--) { + int exceptionsCnt = 0; // 记录异常值出现次数 + int nonExceptionsCnt = 0; // 记录能够正常编码的次数 + int estimatedBitsPerValue; // 预计单值正常编码所需要的位宽 + long estimatedCompressionSize = 0; // 预计编码后所需的总位数 + long maxEncodedValue = Long.MIN_VALUE; + long minEncodedValue = Long.MAX_VALUE; + + int idxIncrements = Math.max(1, (int) Math.ceil((float) sampledVector.size() / ALPConstants.SAMPLES_PER_VECTOR)); // 用于行组采样的向量下标增量 + for (int dbIndex = 0; dbIndex < sampledVector.size(); dbIndex += idxIncrements) { + float db = sampledVector.get(dbIndex); + float tmp_encoded_value = db * EXP_ARR[expIdx] * FRAC_ARR[factorIdx]; + long encoded_value = floatToLong(tmp_encoded_value); // 对应ALPenc + + // The cast to float is needed to prevent a signed integer overflow + float decoded_value = (float) (encoded_value) * ALPConstants.FACT_ARR[factorIdx] * FRAC_ARR[expIdx]; // 对应Pdec + if (decoded_value == db) { + nonExceptionsCnt++; + maxEncodedValue = Math.max(encoded_value, maxEncodedValue); + minEncodedValue = Math.min(encoded_value, minEncodedValue); + } else { + exceptionsCnt++; + } + } + // Skip combinations which yields to almost all exceptions + // Also skip combinations which yields integers bigger than 2^48 + if (nonExceptionsCnt < ALPConstants.SAMPLES_PER_VECTOR * 0.5 || maxEncodedValue >= 1L << 48) { + continue; + } + // Evaluate factor/exponent compression size (we optimize for FOR) + long delta = maxEncodedValue - minEncodedValue; + estimatedBitsPerValue = (int) Math.ceil(Math.log(delta + 1) / Math.log(2)); // FOR单值位宽 + estimatedCompressionSize += (long) nSamples * estimatedBitsPerValue; // 正常编码的部分 + estimatedCompressionSize += (long) exceptionsCnt * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE); // 异常值部分 + + // 更新单个向量中的最佳组合 + if ((estimatedCompressionSize < bestTotalBits) || + // We prefer bigger exponents + (estimatedCompressionSize == bestTotalBits && (bestExponent < expIdx)) || + // We prefer bigger factors + ((estimatedCompressionSize == bestTotalBits && bestExponent == expIdx) && (bestFactor < factorIdx))) { + bestTotalBits = estimatedCompressionSize; + bestExponent = expIdx; + bestFactor = factorIdx; + } + } + } + // 更新行组中的最佳组合 + if (bestTotalBits != (long) nSamples * (EXACT_TYPE_BIT_SIZE + ALPConstants.EXCEPTION_POSITION_SIZE) + (long) nSamples * EXACT_TYPE_BIT_SIZE) { + Map.Entry bestCombination = new AbstractMap.SimpleEntry<>(bestExponent, bestFactor); + int cnt = bestKCombinationsHash.getOrDefault(bestCombination, 0); + bestKCombinationsHash.put(bestCombination, cnt + 1); + } + } + + // Convert our hash pairs to a Combination vector to be able to sort + List bestKCombinations = new ArrayList<>(); + bestKCombinationsHash.forEach((key, value) -> bestKCombinations.add(new ALPCombination(key.getKey(), // Exponent + key.getValue(), // Factor + value // N of times it appeared (hash value) + ))); + + // 使用 List.sort() 进行排序，传入自定义的比较器 等效于C++中的 sort(bestKCombinations.begin(), bestKCombinations.end(), compareALPCombinations); + bestKCombinations.sort((c1, c2) -> { + if (ALPCombination.compareALPCombinations(c1, c2)) { + return -1; // 返回负值表示 c1 应排在 c2 前面 + } else { + return 1; + } + }); + + // Save k' best combinations + for (int i = 0; i < Math.min(ALPConstants.MAX_COMBINATIONS, (byte) bestKCombinations.size()); i++) { + state.bestKCombinations.add(bestKCombinations.get(i)); + } + + // 判断是否使用ALPrd + state.useALP = !bestKCombinations.isEmpty(); + } + + // 第二级采样 + private void findBestFactorAndExponent(List inputVector, int nValues, ALPCompressionState32 state) { + // We sample equidistant values within a vector; to do this we skip a fixed number of values + List vectorSample = new ArrayList<>(); + int idxIncrements = Math.max(1, (int) Math.ceil((float) nValues / ALPConstants.SAMPLES_PER_VECTOR)); + for (int i = 0; i < nValues; i += idxIncrements) { + vectorSample.add(inputVector.get(i)); + } + + byte bestExponent = 0; + byte bestFactor = 0; + long bestTotalBits = 0; + int worseTotalBitsCounter = 0; + int nSamples = vectorSample.size(); + + // We try each K combination in search for the one which minimize the compression size in the vector + for (int combinationIdx = 0; combinationIdx < state.bestKCombinations.size(); combinationIdx++) { + byte exponentIdx = state.bestKCombinations.get(combinationIdx).e; + byte factorIdx = state.bestKCombinations.get(combinationIdx).f; + int exceptionsCount = 0; + long estimatedCompressionSize = 0; + long maxEncodedValue = Long.MIN_VALUE; + long minEncodedValue = Long.MAX_VALUE; + + for (float db : vectorSample) { + float tmpEncodedValue = db * EXP_ARR[exponentIdx] * FRAC_ARR[factorIdx]; + long encodedValue = (long) tmpEncodedValue; + + float decodedValue = encodedValue * ALPConstants.FACT_ARR[factorIdx] * FRAC_ARR[exponentIdx]; + if (decodedValue == db) { + maxEncodedValue = Math.max(encodedValue, maxEncodedValue); + minEncodedValue = Math.min(encodedValue, minEncodedValue); + } else { + exceptionsCount++; + } + } + + long delta = Math.abs(maxEncodedValue - minEncodedValue); + int estimatedBitsPerValue = (int) Math.ceil(Math.log(delta + 1) / Math.log(2)); + estimatedCompressionSize += (long) nSamples * estimatedBitsPerValue; + estimatedCompressionSize += exceptionsCount * (ALPConstants.EXCEPTION_POSITION_SIZE * 8 + EXACT_TYPE_BIT_SIZE); + + if (combinationIdx == 0) { + bestTotalBits = estimatedCompressionSize; + bestFactor = factorIdx; + bestExponent = exponentIdx; + continue; + } + + if (estimatedCompressionSize >= bestTotalBits) { + worseTotalBitsCounter += 1; + // 贪婪提前推出【连续两个组合未优于之前的最佳组合】 + if (worseTotalBitsCounter == ALPConstants.SAMPLING_EARLY_EXIT_THRESHOLD) { + break; + } + continue; + } + + bestTotalBits = estimatedCompressionSize; + bestFactor = factorIdx; + bestExponent = exponentIdx; + worseTotalBitsCounter = 0; + } + state.vectorExponent = bestExponent; + state.vectorFactor = bestFactor; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPCompressionState.java b/src/main/java/com/github/Cwida/alp/ALPCompressionState.java new file mode 100644 index 0000000..225939e --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPCompressionState.java @@ -0,0 +1,37 @@ +package com.github.Cwida.alp; + + +import java.util.ArrayList; +import java.util.List; + +public class ALPCompressionState { + public byte vectorExponent; + public byte vectorFactor; + public short exceptionsCount; + public short bitWidth; + public long frameOfReference; + public long[] encodedIntegers; + public double[] exceptions; + public short[] exceptionsPositions; + public List bestKCombinations = new ArrayList<>(); + + public boolean useALP = true; + + public ALPCompressionState(int vectorSize) { + this.vectorExponent = 0; + this.vectorFactor = 0; + this.exceptionsCount = 0; + this.bitWidth = 0; + this.encodedIntegers = new long[vectorSize]; + this.exceptions = new double[vectorSize]; + this.exceptionsPositions = new short[vectorSize]; + } + + public void reset() { + this.vectorExponent = 0; + this.vectorFactor = 0; + this.exceptionsCount = 0; + this.bitWidth = 0; + } + +} diff --git a/src/main/java/com/github/Cwida/alp/ALPCompressionState32.java b/src/main/java/com/github/Cwida/alp/ALPCompressionState32.java new file mode 100644 index 0000000..05ae6dc --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPCompressionState32.java @@ -0,0 +1,35 @@ +package com.github.Cwida.alp; + +import java.util.ArrayList; +import java.util.List; + +public class ALPCompressionState32 { + public byte vectorExponent; + public byte vectorFactor; + public short exceptionsCount; + public short bitWidth; + public long frameOfReference; + public long[] encodedIntegers; + public float[] exceptions; + public short[] exceptionsPositions; + public List bestKCombinations = new ArrayList<>(); + + public boolean useALP = true; + + public ALPCompressionState32() { + this.vectorExponent = 0; + this.vectorFactor = 0; + this.exceptionsCount = 0; + this.bitWidth = 0; + this.encodedIntegers = new long[ALPConstants.ALP_VECTOR_SIZE]; + this.exceptions = new float[ALPConstants.ALP_VECTOR_SIZE]; + this.exceptionsPositions = new short[ALPConstants.ALP_VECTOR_SIZE]; + } + + public void reset() { + this.vectorExponent = 0; + this.vectorFactor = 0; + this.exceptionsCount = 0; + this.bitWidth = 0; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPConstants.java b/src/main/java/com/github/Cwida/alp/ALPConstants.java new file mode 100644 index 0000000..8a9bba8 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPConstants.java @@ -0,0 +1,29 @@ +package com.github.Cwida.alp; + +public class ALPConstants { + public static int ALP_VECTOR_SIZE = 1000; // 每个向量所含的值的数量 + public static int RG_SAMPLES = 8; // 每行组采样的向量数 + public static short SAMPLES_PER_VECTOR = 32; // 每向量采样的浮点数数量 + public static final byte EXCEPTION_POSITION_SIZE = Short.SIZE;// / Byte.SIZE; + public static final byte SAMPLING_EARLY_EXIT_THRESHOLD = 2; + public static final byte MAX_COMBINATIONS = 5; + + public static final long[] FACT_ARR = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, + 100000000, 1000000000, 10000000000L, 100000000000L, 1000000000000L, + 10000000000000L, 100000000000000L, 1000000000000000L, + 10000000000000000L, 100000000000000000L, 1000000000000000000L + }; + + public static final long[] U_FACT_ARR = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, + 100000000, 1000000000, 10000000000L, 100000000000L, 1000000000000L, + 10000000000000L, 100000000000000L, 1000000000000000L, + 10000000000000000L, 100000000000000000L, 1000000000000000000L + }; + + public static void selfAdaption(int vectorSize){ + ALP_VECTOR_SIZE = vectorSize; + SAMPLES_PER_VECTOR = (short) Math.max(32 * (vectorSize/1000.0),32); + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPDecompression.java b/src/main/java/com/github/Cwida/alp/ALPDecompression.java new file mode 100644 index 0000000..ad6bcae --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPDecompression.java @@ -0,0 +1,112 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ALPDecompression { + private static final double[] FRAC_ARR = { + 1.0, + 0.1, + 0.01, + 0.001, + 0.0001, + 0.00001, + 0.000001, + 0.0000001, + 0.00000001, + 0.000000001, + 0.0000000001, + 0.00000000001, + 0.000000000001, + 0.0000000000001, + 0.00000000000001, + 0.000000000000001, + 0.0000000000000001, + 0.00000000000000001, + 0.000000000000000001, + 0.0000000000000000001, + 0.00000000000000000001 + }; + private final ALPrdDecompression ALPrdDe; + private long[] encodedValue; + private int count; + private byte vectorFactor; + private byte vectorExponent; + private short exceptionsCount; + private double[] exceptions; + private short[] exceptionsPositions; + private long frameOfReference; + private final InputBitStream in; + + public ALPDecompression(byte[] bs) { + in = new InputBitStream(bs); + this.ALPrdDe = new ALPrdDecompression(in); + } + + private void deserialize() { + try { + vectorExponent = (byte) in.readInt(8); + vectorFactor = (byte) in.readInt(8); + short bitWidth = (short) in.readInt(16); + frameOfReference = in.readLong(64); + count = in.readInt(32); + encodedValue = new long[count]; + for (int i = 0; i < count; i++) { + encodedValue[i] = in.readLong(bitWidth); + } + exceptionsCount = (short) in.readInt(16); + exceptions = new double[exceptionsCount]; + exceptionsPositions = new short[exceptionsCount]; + for (int i = 0; i < exceptionsCount; i++) { + exceptions[i] = Double.longBitsToDouble(in.readLong(64)); + exceptionsPositions[i] = (short) in.readLong(16); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private double[] decompress() { + double[] output = new double[count]; + + long factor = ALPConstants.U_FACT_ARR[vectorFactor]; + double exponent = FRAC_ARR[vectorExponent]; + + // unFOR + for (int i = 0; i < count; i++) { + encodedValue[i] = frameOfReference + encodedValue[i]; + } + + // Decoding + for (int i = 0; i < count; i++) { + long encodedInteger = encodedValue[i]; + output[i] = (double) encodedInteger * factor * exponent; + } + + // Exceptions Patching + for (int i = 0; i < exceptionsCount; i++) { + output[exceptionsPositions[i]] = exceptions[i]; + } + + return output; + } + + public List entry() throws IOException { + List result = new ArrayList<>(); + int rowGroupSize = in.readInt(8); + for (int i = 0; i < rowGroupSize; i++) { + int useALP = in.readBit(); + if (useALP == 1) { + deserialize(); + result.add(decompress()); + } else { + ALPrdDe.deserialize(); + result.add(ALPrdDe.decompress()); + } + } + return result; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPDecompression32.java b/src/main/java/com/github/Cwida/alp/ALPDecompression32.java new file mode 100644 index 0000000..ad5c517 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPDecompression32.java @@ -0,0 +1,112 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ALPDecompression32 { + private static final float[] FRAC_ARR = { + 1.0F, + 0.1F, + 0.01F, + 0.001F, + 0.0001F, + 0.00001F, + 0.000001F, + 0.0000001F, + 0.00000001F, + 0.000000001F, + 0.0000000001F, + 0.00000000001F, + 0.000000000001F, + 0.0000000000001F, + 0.00000000000001F, + 0.000000000000001F, + 0.0000000000000001F, + 0.00000000000000001F, + 0.000000000000000001F, + 0.0000000000000000001F, + 0.00000000000000000001F + }; + private final ALPrdDecompression32 ALPrdDe; + private long[] encodedValue; + private int count; + private byte vectorFactor; + private byte vectorExponent; + private short exceptionsCount; + private float[] exceptions; + private short[] exceptionsPositions; + private long frameOfReference; + private final InputBitStream in; + + public ALPDecompression32(byte[] bs) { + in = new InputBitStream(bs); + this.ALPrdDe = new ALPrdDecompression32(in); + } + + public void deserialize() { + try { + vectorExponent = (byte) in.readInt(8); + vectorFactor = (byte) in.readInt(8); + short bitWidth = (short) in.readInt(16); + frameOfReference = in.readLong(64); + count = in.readInt(32); + encodedValue = new long[count]; + for (int i = 0; i < count; i++) { + encodedValue[i] = in.readLong(bitWidth); + } + exceptionsCount = (short) in.readInt(16); + exceptions = new float[exceptionsCount]; + exceptionsPositions = new short[exceptionsCount]; + for (int i = 0; i < exceptionsCount; i++) { + exceptions[i] = Float.intBitsToFloat(in.readInt(32)); + exceptionsPositions[i] = (short) in.readLong(16); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public float[] decompress() { + float[] output = new float[count]; + + long factor = ALPConstants.FACT_ARR[vectorFactor]; + float exponent = FRAC_ARR[vectorExponent]; + + // unFOR + for (int i = 0; i < count; i++) { + encodedValue[i] = frameOfReference + encodedValue[i]; + } + + // Decoding + for (int i = 0; i < count; i++) { + long encodedInteger = encodedValue[i]; + output[i] = encodedInteger * factor * exponent; + } + + // Exceptions Patching + for (int i = 0; i < exceptionsCount; i++) { + output[exceptionsPositions[i]] = exceptions[i]; + } + + return output; + } + + public List entry() throws IOException { + List result = new ArrayList<>(); + int rowGroupSize = in.readInt(8); + for (int i = 0; i < rowGroupSize; i++) { + int useALP = in.readBit(); + if (useALP == 1) { + deserialize(); + result.add(decompress()); + } else { + ALPrdDe.deserialize(); + result.add(ALPrdDe.decompress()); + } + } + return result; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdCompression.java b/src/main/java/com/github/Cwida/alp/ALPrdCompression.java new file mode 100644 index 0000000..7d9544c --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdCompression.java @@ -0,0 +1,159 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.util.*; + +public class ALPrdCompression { + static final byte EXACT_TYPE_BITSIZE = Double.SIZE; + static ALPrdCompressionState state; + private final OutputBitStream out; + private long size; + + public void reset() { + state.reset(); + } + + public ALPrdCompression(OutputBitStream out, long size, int vectorSize) { + this.out = out; + this.size = size; + ALPrdConstants.setVectorSize(vectorSize); + state = new ALPrdCompressionState(vectorSize); + } + + public static double estimateCompressionSize(byte rightBw, byte leftBw, short exceptionsCount, long sampleCount) { + double exceptionsSize = exceptionsCount * ((ALPrdConstants.EXCEPTION_POSITION_SIZE + ALPrdConstants.EXCEPTION_SIZE) * 8); + return rightBw + leftBw + (exceptionsSize / sampleCount); + } + + public static double buildLeftPartsDictionary(List values, byte rightBw, byte leftBw, + boolean persistDict, ALPrdCompressionState state) { + Map leftPartsHash = new HashMap<>(); + List> leftPartsSortedRepetitions = new ArrayList<>(); // <出现次数，左值> + + // Building a hash for all the left parts and how many times they appear + for (Long value : values) { + long leftTmp = value >>> rightBw; + leftPartsHash.put(leftTmp, leftPartsHash.getOrDefault(leftTmp, 0) + 1); + } + + // We build a list from the hash to be able to sort it by repetition count + for (Map.Entry entry : leftPartsHash.entrySet()) { + leftPartsSortedRepetitions.add(new AbstractMap.SimpleEntry<>(entry.getValue(), entry.getKey())); + } + leftPartsSortedRepetitions.sort((a, b) -> Integer.compare(b.getKey(), a.getKey())); // 递减排序 + + // Exceptions are left parts which do not fit in the fixed dictionary size + int exceptionsCount = 0; + for (int i = ALPrdConstants.DICTIONARY_SIZE; i < leftPartsSortedRepetitions.size(); i++) { // 超过字典容量的部分记为异常值 + exceptionsCount += leftPartsSortedRepetitions.get(i).getKey(); + } + + if (persistDict) { + int dictIdx = 0; + int dictSize = Math.min(ALPrdConstants.DICTIONARY_SIZE, leftPartsSortedRepetitions.size()); + for (; dictIdx < dictSize; dictIdx++) { + //! The dict keys are mapped to the left part themselves + state.leftPartsDict[dictIdx] = leftPartsSortedRepetitions.get(dictIdx).getValue().shortValue(); + state.leftPartsDictMap.put(state.leftPartsDict[dictIdx], (short) dictIdx); + } + //! Parallelly we store a map of the dictionary to quickly resolve exceptions during encoding + for (int i = dictIdx; i < leftPartsSortedRepetitions.size(); i++) { + state.leftPartsDictMap.put(leftPartsSortedRepetitions.get(i).getValue().shortValue(), (short) i); + } + state.leftBw = leftBw; + state.rightBw = rightBw; + state.exceptionsCount = (short) exceptionsCount; + } + + return estimateCompressionSize(rightBw, ALPrdConstants.DICTIONARY_BW, (short) exceptionsCount, values.size()); + } + + public static void findBestDictionary(List values, ALPrdCompressionState state) { + int lBw = ALPrdConstants.DICTIONARY_BW; + int rBw = EXACT_TYPE_BITSIZE; + double bestDictSize = Integer.MAX_VALUE; + + //! Finding the best position to CUT the values + for (int i = 1; i <= ALPrdConstants.CUTTING_LIMIT; i++) { + byte candidateLBw = (byte) i; + byte candidateRBw = (byte) (EXACT_TYPE_BITSIZE - i); + double estimatedSize = buildLeftPartsDictionary(values, candidateRBw, candidateLBw, false, state); + if (estimatedSize <= bestDictSize) { + lBw = candidateLBw; + rBw = candidateRBw; + bestDictSize = estimatedSize; + } + } + + buildLeftPartsDictionary(values, (byte) rBw, (byte) lBw, true, state); + } + + public long getSize() { + return size; + } + + public void compress(List in, int nValues, ALPrdCompressionState state) { + long[] rightParts = new long[ALPrdConstants.ALP_VECTOR_SIZE]; + short[] leftParts = new short[ALPrdConstants.ALP_VECTOR_SIZE]; + + // Cutting the floating point values + for (int i = 0; i < nValues; i++) { + Long tmp = in.get(i); + rightParts[i] = tmp & ((1L << state.rightBw) - 1); + leftParts[i] = (short) (tmp >>> state.rightBw); + } + + // Dictionary encoding for left parts + short exceptionsCount = 0; + for (int i = 0; i < nValues; i++) { + short dictionaryIndex; + short dictionaryKey = leftParts[i]; + if (!state.leftPartsDictMap.containsKey(dictionaryKey)) { + // If not found in the dictionary, store the smallest non-key index as an exception (the dict size) + dictionaryIndex = ALPrdConstants.DICTIONARY_SIZE; + } else { + dictionaryIndex = state.leftPartsDictMap.get(dictionaryKey); + } + leftParts[i] = dictionaryIndex; + + // Left parts not found in the dictionary are stored as exceptions + if (dictionaryIndex >= ALPrdConstants.DICTIONARY_SIZE) { + leftParts[i] = 0; // 用0替换 + state.exceptions[exceptionsCount] = dictionaryKey; + state.exceptionsPositions[exceptionsCount] = (short) i; + exceptionsCount++; + } + } + + size += out.writeBit(false); + size += out.writeInt(nValues, 32); + size += out.writeInt(state.rightBw, 8); + for (int i = 0; i < nValues; i++) { + size += out.writeInt(leftParts[i], ALPrdConstants.DICTIONARY_BW); + size += out.writeLong(rightParts[i], state.rightBw); + } + for (int i = 0; i < ALPrdConstants.DICTIONARY_SIZE; i++) { + size += out.writeInt(state.leftPartsDict[i], state.leftBw); + } + size += out.writeInt(exceptionsCount, 16); + for (int i = 0; i < exceptionsCount; i++) { + size += out.writeInt(state.exceptions[i], state.leftBw); + size += out.writeInt(state.exceptionsPositions[i], 16); + } + } + + /** + * ALPrd 算法入口 + * + * @param row 单行数据 + */ + public void entry(List row) { + List rowLong = new ArrayList<>(); + for (double db : row) { + rowLong.add(Double.doubleToLongBits(db)); + } + findBestDictionary(rowLong, state); + compress(rowLong, rowLong.size(), state); + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdCompression32.java b/src/main/java/com/github/Cwida/alp/ALPrdCompression32.java new file mode 100644 index 0000000..cdbb320 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdCompression32.java @@ -0,0 +1,156 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; + +import java.util.*; + +public class ALPrdCompression32 { + static final byte EXACT_TYPE_BITSIZE = Float.SIZE; + static ALPrdCompressionState32 state; + private final OutputBitStream out; + private long size; + + public void reset(){ + state.reset(); + } + + public ALPrdCompression32(OutputBitStream out, long size) { + this.out = out; + this.size = size; + state = new ALPrdCompressionState32(); + } + + public static double estimateCompressionSize(byte rightBw, byte leftBw, short exceptionsCount, long sampleCount) { + double exceptionsSize = exceptionsCount * ((ALPrdConstants.EXCEPTION_POSITION_SIZE + ALPrdConstants.EXCEPTION_SIZE) * 8); + return rightBw + leftBw + (exceptionsSize / sampleCount); + } + + public static double buildLeftPartsDictionary(List values, byte rightBw, byte leftBw, + boolean persistDict, ALPrdCompressionState32 state) { + Map leftPartsHash = new HashMap<>(); + List> leftPartsSortedRepetitions = new ArrayList<>(); // <出现次数，左值> + + // Building a hash for all the left parts and how many times they appear + for (Integer value : values) { + int leftTmp = value >>> rightBw; + leftPartsHash.put(leftTmp, leftPartsHash.getOrDefault(leftTmp, 0) + 1); + } + + // We build a list from the hash to be able to sort it by repetition count + for (Map.Entry entry : leftPartsHash.entrySet()) { + leftPartsSortedRepetitions.add(new AbstractMap.SimpleEntry<>(entry.getValue(), entry.getKey())); + } + leftPartsSortedRepetitions.sort((a, b) -> Integer.compare(b.getKey(), a.getKey())); // 递减排序 + + // Exceptions are left parts which do not fit in the fixed dictionary size + int exceptionsCount = 0; + for (int i = ALPrdConstants.DICTIONARY_SIZE; i < leftPartsSortedRepetitions.size(); i++) { // 超过字典容量的部分记为异常值 + exceptionsCount += leftPartsSortedRepetitions.get(i).getKey(); + } + + if (persistDict) { + int dictIdx = 0; + int dictSize = Math.min(ALPrdConstants.DICTIONARY_SIZE, leftPartsSortedRepetitions.size()); + for (; dictIdx < dictSize; dictIdx++) { + //! The dict keys are mapped to the left part themselves + state.leftPartsDict[dictIdx] = leftPartsSortedRepetitions.get(dictIdx).getValue().shortValue(); + state.leftPartsDictMap.put(state.leftPartsDict[dictIdx], (short) dictIdx); + } + //! Parallelly we store a map of the dictionary to quickly resolve exceptions during encoding + for (int i = dictIdx; i < leftPartsSortedRepetitions.size(); i++) { + state.leftPartsDictMap.put(leftPartsSortedRepetitions.get(i).getValue().shortValue(), (short) i); + } + state.leftBw = leftBw; + state.rightBw = rightBw; + state.exceptionsCount = (short) exceptionsCount; + } + return estimateCompressionSize(rightBw, ALPrdConstants.DICTIONARY_BW, (short) exceptionsCount, values.size()); + } + + public static void findBestDictionary(List values, ALPrdCompressionState32 state) { + int lBw = ALPrdConstants.DICTIONARY_BW; + int rBw = EXACT_TYPE_BITSIZE; + double bestDictSize = Integer.MAX_VALUE; + + //! Finding the best position to CUT the values + for (int i = 1; i <= ALPrdConstants.CUTTING_LIMIT; i++) { + byte candidateLBw = (byte) i; + byte candidateRBw = (byte) (EXACT_TYPE_BITSIZE - i); + double estimatedSize = buildLeftPartsDictionary(values, candidateRBw, candidateLBw, false, state); + if (estimatedSize <= bestDictSize) { + lBw = candidateLBw; + rBw = candidateRBw; + bestDictSize = estimatedSize; + } + } + buildLeftPartsDictionary(values, (byte) rBw, (byte) lBw, true, state); + } + + public long getSize() { + return size; + } + + public void compress(List in, int nValues, ALPrdCompressionState32 state) { + int[] rightParts = new int[ALPrdConstants.ALP_VECTOR_SIZE]; + short[] leftParts = new short[ALPrdConstants.ALP_VECTOR_SIZE]; + + // Cutting the floating point values + for (int i = 0; i < nValues; i++) { + Integer tmp = in.get(i); + rightParts[i] = tmp & ((1 << state.rightBw) - 1); + leftParts[i] = (short) (tmp >>> state.rightBw); + } + + // Dictionary encoding for left parts + short exceptionsCount = 0; + for (int i = 0; i < nValues; i++) { + short dictionaryIndex; + short dictionaryKey = leftParts[i]; + if (!state.leftPartsDictMap.containsKey(dictionaryKey)) { + // If not found in the dictionary, store the smallest non-key index as an exception (the dict size) + dictionaryIndex = ALPrdConstants.DICTIONARY_SIZE; + } else { + dictionaryIndex = state.leftPartsDictMap.get(dictionaryKey); + } + leftParts[i] = dictionaryIndex; + + // Left parts not found in the dictionary are stored as exceptions + if (dictionaryIndex >= ALPrdConstants.DICTIONARY_SIZE) { + leftParts[i] = 0; // 用0替换 + state.exceptions[exceptionsCount] = dictionaryKey; + state.exceptionsPositions[exceptionsCount] = (short) i; + exceptionsCount++; + } + } + + size += out.writeBit(false); + size += out.writeInt(nValues, 32); + size += out.writeInt(state.rightBw, 8); + for (int i = 0; i < nValues; i++) { + size += out.writeInt(leftParts[i], ALPrdConstants.DICTIONARY_BW); + size += out.writeInt(rightParts[i], state.rightBw); + } + for (int i = 0; i < ALPrdConstants.DICTIONARY_SIZE; i++) { + size += out.writeInt(state.leftPartsDict[i], state.leftBw); + } + size += out.writeInt(exceptionsCount, 16); + for (int i = 0; i < exceptionsCount; i++) { + size += out.writeInt(state.exceptions[i], state.leftBw); + size += out.writeInt(state.exceptionsPositions[i], 16); + } + } + + /** + * ALPrd 算法入口 + * + * @param row 单行数据 + */ + public void entry(List row) { + List rowLong = new ArrayList<>(); + for (float db : row) { + rowLong.add(Float.floatToRawIntBits(db)); + } + findBestDictionary(rowLong, state); + compress(rowLong, rowLong.size(), state); + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdCompressionState.java b/src/main/java/com/github/Cwida/alp/ALPrdCompressionState.java new file mode 100644 index 0000000..8167b43 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdCompressionState.java @@ -0,0 +1,31 @@ +package com.github.Cwida.alp; + +import java.util.HashMap; +import java.util.Map; + +public class ALPrdCompressionState { + public byte rightBw; + public byte leftBw; + public short exceptionsCount; + public short[] leftPartsDict; + public short[] exceptions; + public short[] exceptionsPositions; + public int leftBpSize; + public int rightBpSize; + public Map leftPartsDictMap = new HashMap<>(); + + public ALPrdCompressionState(int vecterSize) { + this.rightBw = 0; + this.leftBw = 0; + this.exceptionsCount = 0; + leftPartsDict = new short[vecterSize]; + exceptions = new short[vecterSize]; + exceptionsPositions = new short[vecterSize]; + } + + public void reset() { + this.leftBpSize = 0; + this.rightBpSize = 0; + this.exceptionsCount = 0; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdCompressionState32.java b/src/main/java/com/github/Cwida/alp/ALPrdCompressionState32.java new file mode 100644 index 0000000..d98a11d --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdCompressionState32.java @@ -0,0 +1,31 @@ +package com.github.Cwida.alp; + +import java.util.HashMap; +import java.util.Map; + +public class ALPrdCompressionState32 { + public byte rightBw; + public byte leftBw; + public short exceptionsCount; + public short[] leftPartsDict; + public short[] exceptions; + public short[] exceptionsPositions; + public int leftBpSize; + public int rightBpSize; + public Map leftPartsDictMap = new HashMap<>(); + + public ALPrdCompressionState32() { + this.rightBw = 0; + this.leftBw = 0; + this.exceptionsCount = 0; + leftPartsDict = new short[ALPrdConstants.DICTIONARY_SIZE]; + exceptions = new short[ALPrdConstants.DICTIONARY_SIZE]; + exceptionsPositions = new short[ALPrdConstants.DICTIONARY_SIZE]; + } + + public void reset() { + this.leftBpSize = 0; + this.rightBpSize = 0; + this.exceptionsCount = 0; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdConstants.java b/src/main/java/com/github/Cwida/alp/ALPrdConstants.java new file mode 100644 index 0000000..3e11eb3 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdConstants.java @@ -0,0 +1,14 @@ +package com.github.Cwida.alp; + +public class ALPrdConstants { + public static int ALP_VECTOR_SIZE = 1000; + public static final byte DICTIONARY_BW = 3; + public static final byte DICTIONARY_SIZE = 1 << DICTIONARY_BW; // 8 + public static final byte CUTTING_LIMIT = 16; + public static final byte EXCEPTION_SIZE = Short.BYTES; + public static final byte EXCEPTION_POSITION_SIZE = Short.BYTES; + + public static void setVectorSize(int vectorSize){ + ALP_VECTOR_SIZE = vectorSize; + } +} diff --git a/src/main/java/com/github/Cwida/alp/ALPrdDecompression.java b/src/main/java/com/github/Cwida/alp/ALPrdDecompression.java new file mode 100644 index 0000000..a2a5ca6 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdDecompression.java @@ -0,0 +1,72 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; + +public class ALPrdDecompression { + private int nValues; + private byte rightBW; + private final InputBitStream in; + private long[] rightEncoded; + private int[] leftEncoded; + private int[] leftPartsDict; + private int exceptionsCount; + private int[] exceptions; + private int[] exceptionsPositions; + + public ALPrdDecompression(InputBitStream in) { + this.in = in; + } + + public void deserialize() { + try { + nValues = in.readInt(32); + rightBW = (byte) in.readInt(8); + leftEncoded = new int[nValues]; + rightEncoded = new long[nValues]; + for (int i = 0; i < nValues; i++) { + leftEncoded[i] = in.readInt(ALPrdConstants.DICTIONARY_BW); + rightEncoded[i] = in.readLong(rightBW); + } + int leftBW = 64 - rightBW; + leftPartsDict = new int[ALPrdConstants.DICTIONARY_SIZE]; + for (int i = 0; i < ALPrdConstants.DICTIONARY_SIZE; i++) { + leftPartsDict[i] = in.readInt(leftBW); + } + exceptionsCount = in.readInt(16); + exceptions = new int[exceptionsCount]; + exceptionsPositions = new int[exceptionsCount]; + for (int i = 0; i < exceptionsCount; i++) { + exceptions[i] = in.readInt(leftBW); + exceptionsPositions[i] = in.readInt(16); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public double[] decompress() { + long[] outputLong = new long[nValues]; + double[] output = new double[nValues]; + + // Decoding 拼接 + for (int i = 0; i < nValues; i++) { + int left = leftPartsDict[leftEncoded[i]]; + long right = rightEncoded[i]; + outputLong[i] = ((long) left << rightBW) | right; + } + + // Exceptions Patching (exceptions only occur in left parts) + for (int i = 0; i < exceptionsCount; i++) { + long right = rightEncoded[exceptionsPositions[i]]; + int left = exceptions[i]; + outputLong[exceptionsPositions[i]] = (((long) left << rightBW) | right); + } + + for (int i = 0; i < nValues; i++) { + output[i] = Double.longBitsToDouble(outputLong[i]); + } + return output; + } +} \ No newline at end of file diff --git a/src/main/java/com/github/Cwida/alp/ALPrdDecompression32.java b/src/main/java/com/github/Cwida/alp/ALPrdDecompression32.java new file mode 100644 index 0000000..d59ac65 --- /dev/null +++ b/src/main/java/com/github/Cwida/alp/ALPrdDecompression32.java @@ -0,0 +1,72 @@ +package com.github.Cwida.alp; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; + +import java.io.IOException; + +public class ALPrdDecompression32 { + private int nValues; + private byte rightBW; + private final InputBitStream in; + private int[] rightEncoded; + private int[] leftEncoded; + private int[] leftPartsDict; + private int exceptionsCount; + private int[] exceptions; + private int[] exceptionsPositions; + + public ALPrdDecompression32(InputBitStream in) { + this.in = in; + } + + public void deserialize() { + try { + nValues = in.readInt(32); + rightBW = (byte) in.readInt(8); + leftEncoded = new int[nValues]; + rightEncoded = new int[nValues]; + for (int i = 0; i < nValues; i++) { + leftEncoded[i] = in.readInt(ALPrdConstants.DICTIONARY_BW); + rightEncoded[i] = in.readInt(rightBW); + } + int leftBW = 32 - rightBW; + leftPartsDict = new int[ALPrdConstants.DICTIONARY_SIZE]; + for (int i = 0; i < ALPrdConstants.DICTIONARY_SIZE; i++) { + leftPartsDict[i] = in.readInt(leftBW); + } + exceptionsCount = in.readInt(16); + exceptions = new int[exceptionsCount]; + exceptionsPositions = new int[exceptionsCount]; + for (int i = 0; i < exceptionsCount; i++) { + exceptions[i] = in.readInt(leftBW); + exceptionsPositions[i] = in.readInt(16); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public float[] decompress() { + int[] outputLong = new int[nValues]; + float[] output = new float[nValues]; + + // Decoding 拼接 + for (int i = 0; i < nValues; i++) { + int left = leftPartsDict[leftEncoded[i]]; + int right = rightEncoded[i]; + outputLong[i] = (left << rightBW) | right; + } + + // Exceptions Patching (exceptions only occur in left parts) + for (int i = 0; i < exceptionsCount; i++) { + long right = rightEncoded[exceptionsPositions[i]]; + int left = exceptions[i]; + outputLong[exceptionsPositions[i]] = (int) ((left << rightBW) | right); + } + + for (int i = 0; i < nValues; i++) { + output[i] = Float.floatToRawIntBits(outputLong[i]); + } + return output; + } +} \ No newline at end of file diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfCompressor.java index 38149bf..bc650f0 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfCompressor.java @@ -9,7 +9,6 @@ public class ElfCompressor implements ICompressor { private int compressedSizeInBits = 0; private OutputBitStream os; - private int numberOfValues = 0; public ElfCompressor(IXORCompressor xorCompressor) { @@ -17,7 +16,6 @@ public ElfCompressor(IXORCompressor xorCompressor) { os = xorCompressor.getOutputStream(); } - @Override public void addValue(double v) { long vLong = Double.doubleToRawLongBits(v); @@ -29,7 +27,7 @@ public void addValue(double v) { vPrimeLong = vLong; } else if (Double.isNaN(v)) { compressedSizeInBits += os.writeBit(false); - vPrimeLong = 0xfff8000000000000L & vLong; + vPrimeLong = 0x7ff8000000000000L; } else { int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v); int e = ((int) (vLong >> 52)) & 0x7ff; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfPlusCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfPlusCompressor.java index 31fd8fe..076005f 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfPlusCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfPlusCompressor.java @@ -32,7 +32,7 @@ public void addValue(double v) { vPrimeLong = vLong; } else if (Double.isNaN(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 - vPrimeLong = 0xfff8000000000000L & vLong; + vPrimeLong = 0x7ff8000000000000L; } else { // C1: v is a normal or subnormal int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v, lastBetaStar); diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarCompressor.java index 57e1e63..2a27946 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarCompressor.java @@ -25,10 +25,7 @@ public ElfStarCompressor(IXORCompressor xorCompressor, int window) { } public ElfStarCompressor(IXORCompressor xorCompressor) { - this.xorCompressor = xorCompressor; - this.os = xorCompressor.getOutputStream(); - this.betaStarList = new int[1001]; // one for the end sign - this.vPrimeList = new long[1001]; // one for the end sign + this(xorCompressor, 1000); } public void addValue(double v) { @@ -38,7 +35,7 @@ public void addValue(double v) { vPrimeList[numberOfValues] = vLong; betaStarList[numberOfValues] = Integer.MAX_VALUE; } else if (Double.isNaN(v)) { - vPrimeList[numberOfValues] = 0xfff8000000000000L & vLong; + vPrimeList[numberOfValues] = 0x7ff8000000000000L; betaStarList[numberOfValues] = Integer.MAX_VALUE; } else { // C1: v is a normal or subnormal @@ -49,9 +46,7 @@ public void addValue(double v) { long mask = 0xffffffffffffffffL << eraseBits; long delta = (~mask) & vLong; if (delta != 0 && eraseBits > 4) { // C2 - if (alphaAndBetaStar[1] != lastBetaStar) { - lastBetaStar = alphaAndBetaStar[1]; - } + lastBetaStar = alphaAndBetaStar[1]; betaStarList[numberOfValues] = lastBetaStar; vPrimeList[numberOfValues] = mask & vLong; } else { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarHuffmanCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarHuffmanCompressor.java index d79d349..011a2a9 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarHuffmanCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/ElfStarHuffmanCompressor.java @@ -1,28 +1,25 @@ package org.urbcomp.startdb.selfstar.compressor; -import javafx.util.Pair; import org.urbcomp.startdb.selfstar.compressor.xor.IXORCompressor; import org.urbcomp.startdb.selfstar.utils.Elf64Utils; import org.urbcomp.startdb.selfstar.utils.Huffman.HuffmanEncode; +import org.urbcomp.startdb.selfstar.utils.Huffman.Code; import org.urbcomp.startdb.selfstar.utils.OutputBitStream; import java.util.Arrays; -import java.util.HashMap; public class ElfStarHuffmanCompressor implements ICompressor { - private static final int STATES_NUM = 18; - private static final int[] states = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; private final IXORCompressor xorCompressor; private final int[] betaStarList; private final long[] vPrimeList; private final int[] leadDistribution = new int[64]; private final int[] trailDistribution = new int[64]; - private final int[] frequency = new int[STATES_NUM]; private OutputBitStream os; private int compressedSizeInBits = 0; private int lastBetaStar = Integer.MAX_VALUE; private int numberOfValues = 0; - private HashMap> huffmanCode = new HashMap<>(); + private final int[] frequency = new int[18]; // 0 is for 10-i, 17 is for not erasing + private Code[] huffmanCode; public ElfStarHuffmanCompressor(IXORCompressor xorCompressor, int window) { this.xorCompressor = xorCompressor; @@ -84,15 +81,15 @@ private void calculateDistribution() { } private void compress() { - HuffmanEncode huffmanEncode = new HuffmanEncode(states, frequency); + HuffmanEncode huffmanEncode = new HuffmanEncode(frequency); huffmanCode = huffmanEncode.getHuffmanCodes(); compressedSizeInBits += huffmanEncode.writeHuffmanCodes(os); xorCompressor.setDistribution(leadDistribution, trailDistribution); for (int i = 0; i < numberOfValues; i++) { if (betaStarList[i] == Integer.MAX_VALUE) { - compressedSizeInBits += os.writeLong(huffmanCode.get(17).getKey(), huffmanCode.get(17).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase } else { - compressedSizeInBits += os.writeLong(huffmanCode.get(betaStarList[i]).getKey(), huffmanCode.get(betaStarList[i]).getValue()); // case 11, 2 + 4 = 6 + compressedSizeInBits += os.writeLong(huffmanCode[betaStarList[i]].value, huffmanCode[betaStarList[i]].length); // case 11, 2 + 4 = 6 } compressedSizeInBits += xorCompressor.addValue(vPrimeList[i]); } @@ -116,7 +113,7 @@ public void close() { calculateDistribution(); compress(); // we write one more bit here, for marking an end of the stream. - compressedSizeInBits += os.writeLong(huffmanCode.get(17).getKey(), huffmanCode.get(17).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase compressedSizeInBits += xorCompressor.close(); } @@ -130,7 +127,6 @@ public void refresh() { lastBetaStar = Integer.MAX_VALUE; numberOfValues = 0; os = xorCompressor.getOutputStream(); - huffmanCode.clear(); Arrays.fill(frequency, 0); Arrays.fill(leadDistribution, 0); Arrays.fill(trailDistribution, 0); diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/SBaseCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SBaseCompressor.java new file mode 100644 index 0000000..193210c --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SBaseCompressor.java @@ -0,0 +1,65 @@ +package org.urbcomp.startdb.selfstar.compressor; + +import org.urbcomp.startdb.selfstar.compressor.xor.IXORCompressor; + +import java.util.Arrays; + +public class SBaseCompressor implements ICompressor { + private final IXORCompressor xorCompressor; + private int compressedSizeInBits = 0; + private int numberOfValues = 0; + + private double storeCompressionRatio = 0; + + public SBaseCompressor(IXORCompressor xorCompressor) { + this.xorCompressor = xorCompressor; + } + + @Override + public void addValue(double v) { + numberOfValues++; + compressedSizeInBits += xorCompressor.addValue(Double.doubleToRawLongBits(v)); + } + + @Override + public byte[] getBytes() { + int byteCount = (int) Math.ceil(compressedSizeInBits / 8.0); + return Arrays.copyOf(xorCompressor.getOut(), byteCount); + } + + @Override + public void close() { + double thisCompressionRatio = compressedSizeInBits / (numberOfValues * 64.0); + if (storeCompressionRatio < thisCompressionRatio) { + xorCompressor.setDistribution(null, null); + } + storeCompressionRatio = thisCompressionRatio; + compressedSizeInBits += xorCompressor.close(); + } + + @Override + public double getCompressionRatio() { + return compressedSizeInBits / (numberOfValues * 64.0); + } + + @Override + public long getCompressedSizeInBits() { + return compressedSizeInBits; + } + + public String getKey() { + return xorCompressor.getKey(); + } + + @Override + public void refresh() { + compressedSizeInBits = 0; + numberOfValues = 0; + xorCompressor.refresh(); + } + + @Override + public void setDistribution(int[] leadDistribution, int[] trailDistribution) { + // for streaming scenarios, we do nothing here + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarCompressor.java index aa65fc5..7419d9f 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarCompressor.java @@ -34,7 +34,7 @@ public void addValue(double v) { vPrimeLong = vLong; } else if (Double.isNaN(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 - vPrimeLong = 0xfff8000000000000L & vLong; + vPrimeLong = 0x7ff8000000000000L; } else { // C1: v is a normal or subnormal int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v, lastBetaStar); @@ -78,7 +78,6 @@ public void setDistribution(int[] leadDistribution, int[] trailDistribution) { // for streaming scenarios, we do nothing here } - public void close() { double thisCompressionRatio = compressedSizeInBits / (numberOfValues * 64.0); if (storeCompressionRatio < thisCompressionRatio) { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarHuffmanCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarHuffmanCompressor.java index 4176479..6de0c96 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarHuffmanCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/SElfStarHuffmanCompressor.java @@ -1,13 +1,12 @@ package org.urbcomp.startdb.selfstar.compressor; -import javafx.util.Pair; import org.urbcomp.startdb.selfstar.compressor.xor.IXORCompressor; import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.Huffman.Code; import org.urbcomp.startdb.selfstar.utils.Huffman.HuffmanEncode; import org.urbcomp.startdb.selfstar.utils.OutputBitStream; import java.util.Arrays; -import java.util.HashMap; public class SElfStarHuffmanCompressor implements ICompressor { private final IXORCompressor xorCompressor; @@ -22,35 +21,27 @@ public class SElfStarHuffmanCompressor implements ICompressor { private double storeCompressionRatio = 0; - private static final int STATES_NUM = 18; - private static final int[] states = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; - private boolean isFirst = true; - private HashMap> huffmanCode = new HashMap<>(); + private boolean isFirstBlock = true; // mark if it is the first block - private int[] frequency = new int[STATES_NUM]; + private Code[] huffmanCode; - private boolean init = true; + private final int[] frequency = new int[18]; public SElfStarHuffmanCompressor(IXORCompressor xorCompressor) { this.xorCompressor = xorCompressor; os = xorCompressor.getOutputStream(); } + @Override public void addValue(double v) { - if (init) { - HuffmanEncode huffmanEncode = new HuffmanEncode(states, frequency); - huffmanCode = huffmanEncode.getHuffmanCodes(); - frequency = new int[STATES_NUM]; - init = false; - } - if (isFirst) { - addValueFirst(v); - } else { + if (!isFirstBlock) { addValueHuffman(v); + } else { + addValueFirst(v); } } - public void addValueFirst(double v) { + private void addValueFirst(double v) { long vLong = Double.doubleToRawLongBits(v); long vPrimeLong; numberOfValues++; @@ -58,11 +49,11 @@ public void addValueFirst(double v) { if (v == 0.0 || Double.isInfinite(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 vPrimeLong = vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } else if (Double.isNaN(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 vPrimeLong = 0xfff8000000000000L & vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } else { // C1: v is a normal or subnormal int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v, lastBetaStar); @@ -83,25 +74,25 @@ public void addValueFirst(double v) { } else { compressedSizeInBits += os.writeInt(2, 2); // case 10 vPrimeLong = vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } } compressedSizeInBits += xorCompressor.addValue(vPrimeLong); } - public void addValueHuffman(double v) { + private void addValueHuffman(double v) { long vLong = Double.doubleToRawLongBits(v); long vPrimeLong; numberOfValues++; if (v == 0.0 || Double.isInfinite(v)) { - compressedSizeInBits += os.writeLong(huffmanCode.get(STATES_NUM - 1).getKey(), huffmanCode.get(STATES_NUM - 1).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase vPrimeLong = vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } else if (Double.isNaN(v)) { - compressedSizeInBits += os.writeLong(huffmanCode.get(STATES_NUM - 1).getKey(), huffmanCode.get(STATES_NUM - 1).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase vPrimeLong = 0xfff8000000000000L & vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } else { // C1: v is a normal or subnormal int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v, lastBetaStar); @@ -111,19 +102,20 @@ public void addValueHuffman(double v) { long mask = 0xffffffffffffffffL << eraseBits; long delta = (~mask) & vLong; if (delta != 0 && eraseBits > 4) { // C2 - compressedSizeInBits += os.writeLong(huffmanCode.get(alphaAndBetaStar[1]).getKey(), huffmanCode.get(alphaAndBetaStar[1]).getValue()); // case 11, 2 + 4 = 6 + compressedSizeInBits += os.writeLong(huffmanCode[alphaAndBetaStar[1]].value, huffmanCode[alphaAndBetaStar[1]].length); // case 11, 2 + 4 = 6 lastBetaStar = alphaAndBetaStar[1]; vPrimeLong = mask & vLong; frequency[alphaAndBetaStar[1]]++; } else { - compressedSizeInBits += os.writeLong(huffmanCode.get(STATES_NUM - 1).getKey(), huffmanCode.get(STATES_NUM - 1).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase vPrimeLong = vLong; - frequency[STATES_NUM - 1]++; + frequency[17]++; } } compressedSizeInBits += xorCompressor.addValue(vPrimeLong); } + @Override public double getCompressionRatio() { return compressedSizeInBits / (numberOfValues * 64.0); } @@ -133,6 +125,7 @@ public long getCompressedSizeInBits() { return compressedSizeInBits; } + @Override public byte[] getBytes() { int byteCount = (int) Math.ceil(compressedSizeInBits / 8.0); return Arrays.copyOf(xorCompressor.getOut(), byteCount); @@ -143,7 +136,7 @@ public void setDistribution(int[] leadDistribution, int[] trailDistribution) { // for streaming scenarios, we do nothing here } - + @Override public void close() { double thisCompressionRatio = compressedSizeInBits / (numberOfValues * 64.0); if (storeCompressionRatio < thisCompressionRatio) { @@ -152,20 +145,19 @@ public void close() { storeCompressionRatio = thisCompressionRatio; // we write one more bit here, for marking an end of the stream. - if (isFirst) { + if (isFirstBlock) { compressedSizeInBits += os.writeInt(2, 2); // case 10 + isFirstBlock = false; } else { - compressedSizeInBits += os.writeLong(huffmanCode.get(STATES_NUM - 1).getKey(), huffmanCode.get(STATES_NUM - 1).getValue()); // not erase + compressedSizeInBits += os.writeLong(huffmanCode[17].value, huffmanCode[17].length); // not erase } + HuffmanEncode huffmanEncode = new HuffmanEncode(frequency); + huffmanCode = huffmanEncode.getHuffmanCodes(); + Arrays.fill(frequency, 0); compressedSizeInBits += xorCompressor.close(); - - } - - - public String getKey() { - return getClass().getSimpleName(); } + @Override public void refresh() { compressedSizeInBits = 0; lastBetaStar = Integer.MAX_VALUE; @@ -173,6 +165,5 @@ public void refresh() { xorCompressor.refresh(); // note this refresh should be at the last os = xorCompressor.getOutputStream(); - isFirst = false; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpAdaXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpAdaXORCompressor.java new file mode 100644 index 0000000..e561858 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpAdaXORCompressor.java @@ -0,0 +1,155 @@ +package org.urbcomp.startdb.selfstar.compressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; + +import java.util.Arrays; + +/** + * Implements the Chimp time series compression. Value compression + * is for floating points only. + * + * @author Panagiotis Liakos + */ +public class ChimpAdaXORCompressor implements IXORCompressor { + + private final static int THRESHOLD = 6; + private final int[] leadingRepresentation = { + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7 + }; + private final int[] leadingRound = { + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 8, 8, 8, 12, 12, 12, 12, + 16, 16, 18, 18, 20, 20, 22, 22, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 + }; + private final int[] leadDistribution = new int[64]; + private final int capacity = 1000; + private int storedLeadingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private boolean first = true; + private int[] leadPositions = {0, 8, 12, 16, 18, 20, 22, 24}; + private boolean updatePositions = false; + private OutputBitStream out; + + private int leadingBitsPerValue = 3; + + // We should have access to the series? + public ChimpAdaXORCompressor() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + } + + @Override + public OutputBitStream getOutputStream() { + return this.out; + } + + /** + * Adds a new long value to the series. Note, values must be inserted in order. + * + * @param value next floating point value in the series + */ + @Override + public int addValue(long value) { + if (first) { + return PostOfficeSolver.writePositions(leadPositions, out) + writeFirst(value); + } else { + return compressValue(value); + } + } + + private int writeFirst(long value) { + first = false; + storedVal = value; + out.writeLong(storedVal, 64); + return 64; + } + + /** + * Closes the block and writes the remaining stuff to the BitOutput. + */ + @Override + public int close() { + int thisSize = addValue(Elf64Utils.END_SIGN) + out.writeBit(false); + out.flush(); + if (updatePositions) { + // we update distribution using the inner info + leadPositions = PostOfficeSolver.initRoundAndRepresentation(leadDistribution, leadingRepresentation, leadingRound); + leadingBitsPerValue = PostOfficeSolver.positionLength2Bits[leadPositions.length]; + } + return thisSize; + } + + private int compressValue(long value) { + int thisSize = 0; + long xor = storedVal ^ value; + if (xor == 0) { + // Write 0 + out.writeInt(0,2); + thisSize += 2; + storedLeadingZeros = 65; + } else { + int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)]; + int trailingZeros = Long.numberOfTrailingZeros(xor); + leadDistribution[Long.numberOfLeadingZeros(xor)]++; + + if (trailingZeros > THRESHOLD) { + int significantBits = 64 - leadingZeros - trailingZeros; + out.writeInt(1,2); + out.writeInt(leadingRepresentation[leadingZeros], leadingBitsPerValue); + out.writeInt(significantBits, 6); + out.writeLong(xor >>> trailingZeros, significantBits); // Store the meaningful bits of XOR + thisSize += 8 + leadingBitsPerValue + significantBits; + storedLeadingZeros = 65; + } else if (leadingZeros == storedLeadingZeros) { + out.writeInt(2,2); + int significantBits = 64 - leadingZeros; + out.writeLong(xor, significantBits); + thisSize += 2 + significantBits; + } else { + storedLeadingZeros = leadingZeros; + int significantBits = 64 - leadingZeros; + out.writeInt(3,2); + out.writeInt(leadingRepresentation[leadingZeros], leadingBitsPerValue); + out.writeLong(xor, significantBits); + thisSize += 2 + leadingBitsPerValue + significantBits; + } + } + storedVal = value; + return thisSize; + } + + @Override + public byte[] getOut() { + return out.getBuffer(); + } + + @Override + public void refresh() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + storedLeadingZeros = Integer.MAX_VALUE; + storedVal = 0; + first = true; + updatePositions = false; + Arrays.fill(leadDistribution, 0); + } + + @Override + public void setDistribution(int[] leadDistributionIgnore, int[] trailDistributionIgnore) { + this.updatePositions = true; + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNAdaXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNAdaXORCompressor.java new file mode 100644 index 0000000..36f1447 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNAdaXORCompressor.java @@ -0,0 +1,192 @@ +package org.urbcomp.startdb.selfstar.compressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; + +import java.util.Arrays; + +/** + * Implements the Chimp128 time series compression. Value compression + * is for floating points only. + * + * @author Panagiotis Liakos + */ +public class ChimpNAdaXORCompressor implements IXORCompressor { + + private final int[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7 + }; + private final int[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0, + 8, 8, 8, 8, 12, 12, 12, 12, + 16, 16, 18, 18, 20, 20, 22, 22, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 + }; + private final long[] storedValues; + private final int threshold; + private final int previousValues; + private final int setLsb; + private final int[] indices; + private final int flagOneSize; + private final int flagZeroSize; + private final int capacity = 1000; + private final int[] leadDistribution = new int[64]; + private int storedLeadingZeros = Integer.MAX_VALUE; + private boolean first = true; + private OutputBitStream out; + private int index = 0; + private int current = 0; + private int[] leadPositions = {0, 8, 12, 16, 18, 20, 22, 24}; + private boolean updatePositions = false; + private int leadingBitsPerValue = 3; + + + // We should have access to the series? + public ChimpNAdaXORCompressor(int previousValues) { +// out = output; + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + this.previousValues = previousValues; + int previousValuesLog2 = (int) (Math.log(previousValues) / Math.log(2)); + this.threshold = 6 + previousValuesLog2; + this.setLsb = (int) Math.pow(2, threshold + 1) - 1; + this.indices = new int[(int) Math.pow(2, threshold + 1)]; + this.storedValues = new long[previousValues]; + this.flagZeroSize = previousValuesLog2 + 2; + this.flagOneSize = previousValuesLog2 + 8; + } + + public OutputBitStream getOutputStream() { + return out; + } + + + /** + * Adds a new long value to the series. Note, values must be inserted in order. + * + * @param value next floating point value in the series + */ + public int addValue(long value) { + if (first) { + return PostOfficeSolver.writePositions(leadPositions, out) + writeFirst(value); + } else { + return compressValue(value); + } + } + + private int writeFirst(long value) { + first = false; + storedValues[current] = value; + out.writeLong(storedValues[current], 64); + indices[(int) value & setLsb] = index; + return 64; + } + + /** + * Closes the block and writes the remaining stuff to the BitOutput. + */ + @Override + public int close() { + int thisSize = addValue(Elf64Utils.END_SIGN); + out.flush(); + if (updatePositions) { + // we update distribution using the inner info + leadPositions = PostOfficeSolver.initRoundAndRepresentation(leadDistribution, leadingRepresentation, leadingRound); + leadingBitsPerValue = PostOfficeSolver.positionLength2Bits[leadPositions.length]; + } + return thisSize; + } + + private int compressValue(long value) { + int thisSize = 0; + int key = (int) value & setLsb; + long xor; + int previousIndex; + int trailingZeros = 0; + int currIndex = indices[key]; + if ((index - currIndex) < previousValues) { + long tempXor = value ^ storedValues[currIndex % previousValues]; + trailingZeros = Long.numberOfTrailingZeros(tempXor); + if (trailingZeros > threshold) { + previousIndex = currIndex % previousValues; + xor = tempXor; + } else { + previousIndex = index % previousValues; + xor = storedValues[previousIndex] ^ value; + } + } else { + previousIndex = index % previousValues; + xor = storedValues[previousIndex] ^ value; + } + + if (xor == 0) { + out.writeInt(previousIndex, this.flagZeroSize); + thisSize += this.flagZeroSize; + storedLeadingZeros = 65; + } else { + int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)]; + leadDistribution[Long.numberOfLeadingZeros(xor)]++; + + if (trailingZeros > threshold) { + int significantBits = 64 - leadingZeros - trailingZeros; + out.writeInt(((previousValues + previousIndex) << (leadingBitsPerValue + 6)) | (leadingRepresentation[leadingZeros] << 6) | significantBits, this.flagOneSize + leadingBitsPerValue); + out.writeLong(xor >>> trailingZeros, significantBits); // Store the meaningful bits of XOR + thisSize += significantBits + this.flagOneSize + leadingBitsPerValue; + storedLeadingZeros = 65; + } else if (leadingZeros == storedLeadingZeros) { + out.writeInt(2, 2); + int significantBits = 64 - leadingZeros; + out.writeLong(xor, significantBits); + thisSize += 2 + significantBits; + } else { + storedLeadingZeros = leadingZeros; + int significantBits = 64 - leadingZeros; + out.writeInt(3 << leadingBitsPerValue | leadingRepresentation[leadingZeros], 2 + leadingBitsPerValue); + + out.writeLong(xor, significantBits); + thisSize += 2 + leadingBitsPerValue + significantBits; + } + } + current = (current + 1) % previousValues; + storedValues[current] = value; + index++; + indices[key] = index; + return thisSize; + } + + + @Override + public byte[] getOut() { + return out.getBuffer(); + } + + @Override + public void refresh() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + storedLeadingZeros = Integer.MAX_VALUE; + first = true; + index = 0; + current = 0; + updatePositions = false; + Arrays.fill(storedValues, 0); + Arrays.fill(leadDistribution, 0); + Arrays.fill(indices, 0); + } + + @Override + public void setDistribution(int[] leadDistributionIgnore, int[] trailDistributionIgnore) { + this.updatePositions = true; + } + +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNXORCompressor.java index 013b782..72bf307 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpNXORCompressor.java @@ -13,7 +13,7 @@ */ public class ChimpNXORCompressor implements IXORCompressor { - public final static short[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0, + private final static short[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, @@ -22,7 +22,7 @@ public class ChimpNXORCompressor implements IXORCompressor { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }; - public final static short[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0, + private final static short[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 12, 12, 12, 12, 16, 16, 18, 18, 20, 20, 22, 22, 24, 24, 24, 24, 24, 24, 24, 24, @@ -38,7 +38,7 @@ public class ChimpNXORCompressor implements IXORCompressor { private final int[] indices; private final int flagOneSize; private final int flagZeroSize; - private final int capacity = 1000; + private final int capacity; private int storedLeadingZeros = Integer.MAX_VALUE; private boolean first = true; private OutputBitStream out; @@ -47,7 +47,11 @@ public class ChimpNXORCompressor implements IXORCompressor { // We should have access to the series? public ChimpNXORCompressor(int previousValues) { -// out = output; + this(previousValues, 1000); + } + + public ChimpNXORCompressor(int previousValues, int block) { + capacity = block; out = new OutputBitStream( new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); this.previousValues = previousValues; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpXORCompressor.java index 2046bcd..83beb9c 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ChimpXORCompressor.java @@ -11,8 +11,8 @@ */ public class ChimpXORCompressor implements IXORCompressor { - public final static int THRESHOLD = 6; - public final static short[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0, + private final static int THRESHOLD = 6; + private final static short[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, @@ -21,7 +21,7 @@ public class ChimpXORCompressor implements IXORCompressor { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }; - public final static short[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0, + private final static short[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 12, 12, 12, 12, 16, 16, 18, 18, 20, 20, 22, 22, 24, 24, 24, 24, 24, 24, 24, 24, @@ -30,7 +30,7 @@ public class ChimpXORCompressor implements IXORCompressor { 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; - private final int capacity = 1000; + private final int capacity; private int storedLeadingZeros = Integer.MAX_VALUE; private long storedVal = 0; private boolean first = true; @@ -38,6 +38,11 @@ public class ChimpXORCompressor implements IXORCompressor { // We should have access to the series? public ChimpXORCompressor() { + this(1000); + } + + public ChimpXORCompressor(int block) { + capacity = block; out = new OutputBitStream( new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); } @@ -72,8 +77,7 @@ private int writeFirst(long value) { */ @Override public int close() { - int thisSize = addValue(Elf64Utils.END_SIGN); - out.writeBit(false); + int thisSize = addValue(Elf64Utils.END_SIGN) + out.writeBit(false); out.flush(); return thisSize; } @@ -83,8 +87,7 @@ private int compressValue(long value) { long xor = storedVal ^ value; if (xor == 0) { // Write 0 - out.writeBit(false); - out.writeBit(false); + out.writeInt(0,2); thisSize += 2; storedLeadingZeros = 65; } else { @@ -93,24 +96,21 @@ private int compressValue(long value) { if (trailingZeros > THRESHOLD) { int significantBits = 64 - leadingZeros - trailingZeros; - out.writeBit(false); - out.writeBit(true); + out.writeInt(1,2); out.writeInt(leadingRepresentation[leadingZeros], 3); out.writeInt(significantBits, 6); out.writeLong(xor >>> trailingZeros, significantBits); // Store the meaningful bits of XOR thisSize += 11 + significantBits; storedLeadingZeros = 65; } else if (leadingZeros == storedLeadingZeros) { - out.writeBit(true); - out.writeBit(false); + out.writeInt(2,2); int significantBits = 64 - leadingZeros; out.writeLong(xor, significantBits); thisSize += 2 + significantBits; } else { storedLeadingZeros = leadingZeros; int significantBits = 64 - leadingZeros; - out.writeBit(true); - out.writeBit(true); + out.writeInt(3,2); out.writeInt(leadingRepresentation[leadingZeros], 3); out.writeLong(xor, significantBits); thisSize += 5 + significantBits; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfPlusXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfPlusXORCompressor.java index 3987553..8071569 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfPlusXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfPlusXORCompressor.java @@ -30,11 +30,10 @@ public class ElfPlusXORCompressor implements IXORCompressor { private boolean first = true; private OutputBitStream out; - private int capacity = 1000; + private final int capacity; public ElfPlusXORCompressor() { - out = new OutputBitStream( - new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + this(1000); } public ElfPlusXORCompressor(int capacity) { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressor.java index 8ea5c29..611f141 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressor.java @@ -24,11 +24,10 @@ public class ElfStarXORCompressor implements IXORCompressor { private int trailingBitsPerValue; - private int capacity = 1000; + private final int capacity; public ElfStarXORCompressor() { - out = new OutputBitStream( - new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + this(1000); } public ElfStarXORCompressor(int window) { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRPruning.java new file mode 100644 index 0000000..16b7ef7 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRPruning.java @@ -0,0 +1,164 @@ +package org.urbcomp.startdb.selfstar.compressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolverNoFRPruning; + +import java.util.Arrays; + +// without first-rear pruning +public class ElfStarXORCompressorNoFRPruning implements IXORCompressor { + private final int[] leadingRepresentation = new int[64]; + private final int[] leadingRound = new int[64]; + private final int[] trailingRepresentation = new int[64]; + private final int[] trailingRound = new int[64]; + private int storedLeadingZeros = Integer.MAX_VALUE; + private int storedTrailingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private boolean first = true; + private int[] leadDistribution; + private int[] trailDistribution; + + private OutputBitStream out; + + private int leadingBitsPerValue; + + private int trailingBitsPerValue; + + private final int capacity = 1000; + + public ElfStarXORCompressorNoFRPruning() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + } + + @Override + public OutputBitStream getOutputStream() { + return this.out; + } + + private int initLeadingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRPruning.initRoundAndRepresentation(distribution, leadingRepresentation, leadingRound); + leadingBitsPerValue = PostOfficeSolverNoFRPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRPruning.writePositions(positions, out); + } + + private int initTrailingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRPruning.initRoundAndRepresentation(distribution, trailingRepresentation, trailingRound); + trailingBitsPerValue = PostOfficeSolverNoFRPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRPruning.writePositions(positions, out); + } + + /** + * Adds a new long value to the series. Note, values must be inserted in order. + * + * @param value next floating point value in the series + */ + @Override + public int addValue(long value) { + if (first) { + return initLeadingRoundAndRepresentation(leadDistribution) + + initTrailingRoundAndRepresentation(trailDistribution) + + writeFirst(value); + } else { + return compressValue(value); + } + } + + private int writeFirst(long value) { + first = false; + storedVal = value; + int trailingZeros = Long.numberOfTrailingZeros(value); + out.writeInt(trailingZeros, 7); + if (trailingZeros < 64) { + out.writeLong(storedVal >>> (trailingZeros + 1), 63 - trailingZeros); + return 70 - trailingZeros; + } else { + return 7; + } + } + + /** + * Closes the block and writes the remaining stuff to the BitOutput. + */ + @Override + public int close() { + int thisSize = addValue(Elf64Utils.END_SIGN); + out.flush(); + return thisSize; + } + + private int compressValue(long value) { + int thisSize = 0; + long xor = storedVal ^ value; + + if (xor == 0) { + // case 01 + out.writeInt(1, 2); + thisSize += 2; + } else { + int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)]; + int trailingZeros = trailingRound[Long.numberOfTrailingZeros(xor)]; + + if (leadingZeros >= storedLeadingZeros && trailingZeros >= storedTrailingZeros && + (leadingZeros - storedLeadingZeros) + (trailingZeros - storedTrailingZeros) < 1 + leadingBitsPerValue + trailingBitsPerValue) { + // case 1 + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + int len = 1 + centerBits; + if (len > 64) { + out.writeInt(1, 1); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong((1L << centerBits) | (xor >>> storedTrailingZeros), 1 + centerBits); + } + thisSize += len; + } else { + storedLeadingZeros = leadingZeros; + storedTrailingZeros = trailingZeros; + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + + // case 00 + int len = 2 + leadingBitsPerValue + trailingBitsPerValue + centerBits; + if (len > 64) { + out.writeInt((leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) + | trailingRepresentation[storedTrailingZeros], 2 + leadingBitsPerValue + trailingBitsPerValue); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong( + ((((long) leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) | + trailingRepresentation[storedTrailingZeros]) << centerBits) | (xor >>> storedTrailingZeros), + len + ); + } + thisSize += len; + } + storedVal = value; + } + return thisSize; + } + + @Override + public byte[] getOut() { + return out.getBuffer(); + } + + @Override + public void refresh() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + storedLeadingZeros = Integer.MAX_VALUE; + storedTrailingZeros = Integer.MAX_VALUE; + storedVal = 0; + first = true; + Arrays.fill(leadingRepresentation, 0); + Arrays.fill(leadingRound, 0); + Arrays.fill(trailingRepresentation, 0); + Arrays.fill(trailingRound, 0); + } + + @Override + public void setDistribution(int[] leadDistribution, int[] trailDistribution) { + this.leadDistribution = leadDistribution; + this.trailDistribution = trailDistribution; + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZGPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZGPruning.java new file mode 100644 index 0000000..1d16b02 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZGPruning.java @@ -0,0 +1,164 @@ +package org.urbcomp.startdb.selfstar.compressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolverNoFRZGPruning; + +import java.util.Arrays; + +// without first-rear pruning, zero-pruning and global pruning +public class ElfStarXORCompressorNoFRZGPruning implements IXORCompressor { + private final int[] leadingRepresentation = new int[64]; + private final int[] leadingRound = new int[64]; + private final int[] trailingRepresentation = new int[64]; + private final int[] trailingRound = new int[64]; + private int storedLeadingZeros = Integer.MAX_VALUE; + private int storedTrailingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private boolean first = true; + private int[] leadDistribution; + private int[] trailDistribution; + + private OutputBitStream out; + + private int leadingBitsPerValue; + + private int trailingBitsPerValue; + + private final int capacity = 1000; + + public ElfStarXORCompressorNoFRZGPruning() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + } + + @Override + public OutputBitStream getOutputStream() { + return this.out; + } + + private int initLeadingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRZGPruning.initRoundAndRepresentation(distribution, leadingRepresentation, leadingRound); + leadingBitsPerValue = PostOfficeSolverNoFRZGPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRZGPruning.writePositions(positions, out); + } + + private int initTrailingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRZGPruning.initRoundAndRepresentation(distribution, trailingRepresentation, trailingRound); + trailingBitsPerValue = PostOfficeSolverNoFRZGPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRZGPruning.writePositions(positions, out); + } + + /** + * Adds a new long value to the series. Note, values must be inserted in order. + * + * @param value next floating point value in the series + */ + @Override + public int addValue(long value) { + if (first) { + return initLeadingRoundAndRepresentation(leadDistribution) + + initTrailingRoundAndRepresentation(trailDistribution) + + writeFirst(value); + } else { + return compressValue(value); + } + } + + private int writeFirst(long value) { + first = false; + storedVal = value; + int trailingZeros = Long.numberOfTrailingZeros(value); + out.writeInt(trailingZeros, 7); + if (trailingZeros < 64) { + out.writeLong(storedVal >>> (trailingZeros + 1), 63 - trailingZeros); + return 70 - trailingZeros; + } else { + return 7; + } + } + + /** + * Closes the block and writes the remaining stuff to the BitOutput. + */ + @Override + public int close() { + int thisSize = addValue(Elf64Utils.END_SIGN); + out.flush(); + return thisSize; + } + + private int compressValue(long value) { + int thisSize = 0; + long xor = storedVal ^ value; + + if (xor == 0) { + // case 01 + out.writeInt(1, 2); + thisSize += 2; + } else { + int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)]; + int trailingZeros = trailingRound[Long.numberOfTrailingZeros(xor)]; + + if (leadingZeros >= storedLeadingZeros && trailingZeros >= storedTrailingZeros && + (leadingZeros - storedLeadingZeros) + (trailingZeros - storedTrailingZeros) < 1 + leadingBitsPerValue + trailingBitsPerValue) { + // case 1 + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + int len = 1 + centerBits; + if (len > 64) { + out.writeInt(1, 1); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong((1L << centerBits) | (xor >>> storedTrailingZeros), 1 + centerBits); + } + thisSize += len; + } else { + storedLeadingZeros = leadingZeros; + storedTrailingZeros = trailingZeros; + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + + // case 00 + int len = 2 + leadingBitsPerValue + trailingBitsPerValue + centerBits; + if (len > 64) { + out.writeInt((leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) + | trailingRepresentation[storedTrailingZeros], 2 + leadingBitsPerValue + trailingBitsPerValue); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong( + ((((long) leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) | + trailingRepresentation[storedTrailingZeros]) << centerBits) | (xor >>> storedTrailingZeros), + len + ); + } + thisSize += len; + } + storedVal = value; + } + return thisSize; + } + + @Override + public byte[] getOut() { + return out.getBuffer(); + } + + @Override + public void refresh() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + storedLeadingZeros = Integer.MAX_VALUE; + storedTrailingZeros = Integer.MAX_VALUE; + storedVal = 0; + first = true; + Arrays.fill(leadingRepresentation, 0); + Arrays.fill(leadingRound, 0); + Arrays.fill(trailingRepresentation, 0); + Arrays.fill(trailingRound, 0); + } + + @Override + public void setDistribution(int[] leadDistribution, int[] trailDistribution) { + this.leadDistribution = leadDistribution; + this.trailDistribution = trailDistribution; + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZPruning.java new file mode 100644 index 0000000..de443ab --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfStarXORCompressorNoFRZPruning.java @@ -0,0 +1,164 @@ +package org.urbcomp.startdb.selfstar.compressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.OutputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolverNoFRZPruning; + +import java.util.Arrays; + +// without first-rear pruning and zero-pruning +public class ElfStarXORCompressorNoFRZPruning implements IXORCompressor { + private final int[] leadingRepresentation = new int[64]; + private final int[] leadingRound = new int[64]; + private final int[] trailingRepresentation = new int[64]; + private final int[] trailingRound = new int[64]; + private int storedLeadingZeros = Integer.MAX_VALUE; + private int storedTrailingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private boolean first = true; + private int[] leadDistribution; + private int[] trailDistribution; + + private OutputBitStream out; + + private int leadingBitsPerValue; + + private int trailingBitsPerValue; + + private final int capacity = 1000; + + public ElfStarXORCompressorNoFRZPruning() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + } + + @Override + public OutputBitStream getOutputStream() { + return this.out; + } + + private int initLeadingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRZPruning.initRoundAndRepresentation(distribution, leadingRepresentation, leadingRound); + leadingBitsPerValue = PostOfficeSolverNoFRZPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRZPruning.writePositions(positions, out); + } + + private int initTrailingRoundAndRepresentation(int[] distribution) { + int[] positions = PostOfficeSolverNoFRZPruning.initRoundAndRepresentation(distribution, trailingRepresentation, trailingRound); + trailingBitsPerValue = PostOfficeSolverNoFRZPruning.positionLength2Bits[positions.length]; + return PostOfficeSolverNoFRZPruning.writePositions(positions, out); + } + + /** + * Adds a new long value to the series. Note, values must be inserted in order. + * + * @param value next floating point value in the series + */ + @Override + public int addValue(long value) { + if (first) { + return initLeadingRoundAndRepresentation(leadDistribution) + + initTrailingRoundAndRepresentation(trailDistribution) + + writeFirst(value); + } else { + return compressValue(value); + } + } + + private int writeFirst(long value) { + first = false; + storedVal = value; + int trailingZeros = Long.numberOfTrailingZeros(value); + out.writeInt(trailingZeros, 7); + if (trailingZeros < 64) { + out.writeLong(storedVal >>> (trailingZeros + 1), 63 - trailingZeros); + return 70 - trailingZeros; + } else { + return 7; + } + } + + /** + * Closes the block and writes the remaining stuff to the BitOutput. + */ + @Override + public int close() { + int thisSize = addValue(Elf64Utils.END_SIGN); + out.flush(); + return thisSize; + } + + private int compressValue(long value) { + int thisSize = 0; + long xor = storedVal ^ value; + + if (xor == 0) { + // case 01 + out.writeInt(1, 2); + thisSize += 2; + } else { + int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)]; + int trailingZeros = trailingRound[Long.numberOfTrailingZeros(xor)]; + + if (leadingZeros >= storedLeadingZeros && trailingZeros >= storedTrailingZeros && + (leadingZeros - storedLeadingZeros) + (trailingZeros - storedTrailingZeros) < 1 + leadingBitsPerValue + trailingBitsPerValue) { + // case 1 + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + int len = 1 + centerBits; + if (len > 64) { + out.writeInt(1, 1); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong((1L << centerBits) | (xor >>> storedTrailingZeros), 1 + centerBits); + } + thisSize += len; + } else { + storedLeadingZeros = leadingZeros; + storedTrailingZeros = trailingZeros; + int centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + + // case 00 + int len = 2 + leadingBitsPerValue + trailingBitsPerValue + centerBits; + if (len > 64) { + out.writeInt((leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) + | trailingRepresentation[storedTrailingZeros], 2 + leadingBitsPerValue + trailingBitsPerValue); + out.writeLong(xor >>> storedTrailingZeros, centerBits); + } else { + out.writeLong( + ((((long) leadingRepresentation[storedLeadingZeros] << trailingBitsPerValue) | + trailingRepresentation[storedTrailingZeros]) << centerBits) | (xor >>> storedTrailingZeros), + len + ); + } + thisSize += len; + } + storedVal = value; + } + return thisSize; + } + + @Override + public byte[] getOut() { + return out.getBuffer(); + } + + @Override + public void refresh() { + out = new OutputBitStream( + new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + storedLeadingZeros = Integer.MAX_VALUE; + storedTrailingZeros = Integer.MAX_VALUE; + storedVal = 0; + first = true; + Arrays.fill(leadingRepresentation, 0); + Arrays.fill(leadingRound, 0); + Arrays.fill(trailingRepresentation, 0); + Arrays.fill(trailingRound, 0); + } + + @Override + public void setDistribution(int[] leadDistribution, int[] trailDistribution) { + this.leadDistribution = leadDistribution; + this.trailDistribution = trailDistribution; + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfXORCompressor.java index 481a94c..2c42819 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/ElfXORCompressor.java @@ -22,7 +22,7 @@ public class ElfXORCompressor implements IXORCompressor { 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; - private final int capacity = 1000; + private final int capacity; private int storedLeadingZeros = Integer.MAX_VALUE; private int storedTrailingZeros = Integer.MAX_VALUE; private long storedVal = 0; @@ -30,12 +30,16 @@ public class ElfXORCompressor implements IXORCompressor { private boolean first = true; private OutputBitStream out; - - public ElfXORCompressor() { + public ElfXORCompressor(int block) { + capacity = block; out = new OutputBitStream( new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); } + public ElfXORCompressor() { + this(1000); + } + public OutputBitStream getOutputStream() { return this.out; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/GorillaXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/GorillaXORCompressor.java index bea8d54..811df7a 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/GorillaXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/GorillaXORCompressor.java @@ -4,7 +4,7 @@ import org.urbcomp.startdb.selfstar.utils.OutputBitStream; public class GorillaXORCompressor implements IXORCompressor { - private final int capacity = 1000; + private final int capacity; private int storedLeadingZeros = Integer.MAX_VALUE; private int storedTrailingZeros = 0; private long storedVal = 0; @@ -12,6 +12,11 @@ public class GorillaXORCompressor implements IXORCompressor { private OutputBitStream out; public GorillaXORCompressor() { + this(1000); + } + + public GorillaXORCompressor(int block) { + capacity = block; out = new OutputBitStream( new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/SElfXORCompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/SElfXORCompressor.java index 86a07b5..4f3e4d7 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/SElfXORCompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor/xor/SElfXORCompressor.java @@ -56,13 +56,14 @@ public class SElfXORCompressor implements IXORCompressor { private int[] leadPositions = {0, 8, 12, 16, 18, 20, 22, 24}; private int[] trailPositions = {0, 22, 28, 32, 36, 40, 42, 46}; private boolean updatePositions = false; + private boolean writePositions = false; private OutputBitStream out; private int leadingBitsPerValue = 3; private int trailingBitsPerValue = 3; - private int capacity = 1000; + private final int capacity; public SElfXORCompressor(int window) { @@ -73,8 +74,7 @@ public SElfXORCompressor(int window) { } public SElfXORCompressor() { - out = new OutputBitStream( - new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); + this(1000); } @Override @@ -90,9 +90,14 @@ public OutputBitStream getOutputStream() { @Override public int addValue(long value) { if (first) { - return PostOfficeSolver.writePositions(leadPositions, out) - + PostOfficeSolver.writePositions(trailPositions, out) - + writeFirst(value); + if (writePositions) { + return out.writeBit(true) + + PostOfficeSolver.writePositions(leadPositions, out) + + PostOfficeSolver.writePositions(trailPositions, out) + + writeFirst(value); + } else { + return out.writeBit(false) + writeFirst(value); + } } else { return compressValue(value); } @@ -126,6 +131,7 @@ public int close() { trailPositions = PostOfficeSolver.initRoundAndRepresentation(trailDistribution, trailingRepresentation, trailingRound); trailingBitsPerValue = PostOfficeSolver.positionLength2Bits[trailPositions.length]; } + writePositions = updatePositions; return thisSize; } @@ -190,9 +196,6 @@ public byte[] getOut() { public void refresh() { out = new OutputBitStream( new byte[(int) (((capacity + 1) * 8 + capacity / 8 + 1) * 1.2)]); - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; - storedVal = 0; first = true; updatePositions = false; Arrays.fill(leadDistribution, 0); diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfCompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfCompressor32.java index db3769a..9eec4d3 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfCompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfCompressor32.java @@ -17,7 +17,6 @@ public ElfCompressor32(IXORCompressor32 xorCompressor) { os = xorCompressor.getOutputStream(); } - @Override public void addValue(float v) { int vInt = Float.floatToRawIntBits(v); @@ -29,7 +28,7 @@ public void addValue(float v) { vPrimeInt = vInt; } else if (Float.isNaN(v)) { compressedSizeInBits += os.writeBit(false); - vPrimeInt = 0xffc00000 & vInt; + vPrimeInt = 0x7fc00000; } else { int[] alphaAndBetaStar = Elf32Utils.getAlphaAndBetaStar(v); int e = (vInt >> 23) & 0xff; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfPlusCompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfPlusCompressor32.java index 90899a9..4bcf6d9 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfPlusCompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfPlusCompressor32.java @@ -32,7 +32,7 @@ public void addValue(float v) { vPrimeInt = vInt; } else if (Float.isNaN(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 - vPrimeInt = 0xffc00000 & vInt; + vPrimeInt = 0x7fc00000; } else { //1 // C1: v is a normal or subnormal diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfStarCompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfStarCompressor32.java index e1f4872..eb0be20 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfStarCompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/ElfStarCompressor32.java @@ -17,13 +17,6 @@ public class ElfStarCompressor32 implements ICompressor32 { private int lastBetaStar = Integer.MAX_VALUE; private int numberOfValues = 0; - public ElfStarCompressor32(IXORCompressor32 xorCompressor, int window) { - this.xorCompressor = xorCompressor; - this.os = xorCompressor.getOutputStream(); - this.betaStarList = new int[window + 1]; // one for the end sign - this.vPrimeList = new int[window + 1]; // one for the end sign - } - public ElfStarCompressor32(IXORCompressor32 xorCompressor) { this.xorCompressor = xorCompressor; this.os = xorCompressor.getOutputStream(); @@ -38,7 +31,7 @@ public void addValue(float v) { vPrimeList[numberOfValues] = vInt; betaStarList[numberOfValues] = Integer.MAX_VALUE; } else if (Float.isNaN(v)) { - vPrimeList[numberOfValues] = 0xffc00000 & vInt; + vPrimeList[numberOfValues] = 0x7fc00000; betaStarList[numberOfValues] = Integer.MAX_VALUE; } else { // C1: v is a normal or subnormal diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/SElfStarCompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/SElfStarCompressor32.java index e656b10..96b0e18 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/SElfStarCompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/SElfStarCompressor32.java @@ -34,7 +34,7 @@ public void addValue(float v) { vPrimeInt = vInt; } else if (Float.isNaN(v)) { compressedSizeInBits += os.writeInt(2, 2); // case 10 - vPrimeInt = 0xffc00000 & vInt; + vPrimeInt = 0x7fc00000; } else { // C1: v is a normal or subnormal int[] alphaAndBetaStar = Elf32Utils.getAlphaAndBetaStar(v, lastBetaStar); diff --git a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/xor/SElfXORCompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/xor/SElfXORCompressor32.java index 91826d3..b9d4ffc 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/compressor32/xor/SElfXORCompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/compressor32/xor/SElfXORCompressor32.java @@ -38,6 +38,7 @@ public class SElfXORCompressor32 implements IXORCompressor32 { private int[] leadPositions = {0, 8, 12, 16}; private int[] trailPositions = {0, 16}; private boolean updatePositions = false; + private boolean writePositions = false; private OutputBitStream out; private int leadingBitsPerValue = 2; @@ -72,9 +73,14 @@ public OutputBitStream getOutputStream() { @Override public int addValue(int value) { if (first) { - return PostOfficeSolver32.writePositions(leadPositions, out) - + PostOfficeSolver32.writePositions(trailPositions, out) - + writeFirst(value); + if (writePositions) { + return out.writeBit(true) + + PostOfficeSolver32.writePositions(leadPositions, out) + + PostOfficeSolver32.writePositions(trailPositions, out) + + writeFirst(value); + } else { + return out.writeBit(false) + writeFirst(value); + } } else { return compressValue(value); } @@ -108,6 +114,7 @@ public int close() { trailPositions = PostOfficeSolver32.initRoundAndRepresentation(trailDistribution, trailingRepresentation, trailingRound); trailingBitsPerValue = PostOfficeSolver32.positionLength2Bits[trailPositions.length]; } + writePositions = updatePositions; return thisSize; } @@ -172,9 +179,6 @@ public byte[] getOut() { public void refresh() { out = new OutputBitStream( new byte[(int) (((capacity + 1) * 4 + capacity / 4 + 1) * 1.2)]); - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; - storedVal = 0; first = true; updatePositions = false; Arrays.fill(leadDistribution, 0); diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfPlusDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfPlusDecompressor.java index 81c0ae0..d1a13ca 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfPlusDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfPlusDecompressor.java @@ -54,7 +54,7 @@ public Double nextValue() { private Double recoverVByBetaStar() { double v; Double vPrime = xorDecompressor.readValue(); - int sp = Elf64Utils.getSP(Math.abs(vPrime)); + int sp = Elf64Utils.getSP(vPrime < 0 ? -vPrime : vPrime); if (lastBetaStar == 0) { v = Elf64Utils.get10iN(-sp - 1); if (vPrime < 0) { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfStarHuffmanDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfStarHuffmanDecompressor.java index 988bd61..bae58b4 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfStarHuffmanDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/ElfStarHuffmanDecompressor.java @@ -1,23 +1,22 @@ package org.urbcomp.startdb.selfstar.decompressor; -import javafx.util.Pair; import org.urbcomp.startdb.selfstar.decompressor.xor.IXORDecompressor; import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.Huffman.Code; import org.urbcomp.startdb.selfstar.utils.Huffman.HuffmanEncode; import org.urbcomp.startdb.selfstar.utils.Huffman.Node; import org.urbcomp.startdb.selfstar.utils.InputBitStream; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; public class ElfStarHuffmanDecompressor implements IDecompressor { - private static final HashMap> huffmanCode = new HashMap<>(); - private static final int[] states = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; + private static Code[] huffmanCode = new Code[18]; private final IXORDecompressor xorDecompressor; private int lastBetaStar = Integer.MAX_VALUE; private Node root; + private final HuffmanEncode huffmanEncode = new HuffmanEncode(new int[18]); public ElfStarHuffmanDecompressor(IXORDecompressor xorDecompressor) { this.xorDecompressor = xorDecompressor; @@ -33,20 +32,20 @@ public List decompress() { return values; } - public void initHuffmanTree() { - for (int state : states) { + private void initHuffmanTree() { + for (int i = 0; i < huffmanCode.length; i++) { int length = readInt(5); long code = readInt(length); - huffmanCode.put(state, new Pair<>(code, length)); + huffmanCode[i] = new Code(code, length); } - root = HuffmanEncode.hashMapToTree(huffmanCode); + root = huffmanEncode.hashMapToTree(huffmanCode); } @Override public void refresh() { lastBetaStar = Integer.MAX_VALUE; xorDecompressor.refresh(); - huffmanCode.clear(); + huffmanCode = new Code[18]; } @Override @@ -59,11 +58,7 @@ public Double nextValue() { Double v; Node current = root; while (true) { - if (readInt(1) == 0) { - current = current.left; - } else { - current = current.right; - } + current = current.children[readInt(1)]; if (current.data != -Integer.MAX_VALUE) { if (current.data != 17) { lastBetaStar = current.data; diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/SElfStarHuffmanDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/SElfStarHuffmanDecompressor.java index c960ba5..91c992f 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/SElfStarHuffmanDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/SElfStarHuffmanDecompressor.java @@ -1,8 +1,8 @@ package org.urbcomp.startdb.selfstar.decompressor; -import javafx.util.Pair; import org.urbcomp.startdb.selfstar.decompressor.xor.IXORDecompressor; import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.Huffman.Code; import org.urbcomp.startdb.selfstar.utils.Huffman.HuffmanEncode; import org.urbcomp.startdb.selfstar.utils.Huffman.Node; import org.urbcomp.startdb.selfstar.utils.InputBitStream; @@ -10,7 +10,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; // all ElfStar (batch and stream) share the same decompressor @@ -19,11 +18,8 @@ public class SElfStarHuffmanDecompressor implements IDecompressor { private final IXORDecompressor xorDecompressor; private int lastBetaStar = Integer.MAX_VALUE; - private static final int STATES_NUM = 18; - private static final int[] states = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; - private final int[] frequency = new int[STATES_NUM]; + private final int[] frequency = new int[18]; private boolean isFirst = true; - private HashMap> huffmanCode = new HashMap<>(); private Node root; public SElfStarHuffmanDecompressor(IXORDecompressor xorDecompressor) { @@ -36,9 +32,10 @@ public List decompress() { while ((value = nextValue()) != null) { values.add(value); } - HuffmanEncode huffmanEncode = new HuffmanEncode(states, frequency); - huffmanCode = huffmanEncode.getHuffmanCodes(); - root = HuffmanEncode.hashMapToTree(huffmanCode); + frequency[17]--; + HuffmanEncode huffmanEncode = new HuffmanEncode(frequency); + Code[] huffmanCode = huffmanEncode.getHuffmanCodes(); + root = huffmanEncode.hashMapToTree(huffmanCode); Arrays.fill(frequency, 0); return values; } @@ -57,10 +54,10 @@ public void setBytes(byte[] bs) { @Override public Double nextValue() { - if (isFirst) { - return nextValueFirst(); - } else { + if (!isFirst) { return nextValueHuffman(); + } else { + return nextValueFirst(); } } @@ -72,7 +69,7 @@ public Double nextValueFirst() { frequency[lastBetaStar]++; } else if (readInt(1) == 0) { v = xorDecompressor.readValue(); // case 10 - frequency[STATES_NUM - 1]++; + frequency[17]++; } else { lastBetaStar = readInt(4); // case 11 v = recoverVByBetaStar(); @@ -85,20 +82,16 @@ public Double nextValueHuffman() { Double v; Node current = root; while (true) { - if (readInt(1) == 0) { - current = current.left; - } else { - current = current.right; - } + current = current.children[readInt(1)]; if (current.data != -Integer.MAX_VALUE) { - if (current.data != STATES_NUM - 1) { + if (current.data != 17) { lastBetaStar = current.data; v = recoverVByBetaStar(); frequency[lastBetaStar]++; } else { v = xorDecompressor.readValue(); - frequency[STATES_NUM - 1]++; + frequency[17]++; } break; } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpAdaXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpAdaXORDecompressor.java new file mode 100644 index 0000000..c069f92 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpAdaXORDecompressor.java @@ -0,0 +1,148 @@ +package org.urbcomp.startdb.selfstar.decompressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.InputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; + +import java.io.IOException; +import java.util.Arrays; + +/** + * Decompresses a compressed stream created by the Compressor. Returns pairs of timestamp and floating point value. + */ +public class ChimpAdaXORDecompressor implements IXORDecompressor { + + private int[] leadingRepresentation = {0, 8, 12, 16, 18, 20, 22, 24}; + + private int leadingBitsPerValue; + + private int storedLeadingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private boolean first = true; + private boolean endOfStream = false; + private InputBitStream in; + + public ChimpAdaXORDecompressor() { + this(new byte[0]); + } + + public ChimpAdaXORDecompressor(byte[] bs) { + in = new InputBitStream(bs); + } + + private void initLeadingRepresentation() { + try { + int num = in.readInt(5); + if (num == 0) { + num = 32; + } + leadingBitsPerValue = PostOfficeSolver.positionLength2Bits[num]; + leadingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + leadingRepresentation[i] = in.readInt(6); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + public InputBitStream getInputStream() { + return in; + } + + @Override + public void setBytes(byte[] bs) { + in = new InputBitStream(bs); + } + + @Override + public void refresh() { + storedLeadingZeros = Integer.MAX_VALUE; + storedVal = 0; + first = true; + endOfStream = false; + Arrays.fill(leadingRepresentation, 0); + leadingBitsPerValue = 0; + } + + /** + * Returns the next pair in the time series, if available. + * + * @return Pair if there's next value, null if series is done. + */ + public Double readValue() { + try { + next(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + if (endOfStream) { + return null; + } + return Double.longBitsToDouble(storedVal); + } + + private void next() throws IOException { + if (first) { + initLeadingRepresentation(); + first = false; + storedVal = in.readLong(64); + if (storedVal == Elf64Utils.END_SIGN) { + endOfStream = true; + } + } else { + nextValue(); + } + } + + private void nextValue() throws IOException { + + int significantBits; + long value; + // Read value + int flag = in.readInt(2); + switch (flag) { + case 3: + // New leading zeros + storedLeadingZeros = leadingRepresentation[in.readInt(leadingBitsPerValue)]; + value = in.readLong(64 - storedLeadingZeros); + value = storedVal ^ value; + if (value == Elf64Utils.END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + } + break; + case 2: + value = in.readLong(64 - storedLeadingZeros); + value = storedVal ^ value; + if (value == Elf64Utils.END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + } + break; + case 1: + storedLeadingZeros = leadingRepresentation[in.readInt(leadingBitsPerValue)]; + significantBits = in.readInt(6); + if (significantBits == 0) { + significantBits = 64; + } + int storedTrailingZeros = 64 - significantBits - storedLeadingZeros; + value = in.readLong(64 - storedLeadingZeros - storedTrailingZeros); + value <<= storedTrailingZeros; + value = storedVal ^ value; + if (value == Elf64Utils.END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + } + break; + default: + } + } + +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNAdaXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNAdaXORDecompressor.java new file mode 100644 index 0000000..b155038 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNAdaXORDecompressor.java @@ -0,0 +1,168 @@ +package org.urbcomp.startdb.selfstar.decompressor.xor; + +import org.urbcomp.startdb.selfstar.utils.InputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; + +import java.io.IOException; +import java.util.Arrays; + +public class ChimpNAdaXORDecompressor implements IXORDecompressor { + + private final static long END_SIGN = Double.doubleToLongBits(Double.NaN); + private final long[] storedValues; + private final int previousValues; + private final int previousValuesLog2; + private final int initialFill; + private int[] leadingRepresentation = {0, 8, 12, 16, 18, 20, 22, 24}; + private int storedLeadingZeros = Integer.MAX_VALUE; + private long storedVal = 0; + private int current = 0; + private boolean first = true; + private boolean endOfStream = false; + private InputBitStream in; + private int leadingBitsPerValue; + + public ChimpNAdaXORDecompressor(int previousValues) { + this(new byte[0], previousValues); + } + + public ChimpNAdaXORDecompressor(byte[] bs, int previousValues) { + in = new InputBitStream(bs); + this.previousValues = previousValues; + this.previousValuesLog2 = (int) (Math.log(previousValues) / Math.log(2)); + this.initialFill = previousValuesLog2 + 6; + this.storedValues = new long[previousValues]; + } + + private void initLeadingRepresentation() { + try { + int num = in.readInt(5); + if (num == 0) { + num = 32; + } + leadingBitsPerValue = PostOfficeSolver.positionLength2Bits[num]; + leadingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + leadingRepresentation[i] = in.readInt(6); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + public InputBitStream getInputStream() { + return in; + } + + @Override + public void setBytes(byte[] bs) { + in = new InputBitStream(bs); + } + + @Override + public void refresh() { + storedLeadingZeros = Integer.MAX_VALUE; + storedVal = 0; + current = 0; + first = true; + endOfStream = false; + Arrays.fill(storedValues, 0); + Arrays.fill(leadingRepresentation, 0); + leadingBitsPerValue = 0; + } + + /** + * Returns the next pair in the time series, if available. + * + * @return Pair if there's next value, null if series is done. + */ + public Double readValue() { + try { + next(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + if (endOfStream) { + return null; + } + return Double.longBitsToDouble(storedVal); + } + + private void next() throws IOException { + if (first) { + initLeadingRepresentation(); + first = false; + storedVal = in.readLong(64); + storedValues[current] = storedVal; + if (storedValues[current] == END_SIGN) { + endOfStream = true; + } + } else { + nextValue(); + } + } + + private void nextValue() throws IOException { + // Read value + int flag = in.readInt(2); + long value; + switch (flag) { + case 3: + storedLeadingZeros = leadingRepresentation[in.readInt(leadingBitsPerValue)]; + value = in.readLong(64 - storedLeadingZeros); + value = storedVal ^ value; + + if (value == END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + current = (current + 1) % previousValues; + storedValues[current] = storedVal; + } + break; + case 2: + value = in.readLong(64 - storedLeadingZeros); + value = storedVal ^ value; + if (value == END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + current = (current + 1) % previousValues; + storedValues[current] = storedVal; + } + break; + case 1: + int fill = this.initialFill + leadingBitsPerValue; + int temp = in.readInt(fill); + int index = temp >>> (fill -= previousValuesLog2) & (1 << previousValuesLog2) - 1; + storedLeadingZeros = leadingRepresentation[temp >>> (fill -= leadingBitsPerValue) & (1 << leadingBitsPerValue) - 1]; + int significantBits = temp >>> (fill - 6) & (1 << 6) - 1; + storedVal = storedValues[index]; + if (significantBits == 0) { + significantBits = 64; + } + int storedTrailingZeros = 64 - significantBits - storedLeadingZeros; + value = in.readLong(64 - storedLeadingZeros - storedTrailingZeros); + value <<= storedTrailingZeros; + value = storedVal ^ value; + if (value == END_SIGN) { + endOfStream = true; + return; + } else { + storedVal = value; + current = (current + 1) % previousValues; + storedValues[current] = storedVal; + } + break; + default: + // else -> same value as before + storedVal = storedValues[(int) in.readLong(previousValuesLog2)]; + current = (current + 1) % previousValues; + storedValues[current] = storedVal; + break; + } + } + +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNXORDecompressor.java index 2cee720..77faec8 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpNXORDecompressor.java @@ -3,7 +3,6 @@ import org.urbcomp.startdb.selfstar.utils.InputBitStream; import java.io.IOException; -import java.util.Arrays; public class ChimpNXORDecompressor implements IXORDecompressor { @@ -44,12 +43,9 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedVal = 0; current = 0; first = true; endOfStream = false; - Arrays.fill(storedValues, 0); } /** diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpXORDecompressor.java index 50d6e2f..09e1b76 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ChimpXORDecompressor.java @@ -36,8 +36,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedVal = 0; first = true; endOfStream = false; } @@ -63,10 +61,7 @@ private void next() throws IOException { if (first) { first = false; storedVal = in.readLong(64); - if (storedVal == Elf64Utils.END_SIGN) { - endOfStream = true; - } - + endOfStream = storedVal == Elf64Utils.END_SIGN; } else { nextValue(); } @@ -84,22 +79,14 @@ private void nextValue() throws IOException { storedLeadingZeros = leadingRepresentation[in.readInt(3)]; value = in.readLong(64 - storedLeadingZeros); value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 2: value = in.readLong(64 - storedLeadingZeros); value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 1: storedLeadingZeros = leadingRepresentation[in.readInt(3)]; @@ -111,12 +98,8 @@ private void nextValue() throws IOException { value = in.readLong(64 - storedLeadingZeros - storedTrailingZeros); value <<= storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; default: } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfPlusXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfPlusXORDecompressor.java index b8a4335..dc7f3f1 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfPlusXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfPlusXORDecompressor.java @@ -34,10 +34,6 @@ public InputBitStream getInputStream() { @Override public void refresh() { - in = new InputBitStream(new byte[0]); - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; } @@ -65,9 +61,7 @@ private void next() throws IOException { first = false; int trailingZeros = in.readInt(7); storedVal = in.readLong(64 - trailingZeros) << trailingZeros; - if (storedVal == Elf64Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf64Utils.END_SIGN; } else { nextValue(); } @@ -89,11 +83,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 2: // case 10 @@ -106,11 +97,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 1: // case 01, we do nothing, the same value as before @@ -120,11 +108,8 @@ private void nextValue() throws IOException { centerBits = 64 - storedLeadingZeros - storedTrailingZeros; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressor.java index 1ef45b0..3b77177 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressor.java @@ -5,7 +5,6 @@ import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; import java.io.IOException; -import java.util.Arrays; public class ElfStarXORDecompressor implements IXORDecompressor { private long storedVal = 0; @@ -71,16 +70,8 @@ public InputBitStream getInputStream() { @Override public void refresh() { - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; - - Arrays.fill(trailingRepresentation, 0); - trailingBitsPerValue = 0; - Arrays.fill(leadingRepresentation, 0); - leadingBitsPerValue = 0; } /** @@ -112,9 +103,7 @@ private void next() throws IOException { } else { storedVal = 0; } - if (storedVal == Elf64Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf64Utils.END_SIGN; } else { nextValue(); } @@ -129,11 +118,8 @@ private void nextValue() throws IOException { centerBits = 64 - storedLeadingZeros - storedTrailingZeros; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; } else if (in.readInt(1) == 0) { // case 00 int leadAndTrail = in.readInt(leadingBitsPerValue + trailingBitsPerValue); @@ -145,11 +131,8 @@ private void nextValue() throws IOException { value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; } } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressorAdaLead.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressorAdaLead.java index 2f87ea2..bbed971 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressorAdaLead.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfStarXORDecompressorAdaLead.java @@ -5,7 +5,6 @@ import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; import java.io.IOException; -import java.util.Arrays; public class ElfStarXORDecompressorAdaLead implements IXORDecompressor { private long storedVal = 0; @@ -51,15 +50,8 @@ public InputBitStream getInputStream() { @Override public void refresh() { - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; - - Arrays.fill(leadingRepresentation, 0); - - leadingBitsPerValue = 0; } /** @@ -86,9 +78,7 @@ private void next() throws IOException { first = false; int trailingZeros = in.readInt(7); storedVal = in.readLong(64 - trailingZeros) << trailingZeros; - if (storedVal == Elf64Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf64Utils.END_SIGN; } else { nextValue(); } @@ -110,11 +100,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 2: // case 10 @@ -127,11 +114,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 1: // case 01, we do nothing, the same value as before @@ -141,11 +125,8 @@ private void nextValue() throws IOException { centerBits = 64 - storedLeadingZeros - storedTrailingZeros; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfXORDecompressor.java index cfa7591..1dc3bc7 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/ElfXORDecompressor.java @@ -45,10 +45,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - in = new InputBitStream(new byte[0]); - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; } @@ -75,9 +71,7 @@ private void next() throws IOException { first = false; int trailingZeros = in.readInt(7); storedVal = in.readLong(64 - trailingZeros) << trailingZeros; - if (storedVal == Elf64Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf64Utils.END_SIGN; } else { nextValue(); } @@ -99,11 +93,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 2: // case 10 @@ -116,11 +107,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 64 - storedLeadingZeros - centerBits; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; case 1: // case 01, we do nothing, the same value as before @@ -130,11 +118,8 @@ private void nextValue() throws IOException { centerBits = 64 - storedLeadingZeros - storedTrailingZeros; value = in.readLong(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf64Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; break; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/GorillaXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/GorillaXORDecompressor.java index 574f02f..367dd46 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/GorillaXORDecompressor.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/GorillaXORDecompressor.java @@ -47,9 +47,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = 0; - storedVal = 0; first = true; endOfStream = false; } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/SElfStarXORDecompressor.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/SElfStarXORDecompressor.java new file mode 100644 index 0000000..eb23aa3 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor/xor/SElfStarXORDecompressor.java @@ -0,0 +1,140 @@ +package org.urbcomp.startdb.selfstar.decompressor.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf64Utils; +import org.urbcomp.startdb.selfstar.utils.InputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver; + +import java.io.IOException; + +public class SElfStarXORDecompressor implements IXORDecompressor { + private long storedVal = 0; + private int storedLeadingZeros = Integer.MAX_VALUE; + private int storedTrailingZeros = Integer.MAX_VALUE; + private boolean first = true; + private boolean endOfStream = false; + + private InputBitStream in; + + private int[] leadingRepresentation = {0, 8, 12, 16, 18, 20, 22, 24}; + + private int[] trailingRepresentation = {0, 22, 28, 32, 36, 40, 42, 46}; + + private int leadingBitsPerValue = 3; + + private int trailingBitsPerValue = 3; + + public SElfStarXORDecompressor() { + } + + private void initLeadingRepresentation() { + try { + int num = in.readInt(5); + if (num == 0) { + num = 32; + } + leadingBitsPerValue = PostOfficeSolver.positionLength2Bits[num]; + leadingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + leadingRepresentation[i] = in.readInt(6); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + private void initTrailingRepresentation() { + try { + int num = in.readInt(5); + if (num == 0) { + num = 32; + } + trailingBitsPerValue = PostOfficeSolver.positionLength2Bits[num]; + trailingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + trailingRepresentation[i] = in.readInt(6); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + @Override + public void setBytes(byte[] bs) { + in = new InputBitStream(bs); + } + + @Override + public InputBitStream getInputStream() { + return in; + } + + @Override + public void refresh() { + first = true; + endOfStream = false; + } + + /** + * Returns the next pair in the time series, if available. + * + * @return Pair if there's next value, null if series is done. + */ + @Override + public Double readValue() { + try { + next(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + if (endOfStream) { + return null; + } + return Double.longBitsToDouble(storedVal); + } + + private void next() throws IOException { + if (first) { + if (in.readBit() == 1) { + initLeadingRepresentation(); + initTrailingRepresentation(); + } + first = false; + int trailingZeros = in.readInt(7); + if (trailingZeros < 64) { + storedVal = ((in.readLong(63 - trailingZeros) << 1) + 1) << trailingZeros; + } else { + storedVal = 0; + } + endOfStream = storedVal == Elf64Utils.END_SIGN; + } else { + nextValue(); + } + } + + private void nextValue() throws IOException { + long value; + int centerBits; + + if (in.readInt(1) == 1) { + // case 1 + centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + value = in.readLong(centerBits) << storedTrailingZeros; + value = storedVal ^ value; + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; + } else if (in.readInt(1) == 0) { + // case 00 + int leadAndTrail = in.readInt(leadingBitsPerValue + trailingBitsPerValue); + int lead = leadAndTrail >>> trailingBitsPerValue; + int trail = ~(0xffffffff << trailingBitsPerValue) & leadAndTrail; + storedLeadingZeros = leadingRepresentation[lead]; + storedTrailingZeros = trailingRepresentation[trail]; + centerBits = 64 - storedLeadingZeros - storedTrailingZeros; + + value = in.readLong(centerBits) << storedTrailingZeros; + value = storedVal ^ value; + endOfStream = value == Elf64Utils.END_SIGN; + storedVal = value; + } + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/ElfPlusDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/ElfPlusDecompressor32.java index 4f56286..6f92fdc 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/ElfPlusDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/ElfPlusDecompressor32.java @@ -54,7 +54,7 @@ public Float nextValue() { private Float recoverVByBetaStar() { float v; Float vPrime = xorDecompressor.readValue(); - int sp = Elf32Utils.getSP(Math.abs(vPrime)); + int sp = Elf32Utils.getSP(vPrime < 0 ? -vPrime : vPrime); if (lastBetaStar == 0) { v = Elf32Utils.get10iN(-sp - 1); if (vPrime < 0) { diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpNXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpNXORDecompressor32.java index 5871907..a253bc1 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpNXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpNXORDecompressor32.java @@ -4,7 +4,6 @@ import org.urbcomp.startdb.selfstar.utils.InputBitStream; import java.io.IOException; -import java.util.Arrays; public class ChimpNXORDecompressor32 implements IXORDecompressor32 { @@ -44,12 +43,9 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedVal = 0; current = 0; first = true; endOfStream = false; - Arrays.fill(storedValues, 0); } /** diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpXORDecompressor32.java index fbc3cec..599c977 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ChimpXORDecompressor32.java @@ -36,8 +36,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedVal = 0; first = true; endOfStream = false; } @@ -63,10 +61,7 @@ private void next() throws IOException { if (first) { first = false; storedVal = in.readInt(32); - if (storedVal == Elf32Utils.END_SIGN) { - endOfStream = true; - } - + endOfStream = storedVal == Elf32Utils.END_SIGN; } else { nextValue(); } @@ -84,22 +79,16 @@ private void nextValue() throws IOException { storedLeadingZeros = leadingRepresentation[in.readInt(3)]; value = in.readInt(32 - storedLeadingZeros); value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 2: value = in.readInt(32 - storedLeadingZeros); value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 1: storedLeadingZeros = leadingRepresentation[in.readInt(3)]; @@ -111,12 +100,9 @@ private void nextValue() throws IOException { value = in.readInt(32 - storedLeadingZeros - storedTrailingZeros); value <<= storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - return; - } else { - storedVal = value; - } + + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; default: } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfPlusXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfPlusXORDecompressor32.java index 83b39c1..b49c5e8 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfPlusXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfPlusXORDecompressor32.java @@ -34,10 +34,6 @@ public InputBitStream getInputStream() { @Override public void refresh() { - in = new InputBitStream(new byte[0]); - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; } @@ -65,9 +61,7 @@ private void next() throws IOException { first = false; int trailingZeros = in.readInt(6); storedVal = in.readInt(32 - trailingZeros) << trailingZeros; - if (storedVal == Elf32Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf32Utils.END_SIGN; } else { nextValue(); } @@ -89,11 +83,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 32 - storedLeadingZeros - centerBits; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 2: // case 10 @@ -106,11 +97,9 @@ private void nextValue() throws IOException { storedTrailingZeros = 32 - storedLeadingZeros - centerBits; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 1: // case 01, we do nothing, the same value as before @@ -120,11 +109,8 @@ private void nextValue() throws IOException { centerBits = 32 - storedLeadingZeros - storedTrailingZeros; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfStarXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfStarXORDecompressor32.java index 3ed54b0..39a3af4 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfStarXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfStarXORDecompressor32.java @@ -5,7 +5,6 @@ import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver32; import java.io.IOException; -import java.util.Arrays; public class ElfStarXORDecompressor32 implements IXORDecompressor32 { private int storedVal = 0; @@ -71,16 +70,8 @@ public InputBitStream getInputStream() { @Override public void refresh() { - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; - - Arrays.fill(trailingRepresentation, 0); - trailingBitsPerValue = 0; - Arrays.fill(leadingRepresentation, 0); - leadingBitsPerValue = 0; } /** @@ -112,9 +103,7 @@ private void next() throws IOException { } else { storedVal = 0; } - if (storedVal == Elf32Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf32Utils.END_SIGN; } else { nextValue(); } @@ -129,11 +118,8 @@ private void nextValue() throws IOException { centerBits = 32 - storedLeadingZeros - storedTrailingZeros; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; } else if (in.readInt(1) == 0) { // case 00 int leadAndTrail = in.readInt(leadingBitsPerValue + trailingBitsPerValue); @@ -145,11 +131,8 @@ private void nextValue() throws IOException { value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; } } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfXORDecompressor32.java index b5fc4d7..767da2d 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/ElfXORDecompressor32.java @@ -45,10 +45,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - in = new InputBitStream(new byte[0]); - storedVal = 0; - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = Integer.MAX_VALUE; first = true; endOfStream = false; } @@ -75,9 +71,7 @@ private void next() throws IOException { first = false; int trailingZeros = in.readInt(6); storedVal = in.readInt(32 - trailingZeros) << trailingZeros; - if (storedVal == Elf32Utils.END_SIGN) { - endOfStream = true; - } + endOfStream = storedVal == Elf32Utils.END_SIGN; } else { nextValue(); } @@ -99,11 +93,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 32 - storedLeadingZeros - centerBits; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 2: // case 10 @@ -116,11 +107,8 @@ private void nextValue() throws IOException { storedTrailingZeros = 32 - storedLeadingZeros - centerBits; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; case 1: // case 01, we do nothing, the same value as before @@ -130,11 +118,8 @@ private void nextValue() throws IOException { centerBits = 32 - storedLeadingZeros - storedTrailingZeros; value = in.readInt(centerBits) << storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; break; } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/GorillaXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/GorillaXORDecompressor32.java index a5d86e2..4cec35d 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/GorillaXORDecompressor32.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/GorillaXORDecompressor32.java @@ -47,9 +47,6 @@ public void setBytes(byte[] bs) { @Override public void refresh() { - storedLeadingZeros = Integer.MAX_VALUE; - storedTrailingZeros = 0; - storedVal = 0; first = true; endOfStream = false; } @@ -59,10 +56,7 @@ private void next() throws IOException { if (first) { first = false; storedVal = in.readInt(32); - if (storedVal == Elf32Utils.END_SIGN) { - endOfStream = true; - } - + endOfStream = storedVal == Elf32Utils.END_SIGN; } else { nextValue(); } @@ -85,11 +79,8 @@ private void nextValue() throws IOException { int value = in.readInt(32 - storedLeadingZeros - storedTrailingZeros); value <<= storedTrailingZeros; value = storedVal ^ value; - if (value == Elf32Utils.END_SIGN) { - endOfStream = true; - } else { - storedVal = value; - } + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; } } } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/SElfStarXORDecompressor32.java b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/SElfStarXORDecompressor32.java new file mode 100644 index 0000000..0192243 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/decompressor32/xor/SElfStarXORDecompressor32.java @@ -0,0 +1,140 @@ +package org.urbcomp.startdb.selfstar.decompressor32.xor; + +import org.urbcomp.startdb.selfstar.utils.Elf32Utils; +import org.urbcomp.startdb.selfstar.utils.InputBitStream; +import org.urbcomp.startdb.selfstar.utils.PostOfficeSolver32; + +import java.io.IOException; + +public class SElfStarXORDecompressor32 implements IXORDecompressor32 { + private int storedVal = 0; + private int storedLeadingZeros = Integer.MAX_VALUE; + private int storedTrailingZeros = Integer.MAX_VALUE; + private boolean first = true; + private boolean endOfStream = false; + + private InputBitStream in; + + private int[] leadingRepresentation = {0, 8, 12, 16}; + + private int[] trailingRepresentation = {0, 16}; + + private int leadingBitsPerValue = 2; + + private int trailingBitsPerValue = 1; + + public SElfStarXORDecompressor32() { + } + + private void initLeadingRepresentation() { + try { + int num = in.readInt(4); + if (num == 0) { + num = 16; + } + leadingBitsPerValue = PostOfficeSolver32.positionLength2Bits[num]; + leadingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + leadingRepresentation[i] = in.readInt(5); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + private void initTrailingRepresentation() { + try { + int num = in.readInt(4); + if (num == 0) { + num = 16; + } + trailingBitsPerValue = PostOfficeSolver32.positionLength2Bits[num]; + trailingRepresentation = new int[num]; + for (int i = 0; i < num; i++) { + trailingRepresentation[i] = in.readInt(5); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + @Override + public void setBytes(byte[] bs) { + in = new InputBitStream(bs); + } + + @Override + public InputBitStream getInputStream() { + return in; + } + + @Override + public void refresh() { + first = true; + endOfStream = false; + } + + /** + * Returns the next pair in the time series, if available. + * + * @return Pair if there's next value, null if series is done. + */ + @Override + public Float readValue() { + try { + next(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + if (endOfStream) { + return null; + } + return Float.intBitsToFloat(storedVal); + } + + private void next() throws IOException { + if (first) { + if (in.readBit() == 1) { + initLeadingRepresentation(); + initTrailingRepresentation(); + } + first = false; + int trailingZeros = in.readInt(6); + if (trailingZeros < 32) { + storedVal = ((in.readInt(31 - trailingZeros) << 1) + 1) << trailingZeros; + } else { + storedVal = 0; + } + endOfStream = storedVal == Elf32Utils.END_SIGN; + } else { + nextValue(); + } + } + + private void nextValue() throws IOException { + int value; + int centerBits; + + if (in.readInt(1) == 1) { + // case 1 + centerBits = 32 - storedLeadingZeros - storedTrailingZeros; + value = in.readInt(centerBits) << storedTrailingZeros; + value = storedVal ^ value; + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; + } else if (in.readInt(1) == 0) { + // case 00 + int leadAndTrail = in.readInt(leadingBitsPerValue + trailingBitsPerValue); + int lead = leadAndTrail >>> trailingBitsPerValue; + int trail = ~(0xffff << trailingBitsPerValue) & leadAndTrail; + storedLeadingZeros = leadingRepresentation[lead]; + storedTrailingZeros = trailingRepresentation[trail]; + centerBits = 32 - storedLeadingZeros - storedTrailingZeros; + + value = in.readInt(centerBits) << storedTrailingZeros; + value = storedVal ^ value; + endOfStream = value == Elf32Utils.END_SIGN; + storedVal = value; + } + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Code.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Code.java new file mode 100644 index 0000000..eabdd4a --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Code.java @@ -0,0 +1,10 @@ +package org.urbcomp.startdb.selfstar.utils.Huffman; + +public class Code { + public final long value; + public final int length; + public Code(long value, int length) { + this.value = value; + this.length = length; + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/HuffmanEncode.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/HuffmanEncode.java index 5f2c0ac..21d08dc 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/HuffmanEncode.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/HuffmanEncode.java @@ -1,77 +1,71 @@ package org.urbcomp.startdb.selfstar.utils.Huffman; -import javafx.util.Pair; import org.urbcomp.startdb.selfstar.utils.OutputBitStream; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; import java.util.PriorityQueue; public class HuffmanEncode { // Map value -> - private static final HashMap> huffmanCodes = new HashMap<>(); - private final int[] values; + private final Code[] huffmanCodes = new Code[18]; - public HuffmanEncode(int[] values, int[] frequencies) { - this.values = values; - buildHuffmanTreeAndConToHashMap(values, frequencies); + public HuffmanEncode(int[] frequencies) { + buildHuffmanTreeAndConToHashMap(frequencies); } - public HashMap> getHuffmanCodes(){ + public Code[] getHuffmanCodes(){ return huffmanCodes; } - public static void buildHuffmanTreeAndConToHashMap(int[] values, int[] frequencies) { - PriorityQueue nodePriorityQueue = new PriorityQueue<>(); + private void buildHuffmanTreeAndConToHashMap(int[] frequencies) { + PriorityQueue nodePriorityQueue = new PriorityQueue<>(huffmanCodes.length); // Construct priorityQueue - for (int i = 0; i < values.length; i++) { - nodePriorityQueue.add(new Node(values[i], frequencies[i])); + for (int i = 0; i < frequencies.length; i++) { + nodePriorityQueue.add(new Node(i, frequencies[i])); } // Construct huffman tree while (nodePriorityQueue.size() > 1) { Node left = nodePriorityQueue.poll(); Node right = nodePriorityQueue.poll(); - Node newNode = new Node(-Integer.MAX_VALUE, left.frequency + Objects.requireNonNull(right).frequency); - newNode.left = left; - newNode.right = right; + @SuppressWarnings("all") + Node newNode = new Node(-Integer.MAX_VALUE, left.frequency + right.frequency); + newNode.children[0] = left; + newNode.children[1] = right; nodePriorityQueue.add(newNode); } generateHuffmanCodes(nodePriorityQueue.peek(), 0, 0); } - private static void generateHuffmanCodes(Node root, long code, int length) { + private void generateHuffmanCodes(Node root, long code, int length) { if (root != null) { if (root.data != -Integer.MAX_VALUE) { - huffmanCodes.put(root.data, new Pair<>(code, length)); + huffmanCodes[root.data] = new Code(code, length); } - generateHuffmanCodes(root.left, code << 1, length + 1); - generateHuffmanCodes(root.right, (code << 1) | 1, length + 1); + generateHuffmanCodes(root.children[0], code << 1, length + 1); + generateHuffmanCodes(root.children[1], (code << 1) | 1, length + 1); } } - public static Node hashMapToTree(HashMap> huffmanCodes) { + public Node hashMapToTree(Code[] huffmanCodes) { Node root = new Node(-Integer.MAX_VALUE, 0); Node curNode = root; - for (Map.Entry> valueToCodeAndLen : huffmanCodes.entrySet()) { - int value = valueToCodeAndLen.getKey(); - long code = valueToCodeAndLen.getValue().getKey(); - int length = valueToCodeAndLen.getValue().getValue(); + for (int value = 0; value < huffmanCodes.length; value++) { + long code = huffmanCodes[value].value; + int length = huffmanCodes[value].length; long signal; while (length != 0) { signal = (code >> (length - 1)) & 1; if (signal == 0) { - if (curNode.left == null) { - curNode.left = new Node(-Integer.MAX_VALUE, 0); + if (curNode.children[0] == null) { + curNode.children[0] = new Node(-Integer.MAX_VALUE, 0); } - curNode = curNode.left; + curNode = curNode.children[0]; } else { - if (curNode.right == null) { - curNode.right = new Node(-Integer.MAX_VALUE, 0); + if (curNode.children[1] == null) { + curNode.children[1] = new Node(-Integer.MAX_VALUE, 0); } - curNode = curNode.right; + curNode = curNode.children[1]; } length -= 1; } @@ -83,9 +77,9 @@ public static Node hashMapToTree(HashMap> huffmanCo public int writeHuffmanCodes(OutputBitStream out) { int thisSize = 0; - for (int value : values) { - thisSize += out.writeInt(huffmanCodes.get(value).getValue(), 5); - thisSize += out.writeLong(huffmanCodes.get(value).getKey(), huffmanCodes.get(value).getValue()); + for (Code huffmanCode : huffmanCodes) { + thisSize += out.writeInt(huffmanCode.length, 5); + thisSize += out.writeLong(huffmanCode.value, huffmanCode.length); } return thisSize; } diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Node.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Node.java index b6016d1..a29f76e 100644 --- a/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Node.java +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/Huffman/Node.java @@ -9,7 +9,7 @@ public class Node implements Comparable { public int row; public int depth; public long code; - public Node left, right; + public Node[] children = new Node[2]; public Node(int data, int frequency) { this.data = data; @@ -45,6 +45,7 @@ public int compareTo(Node o) { return this.frequency - o.frequency; } } + class CustomComparator implements Comparator { @Override diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRPruning.java new file mode 100644 index 0000000..8b190e2 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRPruning.java @@ -0,0 +1,199 @@ +package org.urbcomp.startdb.selfstar.utils; + +// without first-rear pruning +public class PostOfficeSolverNoFRPruning { + // 2^index + public static final int[] pow2z = {1, 2, 4, 8, 16, 32}; + + // (int) Math.ceil(Math.log(index) / Math.log(2)) + public static final int[] positionLength2Bits = { + 0, 0, 1, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6 + }; + + /** + * @param distribution distribution of leading or trailing zeros + * @param representation out param, representation of round + * @param round out param, round + * @return positions + */ + public static int[] initRoundAndRepresentation(int[] distribution, int[] representation, int[] round) { + int[] totalCountAndNonZerosCount = calTotalCountAndNonZerosCounts(distribution); + + int maxZ = Math.min(positionLength2Bits[totalCountAndNonZerosCount[1]], 5); // 最多用5个bit来表示 + + int totalCost = Integer.MAX_VALUE; + int[] positions = {}; + + for (int z = 0; z <= maxZ; z++) { + int presentCost = totalCountAndNonZerosCount[0] * z; + if (presentCost >= totalCost) { + break; + } + int num = PostOfficeSolverNoFRPruning.pow2z[z]; // 邮局的总数量 + PostOfficeResult por = PostOfficeSolverNoFRPruning.buildPostOffice( + distribution, num, totalCountAndNonZerosCount[1]); + int tempTotalCost = por.getAppCost() + presentCost; + if (tempTotalCost < totalCost) { + totalCost = tempTotalCost; + positions = por.getOfficePositions(); + } + } + + representation[0] = 0; + round[0] = 0; + int i = 1; + for (int j = 1; j < distribution.length; j++) { + if (i < positions.length && j == positions[i]) { + representation[j] = representation[j - 1] + 1; + round[j] = j; + i++; + } else { + representation[j] = representation[j - 1]; + round[j] = round[j - 1]; + } + } + + return positions; + } + + public static int writePositions(int[] positions, OutputBitStream out) { + int thisSize = out.writeInt(positions.length, 5); + for (int p : positions) { + thisSize += out.writeInt(p, 6); + } + return thisSize; + } + + private static int[] calTotalCountAndNonZerosCounts(int[] arr) { + int nonZerosCount = arr.length; + int totalCount = arr[0]; + for (int i = 1; i < arr.length; i++) { + totalCount += arr[i]; + if (arr[i] == 0) { + nonZerosCount--; + } + } + return new int[]{totalCount, nonZerosCount}; + } + + private static PostOfficeResult buildPostOffice(int[] arr, int num, int nonZerosCount) { + int originalNum = num; + num = Math.min(num, nonZerosCount); + + int[][] dp = new int[arr.length][num]; // 状态矩阵。d[i][j]表示，只考虑前i个居民点，且第i个位置是第j个邮局的总距离，i >= j， + // 下标从0开始。注意，并非是所有居民点的总距离，因为没有考虑第j个邮局之后的居民点的距离 + int[][] pre = new int[arr.length][num]; // 对应于dp[i][j]，表示让dp[i][j]最小时，第j-1个邮局所在的位置信息 + + dp[0][0] = 0; // 第0个位置是第0个邮局，此时状态为0 + pre[0][0] = -1; // 让dp[0][0]最小时，第-1个邮局所在的位置信息为-1 + + for (int i = 1; i < arr.length; i++) { + if (arr[i] == 0) { + continue; + } + for (int j = Math.max(1, num + i - arr.length); j <= i && j < num; j++) { + // arr.length - i < num - j，表示i后面的居民数（arr.length - i）不足以构建剩下的num - j个邮局 + if (i > 1 && j == 1) { + dp[i][j] = 0; + for (int k = 1; k < i; k++) { + dp[i][j] += arr[k] * k; + } + pre[i][j] = 0; + } else { + int appCost = Integer.MAX_VALUE; + int preK = 0; + for (int k = j - 1; k <= i - 1; k++) { + if (arr[k] == 0 && k > 0) { + continue; + } + int sum = dp[k][j - 1]; + for (int p = k + 1; p <= i - 1; p++) { + sum += arr[p] * (p - k); + } + if (appCost > sum) { + appCost = sum; + preK = k; + if (sum == 0) { // 找到其中一个0，提前终止 + break; + } + } + } + if (appCost != Integer.MAX_VALUE) { + dp[i][j] = appCost; + pre[i][j] = preK; + } + } + } + } + int tempTotalAppCost = Integer.MAX_VALUE; + int tempBestLast = Integer.MAX_VALUE; + for (int i = num - 1; i < arr.length; i++) { + if (num - 1 == 0 && i > 0) { + break; + } + if (arr[i] == 0 && i > 0) { + continue; + } + int sum = dp[i][num - 1]; + for (int j = i + 1; j < arr.length; j++) { + sum += arr[j] * (j - i); + } + if (tempTotalAppCost > sum) { + tempTotalAppCost = sum; + tempBestLast = i; + } + } + + int[] officePositions = new int[num]; + int i = 1; + + while (tempBestLast != -1) { + officePositions[num - i] = tempBestLast; + tempBestLast = pre[tempBestLast][num - i]; + i++; + } + + if (originalNum > nonZerosCount) { + int[] modifyingOfficePositions = new int[originalNum]; + int j = 0, k = 0; + while (j < originalNum && k < num) { + if (j - k < originalNum - num && j < officePositions[k]) { + modifyingOfficePositions[j] = j; + j++; + } else { + modifyingOfficePositions[j] = officePositions[k]; + j++; + k++; + } + } + officePositions = modifyingOfficePositions; + } + + return new PostOfficeResult(officePositions, tempTotalAppCost); + } + + public static class PostOfficeResult { + int[] officePositions; + int totalAppCost; + + public PostOfficeResult(int[] officePositions, int totalCost) { + this.officePositions = officePositions; + this.totalAppCost = totalCost; + } + + public int[] getOfficePositions() { + return officePositions; + } + + public int getAppCost() { + return totalAppCost; + } + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZGPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZGPruning.java new file mode 100644 index 0000000..a222516 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZGPruning.java @@ -0,0 +1,156 @@ +package org.urbcomp.startdb.selfstar.utils; + +public class PostOfficeSolverNoFRZGPruning { + // (int) Math.ceil(Math.log(index) / Math.log(2)) + public static final int[] positionLength2Bits = { + 0, 0, 1, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6 + }; + + /** + * @param distribution distribution of leading or trailing zeros + * @param representation out param, representation of round + * @param round out param, round + * @return positions + */ + public static int[] initRoundAndRepresentation(int[] distribution, int[] representation, int[] round) { + int totalCount = calTotalCount(distribution); + int totalCost = Integer.MAX_VALUE; + int[] positions = {}; + + for (int z = 1; z < 64; z++) { + int presentCost = totalCount * positionLength2Bits[z]; + PostOfficeResult por = PostOfficeSolverNoFRZGPruning.buildPostOffice( + distribution, z); + int tempTotalCost = por.getAppCost() + presentCost; + if (tempTotalCost < totalCost) { + totalCost = tempTotalCost; + positions = por.getOfficePositions(); + } + } + + representation[0] = 0; + round[0] = 0; + int i = 1; + for (int j = 1; j < distribution.length; j++) { + if (i < positions.length && j == positions[i]) { + representation[j] = representation[j - 1] + 1; + round[j] = j; + i++; + } else { + representation[j] = representation[j - 1]; + round[j] = round[j - 1]; + } + } + + return positions; + } + + public static int writePositions(int[] positions, OutputBitStream out) { + int thisSize = out.writeInt(positions.length, 5); + for (int p : positions) { + thisSize += out.writeInt(p, 6); + } + return thisSize; + } + + private static int calTotalCount(int[] arr) { + int totalCount = arr[0]; + for (int i = 1; i < arr.length; i++) { + totalCount += arr[i]; + } + return totalCount; + } + + private static PostOfficeResult buildPostOffice(int[] arr, int num) { + int[][] dp = new int[arr.length][num]; // 状态矩阵。d[i][j]表示，只考虑前i个居民点，且第i个位置是第j个邮局的总距离，i >= j， + // 下标从0开始。注意，并非是所有居民点的总距离，因为没有考虑第j个邮局之后的居民点的距离 + int[][] pre = new int[arr.length][num]; // 对应于dp[i][j]，表示让dp[i][j]最小时，第j-1个邮局所在的位置信息 + + dp[0][0] = 0; // 第0个位置是第0个邮局，此时状态为0 + pre[0][0] = -1; // 让dp[0][0]最小时，第-1个邮局所在的位置信息为-1 + + for (int i = 1; i < arr.length; i++) { + for (int j = Math.max(1, num + i - arr.length); j <= i && j < num; j++) { + // arr.length - i < num - j，表示i后面的居民数（arr.length - i）不足以构建剩下的num - j个邮局 + if (i > 1 && j == 1) { + dp[i][j] = 0; + for (int k = 1; k < i; k++) { + dp[i][j] += arr[k] * k; + } + pre[i][j] = 0; + } else { + int appCost = Integer.MAX_VALUE; + int preK = 0; + for (int k = j - 1; k <= i - 1; k++) { + int sum = dp[k][j - 1]; + for (int p = k + 1; p <= i - 1; p++) { + sum += arr[p] * (p - k); + } + if (appCost > sum) { + appCost = sum; + preK = k; + if (sum == 0) { // 找到其中一个0，提前终止 + break; + } + } + } + if (appCost != Integer.MAX_VALUE) { + dp[i][j] = appCost; + pre[i][j] = preK; + } + } + } + } + int tempTotalAppCost = Integer.MAX_VALUE; + int tempBestLast = Integer.MAX_VALUE; + for (int i = num - 1; i < arr.length; i++) { + if (num - 1 == 0 && i > 0) { + break; + } + int sum = dp[i][num - 1]; + for (int j = i + 1; j < arr.length; j++) { + sum += arr[j] * (j - i); + } + if (tempTotalAppCost > sum) { + tempTotalAppCost = sum; + tempBestLast = i; + } + } + + int[] officePositions = new int[num]; + int i = 1; + + while (tempBestLast != -1) { + officePositions[num - i] = tempBestLast; + tempBestLast = pre[tempBestLast][num - i]; + i++; + } + + return new PostOfficeResult(officePositions, tempTotalAppCost); + } + + public static class PostOfficeResult { + int[] officePositions; + int totalAppCost; + + public PostOfficeResult(int[] officePositions, int totalCost) { + this.officePositions = officePositions; + this.totalAppCost = totalCost; + } + + public int[] getOfficePositions() { + return officePositions; + } + + public int getAppCost() { + return totalAppCost; + } + } +} diff --git a/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZPruning.java b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZPruning.java new file mode 100644 index 0000000..d3e8d93 --- /dev/null +++ b/src/main/java/org/urbcomp/startdb/selfstar/utils/PostOfficeSolverNoFRZPruning.java @@ -0,0 +1,171 @@ +package org.urbcomp.startdb.selfstar.utils; + +// without First-Rear Pruning and Zero Pruning +public class PostOfficeSolverNoFRZPruning { + // 2^index + public static final int[] pow2z = {1, 2, 4, 8, 16, 32}; + + // (int) Math.ceil(Math.log(index) / Math.log(2)) + public static final int[] positionLength2Bits = { + 0, 0, 1, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6 + }; + + /** + * @param distribution distribution of leading or trailing zeros + * @param representation out param, representation of round + * @param round out param, round + * @return positions + */ + public static int[] initRoundAndRepresentation(int[] distribution, int[] representation, int[] round) { + int[] totalCountAndNonZerosCount = calTotalCountAndNonZerosCounts(distribution); + + int maxZ = Math.min(positionLength2Bits[totalCountAndNonZerosCount[1]], 5); // 最多用5个bit来表示 + + int totalCost = Integer.MAX_VALUE; + int[] positions = {}; + + for (int z = 0; z <= maxZ; z++) { + int presentCost = totalCountAndNonZerosCount[0] * z; + if (presentCost >= totalCost) { + break; + } + int num = PostOfficeSolverNoFRZPruning.pow2z[z]; // 邮局的总数量 + PostOfficeResult por = PostOfficeSolverNoFRZPruning.buildPostOffice( + distribution, num); + int tempTotalCost = por.getAppCost() + presentCost; + if (tempTotalCost < totalCost) { + totalCost = tempTotalCost; + positions = por.getOfficePositions(); + } + } + + representation[0] = 0; + round[0] = 0; + int i = 1; + for (int j = 1; j < distribution.length; j++) { + if (i < positions.length && j == positions[i]) { + representation[j] = representation[j - 1] + 1; + round[j] = j; + i++; + } else { + representation[j] = representation[j - 1]; + round[j] = round[j - 1]; + } + } + + return positions; + } + + public static int writePositions(int[] positions, OutputBitStream out) { + int thisSize = out.writeInt(positions.length, 5); + for (int p : positions) { + thisSize += out.writeInt(p, 6); + } + return thisSize; + } + + private static int[] calTotalCountAndNonZerosCounts(int[] arr) { + int nonZerosCount = arr.length; + int totalCount = arr[0]; + for (int i = 1; i < arr.length; i++) { + totalCount += arr[i]; + if (arr[i] == 0) { + nonZerosCount--; + } + } + return new int[]{totalCount, nonZerosCount}; + } + + private static PostOfficeResult buildPostOffice(int[] arr, int num) { + int[][] dp = new int[arr.length][num]; // 状态矩阵。d[i][j]表示，只考虑前i个居民点，且第i个位置是第j个邮局的总距离，i >= j， + // 下标从0开始。注意，并非是所有居民点的总距离，因为没有考虑第j个邮局之后的居民点的距离 + int[][] pre = new int[arr.length][num]; // 对应于dp[i][j]，表示让dp[i][j]最小时，第j-1个邮局所在的位置信息 + + dp[0][0] = 0; // 第0个位置是第0个邮局，此时状态为0 + pre[0][0] = -1; // 让dp[0][0]最小时，第-1个邮局所在的位置信息为-1 + + for (int i = 1; i < arr.length; i++) { + for (int j = Math.max(1, num + i - arr.length); j <= i && j < num; j++) { + // arr.length - i < num - j，表示i后面的居民数（arr.length - i）不足以构建剩下的num - j个邮局 + if (i > 1 && j == 1) { + dp[i][j] = 0; + for (int k = 1; k < i; k++) { + dp[i][j] += arr[k] * k; + } + pre[i][j] = 0; + } else { + int appCost = Integer.MAX_VALUE; + int preK = 0; + for (int k = j - 1; k <= i - 1; k++) { + int sum = dp[k][j - 1]; + for (int p = k + 1; p <= i - 1; p++) { + sum += arr[p] * (p - k); + } + if (appCost > sum) { + appCost = sum; + preK = k; + if (sum == 0) { // 找到其中一个0，提前终止 + break; + } + } + } + if (appCost != Integer.MAX_VALUE) { + dp[i][j] = appCost; + pre[i][j] = preK; + } + } + } + } + int tempTotalAppCost = Integer.MAX_VALUE; + int tempBestLast = Integer.MAX_VALUE; + for (int i = num - 1; i < arr.length; i++) { + if (num - 1 == 0 && i > 0) { + break; + } + int sum = dp[i][num - 1]; + for (int j = i + 1; j < arr.length; j++) { + sum += arr[j] * (j - i); + } + if (tempTotalAppCost > sum) { + tempTotalAppCost = sum; + tempBestLast = i; + } + } + + int[] officePositions = new int[num]; + int i = 1; + + while (tempBestLast != -1) { + officePositions[num - i] = tempBestLast; + tempBestLast = pre[tempBestLast][num - i]; + i++; + } + + return new PostOfficeResult(officePositions, tempTotalAppCost); + } + + public static class PostOfficeResult { + int[] officePositions; + int totalAppCost; + + public PostOfficeResult(int[] officePositions, int totalCost) { + this.officePositions = officePositions; + this.totalAppCost = totalCost; + } + + public int[] getOfficePositions() { + return officePositions; + } + + public int getAppCost() { + return totalAppCost; + } + } +} diff --git a/src/main/resources/floating/test.csv b/src/main/resources/floating/test.csv new file mode 100644 index 0000000..b5c8fdb --- /dev/null +++ b/src/main/resources/floating/test.csv @@ -0,0 +1,1279 @@ +29.463283 +29.463282 +29.46328 +29.463277 +29.463272 +29.463268 +29.463262 +29.463253 +29.463243 +29.463237 +29.46323 +29.463222 +29.46322 +29.463227 +29.463227 +29.463238 +29.463253 +29.46326 +29.463263 +29.463265 +29.463263 +29.463257 +29.46325 +29.46325 +29.463243 +29.463243 +29.463252 +29.463257 +29.46326 +29.46326 +29.463267 +29.463273 +29.463277 +29.463283 +29.463292 +29.463298 +29.463307 +29.463307 +29.463313 +29.46332 +29.463323 +29.463327 +29.463327 +29.463327 +29.463328 +29.463328 +29.463328 +29.463328 +29.46333 +29.463332 +29.463332 +29.463335 +29.463338 +29.463337 +29.463335 +29.463328 +29.463323 +29.463318 +29.46331 +29.46331 +29.4633 +29.463288 +29.463278 +29.463268 +29.46326 +29.463255 +29.463252 +29.46325 +29.463252 +29.463257 +29.463265 +29.46327 +29.463273 +29.463273 +29.46328 +29.463283 +29.46329 +29.463295 +29.463298 +29.463302 +29.463305 +29.463307 +29.46331 +29.463312 +29.463315 +29.463322 +29.46333 +29.46333 +29.46334 +29.463347 +29.463353 +29.463358 +29.463357 +29.463353 +29.463347 +29.463343 +29.46334 +29.463335 +29.46333 +29.463323 +29.463318 +29.463315 +29.463313 +29.463317 +29.463318 +29.46332 +29.46332 +29.463318 +29.463315 +29.463312 +29.463308 +29.463305 +29.463302 +29.4633 +29.4633 +29.4633 +29.463297 +29.463295 +29.463295 +29.463297 +29.4633 +29.463303 +29.463302 +29.463298 +29.463297 +29.463293 +29.46329 +29.463283 +29.463283 +29.463277 +29.463267 +29.46325 +29.46325 +29.463233 +29.463222 +29.463213 +29.463205 +29.4632 +29.463202 +29.463203 +29.463203 +29.463208 +29.463218 +29.46323 +29.463238 +29.463243 +29.463248 +29.463252 +29.463255 +29.463255 +29.463257 +29.463258 +29.463262 +29.463263 +29.463263 +29.463262 +29.463262 +29.463258 +29.463257 +29.463255 +29.463255 +29.463257 +29.463257 +29.46326 +29.46326 +29.463262 +29.463258 +29.463257 +29.463257 +29.463258 +29.46326 +29.463263 +29.463265 +29.463265 +29.463267 +29.46327 +29.463273 +29.463275 +29.463275 +29.463275 +29.463275 +29.463273 +29.463272 +29.463265 +29.463262 +29.463258 +29.463257 +29.463255 +29.463257 +29.46326 +29.46326 +29.46326 +29.463255 +29.46325 +29.463245 +29.46324 +29.463237 +29.463233 +29.463228 +29.463225 +29.463222 +29.463217 +29.463212 +29.463207 +29.4632 +29.463193 +29.46319 +29.46319 +29.46319 +29.463195 +29.463197 +29.463202 +29.463205 +29.46321 +29.463215 +29.46322 +29.46322 +29.463225 +29.463228 +29.46323 +29.463232 +29.463233 +29.463235 +29.463237 +29.46324 +29.463243 +29.463245 +29.463248 +29.463252 +29.463255 +29.463255 +29.463258 +29.463263 +29.463268 +29.463273 +29.463283 +29.46329 +29.4633 +29.4633 +29.463305 +29.46331 +29.463315 +29.463318 +29.46332 +29.463318 +29.463317 +29.463315 +29.46331 +29.463303 +29.463298 +29.463293 +29.463285 +29.46328 +29.463277 +29.463273 +29.46327 +29.463267 +29.463263 +29.463258 +29.463253 +29.46325 +29.463247 +29.463243 +29.463242 +29.463243 +29.463245 +29.463248 +29.463248 +29.46325 +29.463252 +29.46325 +29.463252 +29.463253 +29.463255 +29.463255 +29.463253 +29.463252 +29.46325 +29.463252 +29.463252 +29.463253 +29.463255 +29.463257 +29.463257 +29.463257 +29.463255 +29.463255 +29.463257 +29.463258 +29.46326 +29.463263 +29.463267 +29.46327 +29.463273 +29.463273 +29.463278 +29.46328 +29.46328 +29.463282 +29.463282 +29.463282 +29.463283 +29.463285 +29.463288 +29.463292 +29.463293 +29.463297 +29.463298 +29.463298 +29.463297 +29.463293 +29.463292 +29.463292 +29.463288 +29.463285 +29.463282 +29.463277 +29.463273 +29.463272 +29.463268 +29.463267 +29.463267 +29.463265 +29.463262 +29.46326 +29.463262 +29.463262 +29.463267 +29.463275 +29.463283 +29.463288 +29.463288 +29.463293 +29.463293 +29.463293 +29.463295 +29.463297 +29.463298 +29.463297 +29.463297 +29.463297 +29.463295 +29.463288 +29.46328 +29.463273 +29.463265 +29.46326 +29.46326 +29.463265 +29.463272 +29.463273 +29.463277 +29.463277 +29.463277 +29.463273 +29.463268 +29.463267 +29.463258 +29.463252 +29.463243 +29.463233 +29.463227 +29.463218 +29.463213 +29.463208 +29.463208 +29.463205 +29.463205 +29.46321 +29.463213 +29.463212 +29.463213 +29.463217 +29.463227 +29.463237 +29.463245 +29.463248 +29.463245 +29.463245 +29.463243 +29.463243 +29.463243 +29.463232 +29.463223 +29.463218 +29.463213 +29.463208 +29.463203 +29.463203 +29.4632 +29.4632 +29.463203 +29.463205 +29.463207 +29.463208 +29.46321 +29.463213 +29.463215 +29.46322 +29.46322 +29.463225 +29.463232 +29.46324 +29.46324 +29.463252 +29.46326 +29.46326 +29.463272 +29.463282 +29.463287 +29.463292 +29.463292 +29.463292 +29.463292 +29.463295 +29.463298 +29.463303 +29.463308 +29.463308 +29.46331 +29.463315 +29.463323 +29.463327 +29.463332 +29.463335 +29.463338 +29.463338 +29.46334 +29.46334 +29.463337 +29.463337 +29.46334 +29.463342 +29.46334 +29.463335 +29.463333 +29.46333 +29.463328 +29.463328 +29.463325 +29.463322 +29.46332 +29.463318 +29.463317 +29.463312 +29.463305 +29.463298 +29.463293 +29.46329 +29.463288 +29.463287 +29.463285 +29.463283 +29.463282 +29.46328 +29.463277 +29.463273 +29.463273 +29.46327 +29.46327 +29.463267 +29.463263 +29.463263 +29.463262 +29.463262 +29.46326 +29.46326 +29.46326 +29.463257 +29.463255 +29.463253 +29.463253 +29.463252 +29.463252 +29.463253 +29.463255 +29.46326 +29.463263 +29.463267 +29.463268 +29.463267 +29.463262 +29.463257 +29.463257 +29.463253 +29.46325 +29.463247 +29.463245 +29.463245 +29.463245 +29.463247 +29.463248 +29.46325 +29.463252 +29.46325 +29.463252 +29.463252 +29.463252 +29.46325 +29.463248 +29.463248 +29.463247 +29.463247 +29.463247 +29.463247 +29.463243 +29.463243 +29.463243 +29.463243 +29.463243 +29.463245 +29.463247 +29.463247 +29.46325 +29.463252 +29.46325 +29.463247 +29.463242 +29.46324 +29.463237 +29.463235 +29.463238 +29.463242 +29.463245 +29.46325 +29.46325 +29.463257 +29.463262 +29.463268 +29.463272 +29.463278 +29.463282 +29.463285 +29.463288 +29.463288 +29.463285 +29.463285 +29.463285 +29.463288 +29.463292 +29.463298 +29.463298 +29.463298 +29.463298 +29.463295 +29.463292 +29.463287 +29.46328 +29.463273 +29.463263 +29.463257 +29.46325 +29.463243 +29.463243 +29.463237 +29.463237 +29.463232 +29.463227 +29.463223 +29.46322 +29.463217 +29.463215 +29.463213 +29.463215 +29.463213 +29.463212 +29.463213 +29.463213 +29.463213 +29.463212 +29.46321 +29.46321 +29.46321 +29.463212 +29.463213 +29.463217 +29.463222 +29.463225 +29.463228 +29.463228 +29.463232 +29.463233 +29.463233 +29.463237 +29.46324 +29.463247 +29.463253 +29.463267 +29.463275 +29.463285 +29.463285 +29.463288 +29.46329 +29.46328 +29.463265 +29.463265 +29.463255 +29.46325 +29.46325 +29.463247 +29.463245 +29.463238 +29.463232 +29.463228 +29.463225 +29.463225 +29.463227 +29.463225 +29.463225 +29.46322 +29.463213 +29.463205 +29.463205 +29.4632 +29.463195 +29.46319 +29.463188 +29.463188 +29.463188 +29.463187 +29.463185 +29.463185 +29.463188 +29.463193 +29.463195 +29.463198 +29.4632 +29.463202 +29.463203 +29.463202 +29.463203 +29.463203 +29.463205 +29.463207 +29.463202 +29.4632 +29.4632 +29.4632 +29.463203 +29.463205 +29.463207 +29.463208 +29.463208 +29.463215 +29.463223 +29.463223 +29.463232 +29.463238 +29.463242 +29.463245 +29.463245 +29.463243 +29.463243 +29.463243 +29.463242 +29.463238 +29.463235 +29.463232 +29.46323 +29.463228 +29.463227 +29.463223 +29.463223 +29.46322 +29.463215 +29.463208 +29.463202 +29.463195 +29.46319 +29.463187 +29.463185 +29.463185 +29.463183 +29.463182 +29.46318 +29.463182 +29.463183 +29.463187 +29.463192 +29.463195 +29.463197 +29.4632 +29.463205 +29.46321 +29.463212 +29.463212 +29.463208 +29.463208 +29.463207 +29.463203 +29.463208 +29.463208 +29.463225 +29.463237 +29.463245 +29.463248 +29.463252 +29.463252 +29.463257 +29.463265 +29.463277 +29.463285 +29.463293 +29.463298 +29.4633 +29.463303 +29.463303 +29.463305 +29.463305 +29.463303 +29.4633 +29.463298 +29.463295 +29.46329 +29.46329 +29.463283 +29.463282 +29.46328 +29.463282 +29.463287 +29.463287 +29.46329 +29.463293 +29.46329 +29.46329 +29.463285 +29.463285 +29.463282 +29.463278 +29.463277 +29.463275 +29.46327 +29.463263 +29.463258 +29.463258 +29.46325 +29.46325 +29.463248 +29.463247 +29.463247 +29.46325 +29.463257 +29.46327 +29.463282 +29.463282 +29.463292 +29.463303 +29.46331 +29.463343 +29.46334 +29.463328 +29.463313 +29.463282 +29.463262 +29.463255 +29.463253 +29.463253 +29.463253 +29.463257 +29.463258 +29.46326 +29.463263 +29.463268 +29.463275 +29.463278 +29.463282 +29.463283 +29.463285 +29.463285 +29.463285 +29.463285 +29.463285 +29.463287 +29.463287 +29.463287 +29.463285 +29.463287 +29.463287 +29.463288 +29.463292 +29.463298 +29.463307 +29.463313 +29.46332 +29.463323 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463327 +29.463328 +29.463328 +29.46333 +29.463333 +29.463333 +29.463335 +29.463337 +29.463338 +29.463337 +29.463342 +29.463343 +29.463348 +29.463353 +29.46336 +29.463362 +29.463363 +29.463367 +29.463372 +29.463373 +29.463377 +29.463372 +29.463372 +29.463367 +29.46336 +29.463355 +29.463352 +29.463345 +29.463338 +29.463333 +29.463328 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463325 +29.463323 +29.463323 +29.463323 +29.463323 +29.463323 +29.463322 +29.46332 +29.463318 +29.463318 +29.463312 +29.463308 +29.463308 +29.463303 +29.463295 +29.463295 +29.463285 +29.463277 +29.463273 +29.463268 +29.463263 +29.46326 +29.46326 +29.463258 +29.463257 +29.463258 +29.463258 +29.46326 +29.463262 +29.463265 +29.463268 +29.463272 +29.463273 +29.463273 +29.463273 +29.463273 +29.463273 +29.463273 +29.463272 +29.463268 +29.463267 +29.463265 +29.463263 +29.46326 +29.463258 +29.463257 +29.463253 +29.463253 +29.463268 +29.463287 +29.463287 +29.463297 +29.463305 +29.46331 +29.463307 +29.463295 +29.463283 +29.463272 +29.463262 +29.463262 +29.463252 +29.46324 +29.463227 +29.463215 +29.463213 +29.463213 +29.463217 +29.463227 +29.463227 +29.463232 +29.46323 +29.46323 +29.463228 +29.463228 +29.463228 +29.46323 +29.463237 +29.46324 +29.463238 +29.463237 +29.463235 +29.463235 +29.463233 +29.463233 +29.463233 +29.463235 +29.463237 +29.463247 +29.463247 +29.46326 +29.463272 +29.463277 +29.46328 +29.463283 +29.463283 +29.463282 +29.463283 +29.463288 +29.463287 +29.463285 +29.463287 +29.46329 +29.463295 +29.463302 +29.463308 +29.463308 +29.463315 +29.463318 +29.463325 +29.46333 +29.463333 +29.463335 +29.463335 +29.463337 +29.463337 +29.463322 +29.463322 +29.463303 +29.463293 +29.463283 +29.463268 +29.463258 +29.463258 +29.463252 +29.463252 +29.463253 +29.463253 +29.463258 +29.46326 +29.463263 +29.463265 +29.463263 +29.463263 +29.46326 +29.463257 +29.463255 +29.463257 +29.463257 +29.463257 +29.463258 +29.46326 +29.463262 +29.463265 +29.463263 +29.463263 +29.46326 +29.463255 +29.46325 +29.463245 +29.463243 +29.463243 +29.463245 +29.463245 +29.463245 +29.463248 +29.463248 +29.463253 +29.463257 +29.46326 +29.463262 +29.463263 +29.46327 +29.463277 +29.463282 +29.463285 +29.463288 +29.463287 +29.463283 +29.463285 +29.463288 +29.46329 +29.463287 +29.463285 +29.463285 +29.463282 +29.463287 +29.463287 +29.463292 +29.463297 +29.463303 +29.463305 +29.463305 +29.463308 +29.463308 +29.463313 +29.46332 +29.463325 +29.463322 +29.463318 +29.463318 +29.463317 +29.463315 +29.463313 +29.463312 +29.463308 +29.463307 +29.463305 +29.463305 +29.463305 +29.463307 +29.463308 +29.46331 +29.463312 +29.463313 +29.463315 +29.463313 +29.463312 +29.463312 +29.46331 +29.463305 +29.463302 +29.463298 +29.463297 +29.463297 +29.463298 +29.463298 +29.4633 +29.463303 +29.463303 +29.463303 +29.463302 +29.4633 +29.463297 +29.463293 +29.463293 +29.46328 +29.463263 +29.463245 +29.463228 +29.463212 +29.463198 +29.463192 +29.463195 +29.463195 +29.46321 +29.463227 +29.46324 +29.463247 +29.463252 +29.463255 +29.463257 +29.463258 +29.463258 +29.463253 +29.463252 +29.463247 +29.463242 +29.463242 +29.463233 +29.463223 +29.463213 +29.463205 +29.4632 +29.4632 +29.4632 +29.4632 +29.463203 +29.463203 +29.463203 +29.463208 +29.46322 +29.46323 +29.463237 +29.46324 +29.463243 +29.463245 +29.463245 +29.46324 +29.463233 +29.463233 +29.463225 +29.463217 +29.463208 +29.463205 +29.463202 +29.463198 +29.463193 +29.463192 +29.463193 +29.463195 +29.4632 +29.46321 +29.463222 +29.463235 +29.463243 +29.463252 +29.463258 +29.463258 +29.463257 +29.463255 +29.46325 +29.463248 +29.463247 +29.463247 +29.463247 +29.463245 +29.463245 +29.463243 +29.463243 +29.463243 +29.463243 +29.463242 +29.463242 +29.463245 +29.463252 +29.463252 +29.463253 +29.463253 +29.463255 +29.463255 +29.463253 +29.46325 +29.463247 +29.463243 +29.463242 +29.463242 +29.463245 +29.463248 +29.463248 +29.46325 +29.463252 +29.463253 +29.463255 +29.463258 +29.463258 +29.463258 +29.46326 +29.463257 +29.463253 +29.46325 +29.463247 +29.463245 +29.463245 +29.463243 +29.46324 +29.46324 +29.463237 +29.463237 +29.463235 +29.463235 +29.463233 +29.463228 +29.463225 +29.46322 +29.463218 +29.463217 +29.463217 +29.463217 +29.463213 +29.463212 +29.46321 +29.463212 +29.463217 +29.46322 +29.46322 +29.463223 +29.463227 +29.463228 +29.463225 +29.463225 +29.463222 +29.463218 +29.463217 +29.463217 +29.463218 +29.463217 +29.463212 +29.463203 +29.4632 +29.463197 +29.463195 +29.463198 +29.4632 +29.4632 +29.463203 +29.463203 +29.463205 +29.463208 +29.463212 +29.463215 +29.463217 +29.463215 +29.463217 +29.463217 +29.463218 +29.46322 +29.463222 +29.463223 +29.463222 +29.463218 +29.463217 +29.463215 +29.463215 +29.463215 +29.463215 +29.463217 +29.463217 +29.463217 +29.463217 +29.463212 +29.463205 +29.4632 +29.463197 +29.463195 +29.463197 +29.463198 +29.4632 +29.4632 +29.463202 +29.463202 +29.4632 +29.463198 +29.463198 +29.463198 +29.463197 +29.463193 +29.463193 +29.46319 +29.46319 +29.463188 +29.46319 +29.46319 +29.46319 +29.463193 +29.463198 +29.463198 +29.463203 +29.463207 +29.46321 +29.463212 +29.463213 +29.463215 +29.463215 +29.463212 +29.46321 +29.46321 +29.463207 +29.463203 +29.463198 +29.463193 +29.463192 +29.463195 +29.463197 +29.463198 +29.4632 +29.463198 +29.4632 +29.4632 +29.4632 +29.463202 \ No newline at end of file diff --git a/src/test/java/TestCompressor.java b/src/test/java/TestCompressor.java index f3a8f57..94ed5b1 100644 --- a/src/test/java/TestCompressor.java +++ b/src/test/java/TestCompressor.java @@ -1,3 +1,7 @@ +import com.github.Cwida.alp.ALPCompression; +import com.github.Cwida.alp.ALPDecompression; +import com.github.Tranway.buff.BuffCompressor; +import com.github.Tranway.buff.BuffDecompressor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -9,13 +13,9 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.junit.jupiter.api.Test; import org.urbcomp.startdb.selfstar.compressor.*; -import org.urbcomp.startdb.selfstar.compressor.xor.ElfPlusXORCompressor; -import org.urbcomp.startdb.selfstar.compressor.xor.ElfStarXORCompressor; -import org.urbcomp.startdb.selfstar.compressor.xor.SElfXORCompressor; +import org.urbcomp.startdb.selfstar.compressor.xor.*; import org.urbcomp.startdb.selfstar.decompressor.*; -import org.urbcomp.startdb.selfstar.decompressor.xor.ElfPlusXORDecompressor; -import org.urbcomp.startdb.selfstar.decompressor.xor.ElfStarXORCanonicalHuffmanDecompressor; -import org.urbcomp.startdb.selfstar.decompressor.xor.ElfStarXORDecompressor; +import org.urbcomp.startdb.selfstar.decompressor.xor.*; import java.io.*; import java.nio.ByteBuffer; @@ -26,6 +26,7 @@ public class TestCompressor { private static final String STORE_FILE = "src/test/resources/result/result.csv"; + private static final String STORE_PRUNING_FILE = "src/test/resources/result/resultPruning.csv"; private static final String STORE_WINDOW_FILE = "src/test/resources/result/resultWindow.csv"; private static final String STORE_BLOCK_FILE = "src/test/resources/result/resultBlock.csv"; private static final double TIME_PRECISION = 1000.0; @@ -36,26 +37,26 @@ public class TestCompressor { INIT_FILE, "Air-pressure.csv", "Air-sensor.csv", - "Basel-temp.csv", - "Basel-wind.csv", "Bird-migration.csv", "Bitcoin-price.csv", - "Blockchain-tr.csv", - "City-lat.csv", - "City-lon.csv", + "Basel-temp.csv", + "Basel-wind.csv", "City-temp.csv", "Dew-point-temp.csv", - "electric_vehicle_charging.csv", - "Food-price.csv", "IR-bio-temp.csv", "PM10-dust.csv", - "POI-lat.csv", - "POI-lon.csv", - "SSD-bench.csv", "Stocks-DE.csv", "Stocks-UK.csv", "Stocks-USA.csv", - "Wind-Speed.csv" + "Wind-Speed.csv", + "Blockchain-tr.csv", + "City-lat.csv", + "City-lon.csv", + "Food-price.csv", + "POI-lat.csv", + "POI-lon.csv", + "SSD-bench.csv", + "electric_vehicle_charging.csv" }; private final Map fileNameParamToTotalBits = new HashMap<>(); @@ -79,9 +80,11 @@ private static double[] toDoubleArray(byte[] byteArray) { @Test public void testAllCompressor() { for (String fileName : fileNames) { -// testXZCompressor(fileName, NO_PARAM); -// testZstdCompressor(fileName, NO_PARAM); -// testSnappyCompressor(fileName, NO_PARAM); + testALPCompressor(fileName, NO_PARAM); + testXZCompressor(fileName, NO_PARAM); + testZstdCompressor(fileName, NO_PARAM); + testSnappyCompressor(fileName, NO_PARAM); + testBuffCompressor(fileName, NO_PARAM); testFloatingCompressor(fileName); } fileNameParamMethodToCompressedBits.forEach((fileNameParamMethod, compressedBits) -> { @@ -94,6 +97,22 @@ public void testAllCompressor() { System.gc(); } + //In this experiment, we implement window by block. + @Test + public void testPruningCompressor() { + for (String fileName : fileNames) { + testPruningFloatingCompressor(fileName); + } + fileNameParamMethodToCompressedBits.forEach((fileNameParamMethod, compressedBits) -> { + String fileNameParam = fileNameParamMethod.split(",")[0] + "," + fileNameParamMethod.split(",")[1]; + long fileTotalBits = fileNameParamToTotalBits.get(fileNameParam); + fileNameParamMethodToCompressedRatio.put(fileNameParamMethod, (compressedBits * 1.0) / fileTotalBits); + }); + System.out.println("Test Pruning Compressor"); + writeResult(STORE_PRUNING_FILE, fileNameParamMethodToCompressedRatio, fileNameParamMethodToCompressTime, fileNameParamMethodToDecompressTime, fileNameParamToTotalBlock); + System.gc(); + } + //In this experiment, we implement window by block. @Test public void testWindowCompressor() { @@ -106,7 +125,7 @@ public void testWindowCompressor() { IDecompressor[] decompressors = new IDecompressor[]{ new ElfStarDecompressor(new ElfStarXORDecompressor()), - new ElfStarDecompressor(new ElfStarXORDecompressor()), // streaming version is the same + new ElfStarDecompressor(new SElfStarXORDecompressor()), }; testParamCompressor(fileName, window, compressors, decompressors); } @@ -127,13 +146,22 @@ public void testBlockCompressor() { for (int block : blockSizes) { for (String fileName : fileNames) { ICompressor[] compressors = new ICompressor[]{ + new BaseCompressor(new ChimpXORCompressor(block)), + new BaseCompressor(new ChimpNXORCompressor(128, block)), + new BaseCompressor(new GorillaXORCompressor(block)), + new ElfCompressor(new ElfXORCompressor(block)), new ElfStarCompressor(new ElfStarXORCompressor(block), block), }; IDecompressor[] decompressors = new IDecompressor[]{ + new BaseDecompressor(new ChimpXORDecompressor()), + new BaseDecompressor(new ChimpNXORDecompressor(128)), + new BaseDecompressor(new GorillaXORDecompressor()), + new ElfDecompressor(new ElfXORDecompressor()), new ElfStarDecompressor(new ElfStarXORDecompressor()), }; - + testALPCompressor(fileName, block); + testBuffCompressor(fileName, block); testParamCompressor(fileName, block, compressors, decompressors); testZstdCompressor(fileName, block); testSnappyCompressor(fileName, block); @@ -148,42 +176,118 @@ public void testBlockCompressor() { } System.out.println("Test Block"); writeResult(STORE_BLOCK_FILE, fileNameParamMethodToCompressedRatio, fileNameParamMethodToCompressTime, fileNameParamMethodToDecompressTime, fileNameParamToTotalBlock); + } + + private void testPruningFloatingCompressor(String fileName) { + String fileNameParam = fileName + "," + NO_PARAM; + fileNameParamToTotalBits.put(fileNameParam, 0L); + fileNameParamToTotalBlock.put(fileNameParam, 0L); + ICompressor[] compressors = new ICompressor[]{ + new ElfStarCompressor(new ElfStarXORCompressorNoFRZGPruning()), + new ElfStarCompressor(new ElfStarXORCompressorNoFRPruning()), + new ElfStarCompressor(new ElfStarXORCompressorNoFRZPruning()), + new ElfStarCompressor(new ElfStarXORCompressor()), + }; + + IDecompressor[] decompressors = new IDecompressor[]{ + new ElfStarDecompressor(new ElfStarXORDecompressor()), + new ElfStarDecompressor(new ElfStarXORDecompressor()), + new ElfStarDecompressor(new ElfStarXORDecompressor()), + new ElfStarDecompressor(new ElfStarXORDecompressor()), + }; + boolean firstMethod = true; + for (int i = 0; i < compressors.length; i++) { + ICompressor compressor = compressors[i]; + try (BlockReader br = new BlockReader(fileName, BLOCK_SIZE)) { + List floatings; + + while ((floatings = br.nextBlock()) != null) { + + double compressTime = 0; + double decompressTime; + if (floatings.size() != BLOCK_SIZE) { + break; + } + if (firstMethod) { + fileNameParamToTotalBits.put(fileNameParam, fileNameParamToTotalBits.get(fileNameParam) + floatings.size() * 64L); + fileNameParamToTotalBlock.put(fileNameParam, fileNameParamToTotalBlock.get(fileNameParam) + 1L); + } + double start = System.nanoTime(); + floatings.forEach(compressor::addValue); + compressor.close(); + compressTime += (System.nanoTime() - start) / TIME_PRECISION; + IDecompressor decompressor = decompressors[i]; + decompressor.setBytes(compressor.getBytes()); + start = System.nanoTime(); + List deValues = decompressor.decompress(); + decompressTime = (System.nanoTime() - start) / TIME_PRECISION; + + assertEquals(deValues.size(), floatings.size()); + for (int j = 0; j < floatings.size(); j++) { + assertEquals(floatings.get(j), deValues.get(j)); + } + String fileNameParamMethod = fileName + "," + NO_PARAM + "," + compressor.getKey(); + if (!fileNameParamMethodToCompressedBits.containsKey(fileNameParamMethod)) { + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, compressor.getCompressedSizeInBits()); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, compressTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, decompressTime); + } else { + long newSize = fileNameParamMethodToCompressedBits.get(fileNameParamMethod) + compressor.getCompressedSizeInBits(); + double newCTime = fileNameParamMethodToCompressTime.get(fileNameParamMethod) + compressTime; + double newDTime = fileNameParamMethodToDecompressTime.get(fileNameParamMethod) + decompressTime; + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, newSize); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, newCTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, newDTime); + } + compressor.refresh(); + decompressor.refresh(); + + } + } catch (Exception e) { + throw new RuntimeException(fileName, e); + } + firstMethod = false; + } } private void testFloatingCompressor(String fileName) { - String fileNameParam = fileName + "," + NO_PARAM; fileNameParamToTotalBits.put(fileNameParam, 0L); fileNameParamToTotalBlock.put(fileNameParam, 0L); ICompressor[] compressors = new ICompressor[]{ -// new BaseCompressor(new ChimpXORCompressor()), -// new BaseCompressor(new ChimpNXORCompressor(128)), -// new BaseCompressor(new GorillaXORCompressor()), -// new ElfCompressor(new ElfXORCompressor()), + new BaseCompressor(new ChimpXORCompressor()), + new BaseCompressor(new ChimpNXORCompressor(128)), + new BaseCompressor(new GorillaXORCompressor()), + new SBaseCompressor(new ChimpAdaXORCompressor()), + new SBaseCompressor(new ChimpNAdaXORCompressor(128)), + new ElfCompressor(new ElfXORCompressor()), new ElfPlusCompressor(new ElfPlusXORCompressor()), -// new ElfStarCompressor(new ElfStarXORCompressorAdaLead()), -// new ElfStarCompressor(new ElfStarXORCompressorAdaLeadAdaTrail()), + new ElfStarCompressor(new ElfStarXORCompressorAdaLead()), + new ElfStarCompressor(new ElfStarXORCompressorAdaLeadAdaTrail()), new ElfStarCompressor(new ElfStarXORCompressor()), new ElfStarCanonicalHuffmanCompressor(new ElfStarXORCompressor()), new ElfStarHuffmanCompressor(new ElfStarXORCompressor()), new SElfStarCompressor(new SElfXORCompressor()), - new SElfStarHuffmanCompressor(new SElfXORCompressor()) + new SElfStarHuffmanCompressor(new SElfXORCompressor()), }; IDecompressor[] decompressors = new IDecompressor[]{ -// new BaseDecompressor(new ChimpXORDecompressor()), -// new BaseDecompressor(new ChimpNXORDecompressor(128)), -// new BaseDecompressor(new GorillaXORDecompressor()), -// new ElfDecompressor(new ElfXORDecompressor()), + new BaseDecompressor(new ChimpXORDecompressor()), + new BaseDecompressor(new ChimpNXORDecompressor(128)), + new BaseDecompressor(new GorillaXORDecompressor()), + new BaseDecompressor(new ChimpAdaXORDecompressor()), + new BaseDecompressor(new ChimpNAdaXORDecompressor(128)), + new ElfDecompressor(new ElfXORDecompressor()), new ElfPlusDecompressor(new ElfPlusXORDecompressor()), -// new ElfStarDecompressor(new ElfStarXORDecompressorAdaLead()), -// new ElfStarDecompressor(new ElfStarXORDecompressorAdaLeadAdaTrail()), + new ElfStarDecompressor(new ElfStarXORDecompressorAdaLead()), + new ElfStarDecompressor(new ElfStarXORDecompressorAdaLeadAdaTrail()), new ElfStarDecompressor(new ElfStarXORDecompressor()), new ElfStarCanonicalHuffmanDecompressor(new ElfStarXORCanonicalHuffmanDecompressor()), new ElfStarHuffmanDecompressor(new ElfStarXORDecompressor()), new ElfStarDecompressor(new ElfStarXORDecompressor()), // streaming version is the same - new SElfStarHuffmanDecompressor(new ElfStarXORDecompressor()) + new SElfStarHuffmanDecompressor(new ElfStarXORDecompressor()), + }; boolean firstMethod = true; for (int i = 0; i < compressors.length; i++) { @@ -208,7 +312,7 @@ private void testFloatingCompressor(String fileName) { compressTime += (System.nanoTime() - start) / TIME_PRECISION; IDecompressor decompressor = decompressors[i]; decompressor.setBytes(compressor.getBytes()); -// + start = System.nanoTime(); List deValues = decompressor.decompress(); decompressTime = (System.nanoTime() - start) / TIME_PRECISION; @@ -232,7 +336,6 @@ private void testFloatingCompressor(String fileName) { } compressor.refresh(); decompressor.refresh(); - } } catch (Exception e) { e.printStackTrace(); @@ -301,6 +404,145 @@ private void testParamCompressor(String fileName, int window, ICompressor[] comp } } + private void testALPCompressor(String fileName, int block) { + long compressorBits; + String fileNameParam = fileName + "," + block; + if (block == NO_PARAM) { + block = BLOCK_SIZE; + } + fileNameParamToTotalBits.put(fileNameParam, 0L); + fileNameParamToTotalBlock.put(fileNameParam, 0L); + double encodingDuration = 0; + double decodingDuration = 0; + try (BlockReader br = new BlockReader(fileName, block)) { + List>> RowGroups = new ArrayList<>(); + List> floatingsList = new ArrayList<>(); + List floatings; + int RGsize = 100; + while ((floatings = br.nextBlock()) != null) { + if (floatings.size() != block) { + break; + } + floatingsList.add(new ArrayList<>(floatings)); + fileNameParamToTotalBits.put(fileNameParam, fileNameParamToTotalBits.get(fileNameParam) + floatings.size() * 64L); + if (floatingsList.size() == RGsize) { + RowGroups.add(new ArrayList<>(floatingsList)); + floatingsList.clear(); + } + fileNameParamToTotalBlock.put(fileNameParam, fileNameParamToTotalBlock.get(fileNameParam) + 1L); + } + if (!floatingsList.isEmpty()) { + RowGroups.add(floatingsList); + } + + long start = System.nanoTime(); + ALPCompression compressor = new ALPCompression(block); + for (List> rowGroup : RowGroups) { + compressor.entry(rowGroup); + compressor.reset(); + } + compressor.flush(); + encodingDuration += System.nanoTime() - start; + + byte[] result = compressor.getOut(); + + start = System.nanoTime(); + ALPDecompression decompressor = new ALPDecompression(result); + + List> deValues = new ArrayList<>(); + for (int i = 0; i < RowGroups.size(); i++) { + List deValue = decompressor.entry(); + deValues.add(deValue); + } + decodingDuration += System.nanoTime() - start; + + for (int RGidx = 0; RGidx < RowGroups.size(); RGidx++) { + for (int i = 0; i < RowGroups.get(RGidx).size(); i++) { + for (int j = 0; j < RowGroups.get(RGidx).get(i).size(); j++) { + assertEquals(RowGroups.get(RGidx).get(i).get(j), deValues.get(RGidx).get(i)[j], "Value did not match"); + } + } + } + compressorBits = compressor.getSize(); + String fileNameParamMethod = fileNameParam + "," + "ALP"; + if (!fileNameParamMethodToCompressedBits.containsKey(fileNameParamMethod)) { + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, compressorBits); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, encodingDuration / TIME_PRECISION * BLOCK_SIZE / block); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, decodingDuration / TIME_PRECISION * BLOCK_SIZE / block); + } else { + long newSize = fileNameParamMethodToCompressedBits.get(fileNameParamMethod) + compressorBits; + double newCTime = fileNameParamMethodToCompressTime.get(fileNameParamMethod) + encodingDuration / TIME_PRECISION * BLOCK_SIZE / block; + double newDTime = fileNameParamMethodToDecompressTime.get(fileNameParamMethod) + decodingDuration / TIME_PRECISION * BLOCK_SIZE / block; + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, newSize); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, newCTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, newDTime); + } + + } catch (Exception e) { + throw new RuntimeException(fileName, e); + } + } + + private void testBuffCompressor(String fileName, int block) { + long compressorBits; + String fileNameParam = fileName + "," + block; + if (block == NO_PARAM) { + block = BLOCK_SIZE; + } + fileNameParamToTotalBits.put(fileNameParam, 0L); + fileNameParamToTotalBlock.put(fileNameParam, 0L); + try (BlockReader br = new BlockReader(fileName, block)) { + List floatings; + while ((floatings = br.nextBlock()) != null) { + + if (floatings.size() != block) { + break; + } + double[] values = floatings.stream() + .mapToDouble(Double::doubleValue) + .toArray(); + fileNameParamToTotalBits.put(fileNameParam, fileNameParamToTotalBits.get(fileNameParam) + floatings.size() * 64L); + fileNameParamToTotalBlock.put(fileNameParam, fileNameParamToTotalBlock.get(fileNameParam) + 1L); + double encodingDuration = 0; + double decodingDuration = 0; + BuffCompressor compressor = new BuffCompressor(block); + // Compress + long start = System.nanoTime(); + compressor.compress(values); + encodingDuration += System.nanoTime() - start; + compressorBits = compressor.getSize(); + + byte[] result = compressor.getOut(); + BuffDecompressor decompressor = new BuffDecompressor(result); + + start = System.nanoTime(); + + double[] uncompressed = decompressor.decompress(); + decodingDuration += System.nanoTime() - start; + + // Decompressed bytes should equal the original + for (int i = 0; i < floatings.size(); i++) { + assertEquals(floatings.get(i), uncompressed[i], "Value did not match"); + } + String fileNameParamMethod = fileNameParam + "," + "Buff"; + if (!fileNameParamMethodToCompressedBits.containsKey(fileNameParamMethod)) { + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, compressorBits); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, encodingDuration / TIME_PRECISION * BLOCK_SIZE / block); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, decodingDuration / TIME_PRECISION * BLOCK_SIZE / block); + } else { + long newSize = fileNameParamMethodToCompressedBits.get(fileNameParamMethod) + compressorBits; + double newCTime = fileNameParamMethodToCompressTime.get(fileNameParamMethod) + encodingDuration / TIME_PRECISION * BLOCK_SIZE / block; + double newDTime = fileNameParamMethodToDecompressTime.get(fileNameParamMethod) + decodingDuration / TIME_PRECISION * BLOCK_SIZE / block; + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, newSize); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, newCTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, newDTime); + } + } + } catch (Exception e) { + throw new RuntimeException(fileName, e); + } + } + private void testXZCompressor(String fileName, int block) { long compressorBits; String fileNameParam = fileName + "," + block; @@ -547,7 +789,7 @@ private void writeResult(String storeFile, if (fileName.equals(INIT_FILE)) { continue; } - String paramMethod = param + "," + method; + String paramMethod = param + "\t" + method; if (!methodToRatios.containsKey(paramMethod)) { methodToRatios.put(paramMethod, new ArrayList<>()); methodToCTimes.put(paramMethod, new ArrayList<>()); @@ -572,8 +814,8 @@ private void writeResult(String storeFile, if (!file.exists() && !file.mkdirs()) { throw new IOException("Create directory failed: " + file); } - try (FileWriter writer = new FileWriter(storeFile, true)) { - writer.write("Param, Method, Ratio, CTime, DTime"); + try (FileWriter writer = new FileWriter(storeFile, false)) { + writer.write("Dataset, Param, Method, Ratio, CTime, DTime"); writer.write("\r\n"); // 遍历键，并写入对应的值 for (String fileNameParamMethod : fileNameParamMethodToRatio.keySet()) { diff --git a/src/test/java/TestSingleCompressor.java b/src/test/java/TestSingleCompressor.java index 8d2843d..de0c272 100644 --- a/src/test/java/TestSingleCompressor.java +++ b/src/test/java/TestSingleCompressor.java @@ -1,3 +1,7 @@ +import com.github.Cwida.alp.ALPCompression32; +import com.github.Cwida.alp.ALPDecompression32; +import com.github.Tranway.buff.BuffCompressor32; +import com.github.Tranway.buff.BuffDecompressor32; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -56,6 +60,8 @@ public class TestSingleCompressor { public void testAllCompressor() { for (String fileName : fileNames) { testFloatingCompressor(fileName); + testALPCompressor(fileName); + testBuffCompressor(fileName); testXZCompressor(fileName); testZstdCompressor(fileName); testSnappyCompressor(fileName); @@ -94,7 +100,7 @@ private void testFloatingCompressor(String fileName) { new ElfDecompressor32(new ElfXORDecompressor32()), new ElfPlusDecompressor32(new ElfPlusXORDecompressor32()), new ElfStarDecompressor32(new ElfStarXORDecompressor32()), - new ElfStarDecompressor32(new ElfStarXORDecompressor32()) // streaming version is the same + new ElfStarDecompressor32(new SElfStarXORDecompressor32()) }; boolean firstMethod = true; for (int i = 0; i < compressors.length; i++) { @@ -152,6 +158,140 @@ private void testFloatingCompressor(String fileName) { } } + private void testALPCompressor(String fileName) { + long compressorBits; + String fileNameParam = fileName + "," + NO_PARAM; + fileNameParamToTotalBits.put(fileNameParam, 0L); + fileNameParamToTotalBlock.put(fileNameParam, 0L); + double encodingDuration = 0; + double decodingDuration = 0; + try (BlockReader br = new BlockReader(fileName, BLOCK_SIZE)) { + List>> RowGroups = new ArrayList<>(); + List> floatingsList = new ArrayList<>(); + List floatings; + int RGsize = 100; + while ((floatings = br.nextSingleBlock()) != null) { + if (floatings.size() != BLOCK_SIZE) { + break; + } + floatingsList.add(new ArrayList<>(floatings)); + fileNameParamToTotalBits.put(fileNameParam, fileNameParamToTotalBits.get(fileNameParam) + floatings.size() * 32L); + if (floatingsList.size() == RGsize) { + RowGroups.add(new ArrayList<>(floatingsList)); + floatingsList.clear(); + } + fileNameParamToTotalBlock.put(fileNameParam, fileNameParamToTotalBlock.get(fileNameParam) + 1L); + } + if (!floatingsList.isEmpty()) { + RowGroups.add(floatingsList); + } + + long start = System.nanoTime(); + ALPCompression32 compressor = new ALPCompression32(); + for (List> rowGroup : RowGroups) { + compressor.entry(rowGroup); + compressor.reset(); + } + compressor.flush(); + encodingDuration += System.nanoTime() - start; + + byte[] result = compressor.getOut(); + + start = System.nanoTime(); + ALPDecompression32 decompressor = new ALPDecompression32(result); + + List> deValues = new ArrayList<>(); + for (int i = 0; i < RowGroups.size(); i++) { + List deValue = decompressor.entry(); + deValues.add(deValue); + } + decodingDuration += System.nanoTime() - start; + + for (int RGidx = 0; RGidx < RowGroups.size(); RGidx++) { + for (int i = 0; i < RowGroups.get(RGidx).size(); i++) { + for (int j = 0; j < RowGroups.get(RGidx).get(i).size(); j++) { + assertEquals(RowGroups.get(RGidx).get(i).get(j), deValues.get(RGidx).get(i)[j], "Value did not match"); + } + } + } + compressorBits = compressor.getSize(); + String fileNameParamMethod = fileNameParam + "," + "ALP32"; + if (!fileNameParamMethodToCompressedBits.containsKey(fileNameParamMethod)) { + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, compressorBits); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, encodingDuration / TIME_PRECISION); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, decodingDuration / TIME_PRECISION); + } else { + long newSize = fileNameParamMethodToCompressedBits.get(fileNameParamMethod) + compressorBits; + double newCTime = fileNameParamMethodToCompressTime.get(fileNameParamMethod) + encodingDuration / TIME_PRECISION; + double newDTime = fileNameParamMethodToDecompressTime.get(fileNameParamMethod) + decodingDuration / TIME_PRECISION; + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, newSize); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, newCTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, newDTime); + } + + } catch (Exception e) { + throw new RuntimeException(fileName, e); + } + } + + + private void testBuffCompressor(String fileName) { + long compressorBits; + String fileNameParam = fileName + "," + NO_PARAM; + fileNameParamToTotalBits.put(fileNameParam, 0L); + fileNameParamToTotalBlock.put(fileNameParam, 0L); + try (BlockReader br = new BlockReader(fileName, BLOCK_SIZE)) { + List floatings; + while ((floatings = br.nextSingleBlock()) != null) { + if (floatings.size() != BLOCK_SIZE) { + break; + } + float[] values = new float[floatings.size()]; + for (int i = 0; i < floatings.size(); i++) { + values[i] = floatings.get(i); + } + fileNameParamToTotalBits.put(fileNameParam, fileNameParamToTotalBits.get(fileNameParam) + floatings.size() * 64L); + fileNameParamToTotalBlock.put(fileNameParam, fileNameParamToTotalBlock.get(fileNameParam) + 1L); + double encodingDuration = 0; + double decodingDuration = 0; + BuffCompressor32 compressor = new BuffCompressor32(); + // Compress + long start = System.nanoTime(); + compressor.compress(values); + encodingDuration += System.nanoTime() - start; + compressorBits = compressor.getSize(); + + byte[] result = compressor.getOut(); + BuffDecompressor32 decompressor = new BuffDecompressor32(result); + + start = System.nanoTime(); + + float[] uncompressed = decompressor.decompress(); + decodingDuration += System.nanoTime() - start; + + // Decompressed bytes should equal the original + for (int i = 0; i < floatings.size(); i++) { + assertEquals(floatings.get(i), uncompressed[i], "Value did not match"); + } + String fileNameParamMethod = fileNameParam + "," + "Buff32"; + if (!fileNameParamMethodToCompressedBits.containsKey(fileNameParamMethod)) { + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, compressorBits); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, encodingDuration / TIME_PRECISION); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, decodingDuration / TIME_PRECISION); + } else { + long newSize = fileNameParamMethodToCompressedBits.get(fileNameParamMethod) + compressorBits; + double newCTime = fileNameParamMethodToCompressTime.get(fileNameParamMethod) + encodingDuration / TIME_PRECISION; + double newDTime = fileNameParamMethodToDecompressTime.get(fileNameParamMethod) + decodingDuration / TIME_PRECISION; + fileNameParamMethodToCompressedBits.put(fileNameParamMethod, newSize); + fileNameParamMethodToCompressTime.put(fileNameParamMethod, newCTime); + fileNameParamMethodToDecompressTime.put(fileNameParamMethod, newDTime); + } + } + } catch (Exception e) { + throw new RuntimeException(fileName, e); + } + } + private void testXZCompressor(String fileName) { long compressorBits; @@ -391,7 +531,7 @@ private void writeResult(String storeFile, if (fileName.equals(INIT_FILE)) { continue; } - String paramMethod = param + "," + method; + String paramMethod = param + "\t" + method; if (!methodToRatios.containsKey(paramMethod)) { methodToRatios.put(paramMethod, new ArrayList<>()); methodToCTimes.put(paramMethod, new ArrayList<>()); @@ -416,8 +556,8 @@ private void writeResult(String storeFile, if (!file.exists() && !file.mkdirs()) { throw new IOException("Create directory failed: " + file); } - try (FileWriter writer = new FileWriter(storeFile, true)) { - writer.write("Param, Method, Ratio, CTime, DTime"); + try (FileWriter writer = new FileWriter(storeFile, false)) { + writer.write("Dataset, Param, Method, Ratio, CTime, DTime"); writer.write("\r\n"); // 遍历键，并写入对应的值 for (String fileNameParamMethod : fileNameParamMethodToRatio.keySet()) {