Skip to content

Commit bf29d7b

Browse files
popemattpopematt
committed
Misc changes and new visitor API
1 parent 9dc6965 commit bf29d7b

13 files changed

Lines changed: 1511 additions & 4 deletions

File tree

‎src/main/java/com/amazon/ion/v3/MacroInvocation.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ class MacroInvocation(
1818
private val pool: ResourcePool,
1919
private val symbolTable: Array<String?>,
2020
private val macroTable: Array<MacroV2>,
21+
private val evaluateFn: () -> ValueReader = { startMacroEvaluation(macro, arguments, pool, symbolTable, macroTable) }
2122
) {
22-
fun evaluate(): ValueReader { return startMacroEvaluation(macro, arguments, pool, symbolTable, macroTable) }
23+
fun evaluate(): ValueReader = evaluateFn()
2324

2425
fun iterateArguments(): Iterator<ValueReader> = object : Iterator<ValueReader> {
2526
private var i = 0

‎src/main/java/com/amazon/ion/v3/impl_1_0/Helpers.kt‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
package com.amazon.ion.v3.impl_1_0
2+
3+
import com.amazon.ion.*
4+
import com.amazon.ion.v3.*
5+
import com.amazon.ion.v3.impl_1_0.StaticFunctions.Companion.readVarUInt
6+
import java.nio.ByteBuffer
7+
8+
object Helpers {
9+
@JvmStatic
10+
fun ionType(state: Int): IonType? {
11+
return when (state shr 4) {
12+
0x0 -> IonType.NULL
13+
0x1 -> IonType.BOOL
14+
0x2, 0x3 -> IonType.INT
15+
0x4 -> IonType.FLOAT
16+
0x5 -> IonType.DECIMAL
17+
0x6 -> IonType.TIMESTAMP
18+
0x7 -> IonType.SYMBOL
19+
0x8 -> IonType.STRING
20+
0x9 -> IonType.CLOB
21+
0xA -> IonType.BLOB
22+
0xB -> IonType.LIST
23+
0xC -> IonType.SEXP
24+
0xD -> IonType.STRUCT
25+
else -> null
26+
}
27+
}
28+
29+
@JvmStatic
30+
fun tokenType(state: Int): Int {
31+
if (state < 0) {
32+
if (state == ValueReaderBase.TID_END.toInt()) TokenTypeConst.END
33+
return TokenTypeConst.UNSET
34+
}
35+
return typeLookup[state]
36+
}
37+
38+
@JvmStatic
39+
val typeLookup = IntArray(256) {
40+
try {
41+
initType(it)
42+
} catch (e: IonException) {
43+
// TODO: Use "INVALID" here
44+
TokenTypeConst.UNSET
45+
}
46+
}
47+
48+
@JvmStatic
49+
fun initType(state: Int): Int {
50+
return when (state) {
51+
// NOP
52+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E -> TokenTypeConst.NOP
53+
0x0F -> TokenTypeConst.NULL
54+
0x10, 0x11 -> TokenTypeConst.BOOL
55+
0x1F -> TokenTypeConst.NULL
56+
0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E -> throw IonException("Invalid input: illegal typeId: $state")
57+
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E -> TokenTypeConst.INT
58+
0x2F -> TokenTypeConst.NULL
59+
0x30 -> throw IonException("Invalid input: illegal typeId: $state")
60+
0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E -> TokenTypeConst.INT
61+
0x3F -> TokenTypeConst.NULL
62+
0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E -> TokenTypeConst.FLOAT
63+
0x4F -> TokenTypeConst.NULL
64+
0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E -> TokenTypeConst.DECIMAL
65+
0x5F -> TokenTypeConst.NULL
66+
0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E -> TokenTypeConst.TIMESTAMP
67+
0x6F -> TokenTypeConst.NULL
68+
0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E -> TokenTypeConst.SYMBOL
69+
0x7F -> TokenTypeConst.NULL
70+
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E -> TokenTypeConst.STRING
71+
0x8F -> TokenTypeConst.NULL
72+
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E -> TokenTypeConst.CLOB
73+
0x9F -> TokenTypeConst.NULL
74+
0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE -> TokenTypeConst.BLOB
75+
0xAF -> TokenTypeConst.NULL
76+
0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE -> TokenTypeConst.LIST
77+
0xBF -> TokenTypeConst.NULL
78+
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE -> TokenTypeConst.SEXP
79+
0xCF -> TokenTypeConst.NULL
80+
0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE -> TokenTypeConst.STRUCT
81+
0xDF -> TokenTypeConst.NULL
82+
// IVM
83+
0xE0 -> TokenTypeConst.IVM
84+
// Annotations
85+
0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE -> TokenTypeConst.ANNOTATIONS
86+
// Reserved
87+
0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF -> throw IonException(
88+
"Invalid input: illegal typeId: $state"
89+
)
90+
91+
else -> TokenTypeConst.UNSET
92+
}
93+
}
94+
95+
/**
96+
* Returns the next byte after the token being skipped.
97+
*/
98+
@JvmStatic
99+
@OptIn(ExperimentalStdlibApi::class)
100+
fun skip(typeId: Int, buffer: ByteBuffer) {
101+
when (typeId) {
102+
0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xA0, 0xB0, 0xC0, 0xD0 -> { /* Zero-length */
103+
}
104+
105+
0x01, 0x11, 0x21, 0x31, 0x41, 0x51, 0x61, 0x71, 0x81, 0x91, 0xA1, 0xB1, 0xC1, 0xD1, 0xE1 -> {
106+
buffer.position(buffer.position() + 1)
107+
}
108+
109+
0x02, 0x12, 0x22, 0x32, 0x42, 0x52, 0x62, 0x72, 0x82, 0x92, 0xA2, 0xB2, 0xC2, 0xD2, 0xE2 -> {
110+
buffer.position(buffer.position() + 2)
111+
}
112+
113+
0x03, 0x13, 0x23, 0x33, 0x43, 0x53, 0x63, 0x73, 0x83, 0x93, 0xA3, 0xB3, 0xC3, 0xD3, 0xE3 -> {
114+
buffer.position(buffer.position() + 3)
115+
}
116+
117+
0x04, 0x14, 0x24, 0x34, 0x44, 0x54, 0x64, 0x74, 0x84, 0x94, 0xA4, 0xB4, 0xC4, 0xD4, 0xE4 -> {
118+
buffer.position(buffer.position() + 4)
119+
}
120+
121+
0x05, 0x15, 0x25, 0x35, 0x45, 0x55, 0x65, 0x75, 0x85, 0x95, 0xA5, 0xB5, 0xC5, 0xD5, 0xE5 -> {
122+
buffer.position(buffer.position() + 5)
123+
}
124+
125+
0x06, 0x16, 0x26, 0x36, 0x46, 0x56, 0x66, 0x76, 0x86, 0x96, 0xA6, 0xB6, 0xC6, 0xD6, 0xE6 -> {
126+
buffer.position(buffer.position() + 6)
127+
}
128+
129+
0x07, 0x17, 0x27, 0x37, 0x47, 0x57, 0x67, 0x77, 0x87, 0x97, 0xA7, 0xB7, 0xC7, 0xD7, 0xE7 -> {
130+
buffer.position(buffer.position() + 7)
131+
}
132+
133+
0x08, 0x18, 0x28, 0x38, 0x48, 0x58, 0x68, 0x78, 0x88, 0x98, 0xA8, 0xB8, 0xC8, 0xD8, 0xE8 -> {
134+
buffer.position(buffer.position() + 8)
135+
}
136+
137+
0x09, 0x19, 0x29, 0x39, 0x49, 0x59, 0x69, 0x79, 0x89, 0x99, 0xA9, 0xB9, 0xC9, 0xD9, 0xE9 -> {
138+
buffer.position(buffer.position() + 9)
139+
}
140+
141+
0x0A, 0x1A, 0x2A, 0x3A, 0x4A, 0x5A, 0x6A, 0x7A, 0x8A, 0x9A, 0xAA, 0xBA, 0xCA, 0xDA, 0xEA -> {
142+
buffer.position(buffer.position() + 10)
143+
}
144+
145+
0x0B, 0x1B, 0x2B, 0x3B, 0x4B, 0x5B, 0x6B, 0x7B, 0x8B, 0x9B, 0xAB, 0xBB, 0xCB, 0xDB, 0xEB -> {
146+
buffer.position(buffer.position() + 11)
147+
}
148+
149+
0x0C, 0x1C, 0x2C, 0x3C, 0x4C, 0x5C, 0x6C, 0x7C, 0x8C, 0x9C, 0xAC, 0xBC, 0xCC, 0xDC, 0xEC -> {
150+
buffer.position(buffer.position() + 12)
151+
}
152+
153+
0x0D, 0x1D, 0x2D, 0x3D, 0x4D, 0x5D, 0x6D, 0x7D, 0x8D, 0x9D, 0xAD, 0xBD, 0xCD, 0xDD, 0xED -> {
154+
buffer.position(buffer.position() + 13)
155+
}
156+
157+
0x0E, 0x1E, 0x2E, 0x3E, 0x4E, 0x5E, 0x6E, 0x7E, 0x8E, 0x9E, 0xAE, 0xBE, 0xCE, 0xDE, 0xEE -> {
158+
val length = readVarUInt(buffer).toInt()
159+
buffer.position(buffer.position() + length)
160+
}
161+
162+
0x0F, 0x1F, 0x2F, 0x3F, 0x4F, 0x5F, 0x6F, 0x7F, 0x8F, 0x9F, 0xAF, 0xBF, 0xCF, 0xDF -> { /* Nulls */
163+
}
164+
// IVM
165+
0xE0 -> throw IonException("Cannot skip IVM")
166+
// Reserved
167+
0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF -> throw IonException(
168+
"Invalid input: illegal typeId: $typeId"
169+
)
170+
171+
else -> TODO("This should be unreachable: ${typeId.toHexString()}")
172+
}
173+
}
174+
175+
@JvmStatic
176+
fun length(typeId: Int, buffer: ByteBuffer): Int {
177+
return when (val it = lengthAmounts[typeId]) {
178+
-2 -> throw IonException("Invalid input: illegal typeId: $typeId")
179+
-1 -> readVarUInt(buffer).toInt()
180+
else -> it
181+
}
182+
}
183+
184+
@JvmStatic
185+
val lengthAmounts = IntArray(256) { lengthInit(it) }
186+
187+
@JvmStatic
188+
@OptIn(ExperimentalStdlibApi::class)
189+
fun lengthInit(typeId: Int): Int {
190+
return when (typeId) {
191+
0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xA0, 0xB0, 0xC0, 0xD0 -> 0
192+
0x01, 0x11, 0x21, 0x31, 0x41, 0x51, 0x61, 0x71, 0x81, 0x91, 0xA1, 0xB1, 0xC1, 0xD1, 0xE1 -> 1
193+
0x02, 0x12, 0x22, 0x32, 0x42, 0x52, 0x62, 0x72, 0x82, 0x92, 0xA2, 0xB2, 0xC2, 0xD2, 0xE2 -> 2
194+
0x03, 0x13, 0x23, 0x33, 0x43, 0x53, 0x63, 0x73, 0x83, 0x93, 0xA3, 0xB3, 0xC3, 0xD3, 0xE3 -> 3
195+
0x04, 0x14, 0x24, 0x34, 0x44, 0x54, 0x64, 0x74, 0x84, 0x94, 0xA4, 0xB4, 0xC4, 0xD4, 0xE4 -> 4
196+
0x05, 0x15, 0x25, 0x35, 0x45, 0x55, 0x65, 0x75, 0x85, 0x95, 0xA5, 0xB5, 0xC5, 0xD5, 0xE5 -> 5
197+
0x06, 0x16, 0x26, 0x36, 0x46, 0x56, 0x66, 0x76, 0x86, 0x96, 0xA6, 0xB6, 0xC6, 0xD6, 0xE6 -> 6
198+
0x07, 0x17, 0x27, 0x37, 0x47, 0x57, 0x67, 0x77, 0x87, 0x97, 0xA7, 0xB7, 0xC7, 0xD7, 0xE7 -> 7
199+
0x08, 0x18, 0x28, 0x38, 0x48, 0x58, 0x68, 0x78, 0x88, 0x98, 0xA8, 0xB8, 0xC8, 0xD8, 0xE8 -> 8
200+
0x09, 0x19, 0x29, 0x39, 0x49, 0x59, 0x69, 0x79, 0x89, 0x99, 0xA9, 0xB9, 0xC9, 0xD9, 0xE9 -> 9
201+
0x0A, 0x1A, 0x2A, 0x3A, 0x4A, 0x5A, 0x6A, 0x7A, 0x8A, 0x9A, 0xAA, 0xBA, 0xCA, 0xDA, 0xEA -> 10
202+
0x0B, 0x1B, 0x2B, 0x3B, 0x4B, 0x5B, 0x6B, 0x7B, 0x8B, 0x9B, 0xAB, 0xBB, 0xCB, 0xDB, 0xEB -> 11
203+
0x0C, 0x1C, 0x2C, 0x3C, 0x4C, 0x5C, 0x6C, 0x7C, 0x8C, 0x9C, 0xAC, 0xBC, 0xCC, 0xDC, 0xEC -> 12
204+
0x0D, 0x1D, 0x2D, 0x3D, 0x4D, 0x5D, 0x6D, 0x7D, 0x8D, 0x9D, 0xAD, 0xBD, 0xCD, 0xDD, 0xED -> 13
205+
0x0E, 0x1E, 0x2E, 0x3E, 0x4E, 0x5E, 0x6E, 0x7E, 0x8E, 0x9E, 0xAE, 0xBE, 0xCE, 0xDE, 0xEE -> -1
206+
// Nulls
207+
0x0F, 0x1F, 0x2F, 0x3F, 0x4F, 0x5F, 0x6F, 0x7F, 0x8F, 0x9F, 0xAF, 0xBF, 0xCF, 0xDF -> 0
208+
// IVM
209+
0xE0 -> 3 // ...3 more than the typeId byte.
210+
// Reserved
211+
0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF -> -2
212+
213+
else -> TODO("This should be unreachable: ${typeId.toHexString()}")
214+
}
215+
}
216+
}

‎src/main/java/com/amazon/ion/v3/impl_1_0/VarIntHelper.kt‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package com.amazon.ion.v3.impl_1_0
2+
3+
import com.amazon.ion.*
4+
import java.nio.ByteBuffer
5+
6+
class VarIntHelper {
7+
8+
companion object {
9+
/**
10+
* Returns an unsigned integer up to 7 bytes, with an 1 byte integer signifying how many varuint bytes were used in its encoding.
11+
*
12+
*/
13+
@JvmStatic
14+
fun readVarUIntValueAndLength0(source: ByteBuffer, position: Int): Long {
15+
var p = position
16+
var result = 0L
17+
var length = 0
18+
var currentByte: Int
19+
do {
20+
length++
21+
currentByte = source.get(p++).toInt()
22+
result = (result shl 7) or (currentByte and 0b01111111).toLong()
23+
} while (currentByte >= 0)
24+
25+
return (result shl 8) or length.toLong()
26+
}
27+
28+
@JvmStatic
29+
fun readVarUIntValueAndLength(source: ByteBuffer, position: Int): Long {
30+
val currentByte: Int = source.get(position).toInt()
31+
val result = (currentByte and 0b01111111).toLong()
32+
if (currentByte < 0) {
33+
return (result shl 8) or 1L
34+
} else {
35+
return readVarUIntValueAndLength2(source, position + 1, result)
36+
}
37+
}
38+
39+
@JvmStatic
40+
fun readVarUIntValueAndLength2(source: ByteBuffer, position: Int, partialResult: Long): Long {
41+
var currentByte: Int = source.get(position).toInt()
42+
var result = (partialResult shl 7) or (currentByte and 0b01111111).toLong()
43+
if (currentByte < 0) {
44+
return (result shl 8) or 2
45+
}
46+
47+
var p = position
48+
var length = 2
49+
do {
50+
length++
51+
currentByte = source.get(++p).toInt()
52+
result = (result shl 7) or (currentByte and 0b01111111).toLong()
53+
} while (currentByte >= 0)
54+
55+
return (result shl 8) or length.toLong()
56+
57+
}
58+
59+
/**
60+
* Returns a signed integer up to 7 bytes, with an 1 byte integer signifying how many varuint bytes were used in its encoding.
61+
*
62+
*/
63+
@JvmStatic
64+
fun readVarIntValueAndLength(source: ByteBuffer, position: Int): Long {
65+
var p = position
66+
67+
var length = 1
68+
var currentByte = source.get(p++)
69+
var result = (currentByte.toInt() and 0x3F).toLong()
70+
// 1. Shift the sign-bit leftwards to the two's-complement sign-bit of a java integer
71+
// 2. Signed shift right so that we have either -1 or 0
72+
// 3. "or" with 1 so that we have -1 or 1
73+
val sign = (((currentByte.toInt() and 0b01000000) shl 25) shr 32) or 1
74+
75+
while (currentByte >= 0) {
76+
length++
77+
currentByte = source.get(p++)
78+
result = (result shl 7) or (currentByte.toInt() and 0b01111111).toLong()
79+
}
80+
81+
return ((sign * result) shl 8) or length.toLong()
82+
}
83+
}
84+
85+
86+
87+
class ValueAndLength(
88+
@JvmField
89+
var value: Long,
90+
@JvmField
91+
var length: Int,
92+
)
93+
94+
@JvmField
95+
val returnValue = ValueAndLength(0, -1)
96+
97+
fun readVarUIntAt(source: ByteBuffer, position: Int): ValueAndLength {
98+
val firstByte = source.get(position)
99+
if (firstByte < 0) {
100+
101+
val r = returnValue
102+
r.value = firstByte.toLong() and 0x7F
103+
r.length = 1
104+
return r
105+
} else {
106+
return readVarUIntAt(source, position + 1, firstByte)
107+
}
108+
}
109+
110+
fun readVarUIntAt(source: ByteBuffer, position: Int, firstByte: Byte): ValueAndLength {
111+
var p = position
112+
var result = firstByte.toLong()
113+
do {
114+
val currentByte = source.get(p++)
115+
result = (result shl 7) or (currentByte.toInt() and 0b01111111).toLong()
116+
} while (currentByte >= 0)
117+
if (result > Int.MAX_VALUE) throw IonException("Found a VarUInt that was too large to fit in a `int` -- $result")
118+
val r = returnValue
119+
r.value = result
120+
r.length = (p - position + 1)
121+
return r
122+
}
123+
124+
fun readVarIntAt(source: ByteBuffer, position: Int): ValueAndLength {
125+
var result: Long
126+
var p = position
127+
var currentByte = source.get(p++)
128+
// 1. Shift the sign-bit leftwards to the two's-complement sign-bit of a java integer
129+
// 2. Signed shift right so that we have either -1 or 0
130+
// 3. "or" with 1 so that we have -1 or 1
131+
val sign = (((currentByte.toInt() and 0b01000000) shl 25) shr 32) or 1
132+
result = (currentByte.toInt() and 0x3F).toLong()
133+
while (currentByte >= 0) {
134+
currentByte = source.get(p++)
135+
result = (result shl 7) or (currentByte.toInt() and 0b01111111).toLong()
136+
if (result > Int.MAX_VALUE) throw IonException("Found a VarInt that was too large to fit in a `Int` -- $result")
137+
}
138+
val r = returnValue
139+
r.value = result * sign
140+
r.length = p - position
141+
return r
142+
}
143+
}

0 commit comments

Comments
 (0)