From 113ba0a6c2f99778d2c5bc05088d45876ab447f3 Mon Sep 17 00:00:00 2001 From: Arena Agent Date: Fri, 19 Jun 2026 04:54:08 +0000 Subject: [PATCH] Migrate codebase to Python 3 --- bdec/compiler.py | 9 +- bdec/constraints.py | 6 +- bdec/data.py | 80 +++++++++++------ bdec/decode/__init__.py | 4 +- bdec/decode/entry.py | 2 +- bdec/decode/field.py | 2 +- bdec/decode/sequenceof.py | 2 +- bdec/encode/choice.py | 2 +- bdec/encode/entry.py | 10 +-- bdec/encode/field.py | 22 ++--- bdec/encode/sequence.py | 12 ++- bdec/encode/sequenceof.py | 2 +- bdec/encode/test/testfield.py | 2 +- bdec/encode/test/testsequence.py | 2 +- bdec/expression.py | 19 ++-- bdec/inspect/chooser.py | 13 +-- bdec/inspect/param.py | 20 +++-- bdec/inspect/range.py | 57 ++++++++++-- bdec/inspect/solver.py | 22 ++--- bdec/inspect/test/testchooser.py | 26 +++--- bdec/inspect/test/testparam.py | 2 +- bdec/inspect/test/testsolver.py | 2 +- bdec/inspect/type.py | 7 ++ bdec/output/instance.py | 3 +- bdec/output/test/testinstance.py | 2 +- bdec/output/xmlout.py | 15 ++-- bdec/parsing/__init__.py | 17 ++-- bdec/parsing/test/testparsing.py | 10 +-- bdec/spec/__init__.py | 8 +- bdec/spec/asn1.py | 13 +-- bdec/spec/integer.py | 3 +- bdec/spec/protobuffer.py | 4 +- bdec/spec/references.py | 2 +- bdec/spec/test/testxmlspec.py | 45 +++++----- bdec/spec/xmlspec.py | 46 +++++----- bdec/templates/c/settings.py | 12 +-- bdec/test/decoders.py | 46 +++++----- bdec/test/testchoice.py | 5 +- bdec/test/testconstraints.py | 6 +- bdec/test/testdata.py | 18 ++-- bdec/test/testexpression.py | 8 +- bdec/test/testfield.py | 17 ++-- bdec/test/testsequence.py | 1 + bdec/tools/compile.py | 48 +++++------ bdec/tools/decode.py | 50 +++++------ bdec/tools/encode.py | 6 +- docs/source/conf.py | 2 +- regression/test.py | 8 +- specs/msword/python/decode.py | 6 +- specs/test/__init__.py | 4 +- tools/release.py | 144 +++++++++++++++---------------- tools/test/testrelease.py | 4 +- 52 files changed, 489 insertions(+), 389 deletions(-) diff --git a/bdec/compiler.py b/bdec/compiler.py index 9fc98f6..686e080 100644 --- a/bdec/compiler.py +++ b/bdec/compiler.py @@ -51,7 +51,7 @@ import mako.runtime import os import os.path -import pkg_resources +from importlib import resources import sys import bdec.choice as chc @@ -74,11 +74,10 @@ def __init__(self, name): self.directory = os.path.join('templates', name) def listdir(self): - return pkg_resources.resource_listdir('bdec', self.directory) + return [child.name for child in resources.files('bdec').joinpath(self.directory).iterdir() if child.is_file()] def read(self, filename): - return pkg_resources.resource_string('bdec', - os.path.join(self.directory, filename)) + return resources.files('bdec').joinpath(self.directory, filename).read_text() class FilesystemTemplate(TemplateDir): def __init__(self, directory): @@ -133,7 +132,7 @@ def load_templates(template_dir): return Templates(common_templates, entry_templates, config_file) def _generate_template(output_dir, filename, lookup, template): - output = file(os.path.join(output_dir, filename), 'w') + output = open(os.path.join(output_dir, filename), 'w') try: context = mako.runtime.Context(output, **lookup) template.render_context(context) diff --git a/bdec/constraints.py b/bdec/constraints.py index 4b200d2..b56cee8 100644 --- a/bdec/constraints.py +++ b/bdec/constraints.py @@ -57,7 +57,7 @@ class Minimum(Constraint): def check(self, entry, value, context): expected = self.limit.evaluate(context) - if isinstance(value, basestring): + if isinstance(value, str): # It is useful to check the bounds of a text character... value = ord(value) if int(value) < expected: @@ -72,7 +72,7 @@ class Maximum(Constraint): def check(self, entry, value, context): expected = self.limit.evaluate(context) - if isinstance(value, basestring): + if isinstance(value, str): # It is useful to check the bounds of a text character... value = ord(value) if int(value) > expected: @@ -98,7 +98,7 @@ def check(self, entry, value, context): length_diff = len(value) - len(expected) if length_diff > 0: # The expected value is shorter than the actual, so grow it. - expected = Data('\x00' * (length_diff / 8 + 1), 0, length_diff) + expected + expected = Data('\x00' * (length_diff // 8 + 1), 0, length_diff) + expected elif length_diff < 0: # The expected value is longer than the actual, so shrink it. shorter = expected.copy() diff --git a/bdec/data.py b/bdec/data.py index 14003e5..6c3cf0c 100644 --- a/bdec/data.py +++ b/bdec/data.py @@ -48,6 +48,22 @@ import struct import weakref + +def _byte_value(value): + """Return an integer byte from a one-byte str/bytes/int value.""" + if isinstance(value, int): + return value + if isinstance(value, bytes): + return value[0] + return ord(value) + + +def _to_text_bytes(value): + """Normalize bytes-like input to the Python-2 style latin1 str used internally.""" + if isinstance(value, bytes): + return value.decode('latin1') + return value + import bdec class DataError(Exception): @@ -165,7 +181,7 @@ def _shift_chars(self, num_bits): byte = 0 for char in self._bytes(): value = byte << (8 - num_bits) - byte = ord(char) + byte = _byte_value(char) value |= byte >> num_bits yield chr(value & 0xFF) yield chr((byte << (8 - num_bits)) & 0xff) @@ -174,9 +190,14 @@ def __rshift__(self, num_bits): if num_bits == 0: return self elif num_bits % 8 == 0: - return _MemoryBuffer('\x00' * (num_bits / 8) + self._bytes()) + return _MemoryBuffer('\x00' * (num_bits // 8) + self._bytes()) return _MemoryBuffer(''.join(self._shift_chars(num_bits))) + def __getitem__(self, key): + if isinstance(key, slice): + return _MemoryBuffer(self._bytes()[key]) + return self.read_byte(key) + def __getslice__(self, start, end): result = _MemoryBuffer(self._bytes()[start:end]) return result @@ -195,7 +216,7 @@ def read_byte(self, offset): if len(result) == 0: raise _OutOfDataError() self._offset = offset + 1 - return ord(result) + return _byte_value(result) def __len__(self): pos = self._file.tell() @@ -219,7 +240,7 @@ def read_byte(self, offset): self._buffer += self._file.read(extra_bytes_needed) if len(self._buffer) < offset + 1: raise _OutOfDataError() - return ord(self._buffer[offset]) + return _byte_value(self._buffer[offset]) def __len__(self): try: @@ -232,14 +253,15 @@ def __len__(self): class _MemoryBuffer(_ByteBuffer): """Byte buffer that reads directly from in memory data.""" def __init__(self, buffer): - assert isinstance(buffer, str), "Expected a string; got %s!" % repr(buffer) + buffer = _to_text_bytes(buffer) + assert isinstance(buffer, str), "Expected a string/bytes; got %s!" % repr(buffer) self._buffer = buffer def read_byte(self, offset): """Read a byte at a given offset""" if offset >= len(self._buffer): raise _OutOfDataError() - return ord(self._buffer[offset]) + return _byte_value(self._buffer[offset]) def __len__(self): return len(self._buffer) @@ -271,7 +293,7 @@ def __init__(self, buffer="", start=None, end=None): # data objects behave differently depending on what they were # constructed from (a source of bugs) (eg: verification of length # at pop time, as opposed to read time). - if isinstance(buffer, str): + if isinstance(buffer, (str, bytes)): self._buffer = _MemoryBuffer(buffer) elif isinstance(buffer, _ByteBuffer): self._buffer = buffer @@ -337,14 +359,14 @@ def text(self, encoding): encoding -- The unicode encoding the data is in. """ try: - return unicode(self.bytes(), encoding) + return self.bytes().encode('latin1').decode(encoding) except UnicodeDecodeError: raise BadTextEncodingError(self, encoding) def __repr__(self): """Return a textual representation of the data.""" if len(self) % 8 == 0: - return 'hex (%i bytes): %s' % (len(self) / 8, self.get_hex()) + return 'hex (%i bytes): %s' % (len(self) // 8, self.get_hex()) else: return 'bin (%i bits): %s' % (len(self), self.get_binary_text()) @@ -374,10 +396,10 @@ def __eq__(self, other): b = other._get_bits() while 1: try: - a_bit = a.next() + a_bit = next(a) except StopIteration: try: - b.next() + next(b) # Not equal, as 'other' is longer then we are return False except StopIteration: @@ -385,7 +407,7 @@ def __eq__(self, other): return True try: - b_bit = b.next() + b_bit = next(b) except StopIteration: # Not equal, as we are longer than 'other' return False @@ -407,7 +429,7 @@ def __len__(self): self._end = end return self._end - self._start - def __nonzero__(self): + def __bool__(self): return not self.empty() def empty(self): @@ -422,7 +444,7 @@ def empty(self): # We don't know where the data ends, so look to see if we can read # more of the data. try: - self._get_bits().next() + next(self._get_bits()) return False except StopIteration: pass @@ -438,7 +460,7 @@ def _get_bit(self, i): i += self._start if self._end is not None and i >= self._end: raise _OutOfDataError() - byte = i / 8 + byte = i // 8 i = i % 8 return (self._buffer.read_byte(byte) >> (7 - i)) & 1 @@ -504,8 +526,8 @@ def __add__(self, other): if not other: return self - left = self._buffer[self._start / 8:(self._end - 1) / 8 + 1] - right = other._buffer[other._start / 8:(other._end - 1) / 8 + 1] + left = self._buffer[self._start // 8:(self._end - 1) // 8 + 1] + right = other._buffer[other._start // 8:(other._end - 1) // 8 + 1] # Shift the shorter data object (as the shift is relatively intensive) if len(self) < len(other): @@ -518,7 +540,7 @@ def __add__(self, other): # It's possible we have to truncate the buffer here, as we may # have create an extra byte on the right that contains data we # don't care about. - left = left[:(left_start + len(self) - 1) / 8 + 1] + left = left[:(left_start + len(self) - 1) // 8 + 1] else: # The right hand buffer is shorter than the left, so shift it so it # aligns with the left @@ -527,12 +549,12 @@ def __add__(self, other): left_start = self._start % 8 right >>= distance right_start = other._start % 8 + distance - right = right[right_start / 8:(right_start + len(other) - 1) / 8 + 1] + right = right[right_start // 8:(right_start + len(other) - 1) // 8 + 1] if (left_start + len(self)) % 8 and len(self) and len(other): # The left doesn't end on a whole byte; create a joining byte to # connect the left & right - merge_byte = (left_start + len(self) - 1) / 8 + merge_byte = (left_start + len(self) - 1) // 8 overlapping_bits = (left_start + len(self)) % 8 overlap = left.read_byte(merge_byte) & (0xff << (8 - overlapping_bits)) @@ -550,7 +572,7 @@ def _get_bytes(self): if self._start % 8 == 0 and self._end is not None and self._end % 8 == 0: # Optimise for the case where we know the length of the data, and # it is byte aligned. - for i in xrange(self._start / 8, self._end / 8): + for i in range(self._start // 8, self._end // 8): try: yield self._buffer.read_byte(i) except _OutOfDataError: @@ -572,10 +594,11 @@ def __float__(self): """ Convert the data buffer to a float that has been encoded in big endian. """ + raw = self.bytes().encode('latin1') if len(self) == 4 * 8: - return struct.unpack('>f', self.bytes())[0] + return struct.unpack('>f', raw)[0] elif len(self) == 8 * 8: - return struct.unpack('>d', self.bytes())[0] + return struct.unpack('>d', raw)[0] else: raise FloatLengthError(self) @@ -583,10 +606,11 @@ def get_litten_endian_float(self): """ Convert the data buffer to a float that has been encoded in little endian. """ + raw = self.bytes().encode('latin1') if len(self) == 4 * 8: - return struct.unpack('>= 8 if data != 0: @@ -674,7 +698,7 @@ def from_int_big_endian(value, length): length -- The length in bits of the data buffer to create.""" data = int(value) chars = [] - num_bytes = length / 8 + num_bytes = length // 8 if length % 8 != 0: num_bytes += 1 for i in range(num_bytes): @@ -710,7 +734,7 @@ def from_hex(hex): if len(entry) % 2: entry = '0' + entry - for i in range(len(entry) / 2): + for i in range(len(entry) // 2): offset = i * 2 value = entry[offset:offset + 2] for char in value: diff --git a/bdec/decode/__init__.py b/bdec/decode/__init__.py index 01ed1a7..17f8ab1 100644 --- a/bdec/decode/__init__.py +++ b/bdec/decode/__init__.py @@ -67,8 +67,8 @@ def _get_decoder(self, entry, lookup): # Construct the children decoders for child in entry.children: - passed_params = zip(lookup.get_passed_variables(entry, child), - lookup.get_params(child.entry)) + passed_params = list(zip(lookup.get_passed_variables(entry, child), + lookup.get_params(child.entry))) decoder.children.append(Child(child.name, self._get_decoder(child.entry, lookup), passed_params)) diff --git a/bdec/decode/entry.py b/bdec/decode/entry.py index 7f61b99..207ec82 100644 --- a/bdec/decode/entry.py +++ b/bdec/decode/entry.py @@ -133,7 +133,7 @@ def decode(self, data, context, name=None): if self.entry.length is not None: try: data = data.pop(self.entry.length.evaluate(context)) - except dt.DataError, ex: + except dt.DataError as ex: raise EntryDataError(self.entry, ex) # Do the actual decode of this entry (and all child entries). diff --git a/bdec/decode/field.py b/bdec/decode/field.py index 2b38142..b1f2363 100644 --- a/bdec/decode/field.py +++ b/bdec/decode/field.py @@ -32,7 +32,7 @@ def _decode(self, data, context, name): # wrap all access to it in an exception handler. try: value = self.entry.decode_value(field_data) - except dt.DataError, ex: + except dt.DataError as ex: raise FieldDataError(self.entry, ex) yield (False, name, self.entry, field_data, value) diff --git a/bdec/decode/sequenceof.py b/bdec/decode/sequenceof.py index bdd1e0b..fe84958 100644 --- a/bdec/decode/sequenceof.py +++ b/bdec/decode/sequenceof.py @@ -32,7 +32,7 @@ def _loop(self, context, data): if count < 0: raise NegativeSequenceofLoop(self.entry, count) - for i in xrange(count): + for i in range(count): yield None elif self.entry.end_entries: while not context['should end']: diff --git a/bdec/encode/choice.py b/bdec/encode/choice.py index 77bbd96..b3c53f4 100644 --- a/bdec/encode/choice.py +++ b/bdec/encode/choice.py @@ -118,7 +118,7 @@ def get_default_option_params(choice, child, expression_params, encode_expressio Returns a {param_name: value} dictionary.""" result = {} outputs = _get_unpopulated_outputs(choice, child, encode_expression_params) - for name, params in outputs.items(): + for name, params in list(outputs.items()): # Find the possible range of source values, and intersect it with # the constraints placed on it by other earlier children. possible = Ranges(p.type.range(expression_params) for p in params) diff --git a/bdec/encode/entry.py b/bdec/encode/entry.py index 0c580a3..b39bfc0 100644 --- a/bdec/encode/entry.py +++ b/bdec/encode/entry.py @@ -99,7 +99,7 @@ def _mock_query(parent, entry, offset, name): """A mock query object to return data for hidden common entries. It will return null data for fields, etc.""" - if isinstance(parent, (int, long)) or parent is None: + if isinstance(parent, int) or parent is None: raise MissingInstanceError(parent, entry) try: @@ -109,7 +109,7 @@ def _mock_query(parent, entry, offset, name): # Check to see if it's a compound name... result = {} - for param, value in parent.items(): + for param, value in list(parent.items()): names = param.split('.') if names[0] == name: child_name = '.'.join(names[1:]) @@ -131,7 +131,7 @@ def _mock_query(parent, entry, offset, name): elif entry.format == Field.FLOAT: return 0.0 else: - return Data('\x00' * (length / 8 + 1), length) + return Data('\x00' * (length // 8 + 1), length) elif isinstance(entry, Sequence): return MockSequenceValue() elif isinstance(entry, SequenceOf): @@ -162,7 +162,7 @@ def _solve(self, expression, value, context): be resolved correctly.''' ref_values = solve(expression, self.entry, self._params, context, value) - for ref, ref_value in ref_values.items(): + for ref, ref_value in list(ref_values.items()): context[ref.name] = ref_value def _get_value(self, query, parent, offset, name, context): @@ -262,7 +262,7 @@ def encode(self, query, value, offset, context, name): # Check that the expression length matches the 'real' length try: self._solve(self.entry.length, encode_length, context) - except SolverError, ex: + except SolverError as ex: raise DataLengthError(self.entry, ex.expr, ex.expected) if self._is_length_referenced: context[self.entry.name + ' length'] = length diff --git a/bdec/encode/field.py b/bdec/encode/field.py index 96f59ea..d55d4b9 100644 --- a/bdec/encode/field.py +++ b/bdec/encode/field.py @@ -114,14 +114,14 @@ def convert_value(entry, value, length, params=None): try: if isinstance(value, Data): value = value.copy() - elif isinstance(value, int) or isinstance(value, long): + elif isinstance(value, int) or isinstance(value, int): try: value = Data.from_int_big_endian(value, int(length)) except UndecodedReferenceError: value = _encode_unknown_variable_length_integer(entry, value, params) else: value = Data.from_binary_text(_convert_type(entry, value, str)) - except DataError, ex: + except DataError as ex: raise FieldDataError(entry, ex) elif entry.format == Field.HEX: if isinstance(value, Data): @@ -129,7 +129,7 @@ def convert_value(entry, value, length, params=None): else: value = Data.from_hex(_convert_type(entry, value, str)) elif entry.format == Field.TEXT: - value = _convert_type(entry, value, unicode) + value = _convert_type(entry, value, str) elif entry.format == Field.INTEGER: value = _convert_type(entry, value, int) elif entry.format == Field.FLOAT: @@ -154,13 +154,13 @@ def _encode_data(entry, value, length): assert isinstance(value, Data) result = value.copy() elif entry.format == Field.TEXT: - assert isinstance(value, basestring) + assert isinstance(value, str) try: result = Data(value.encode(entry.encoding)) except UnicodeDecodeError: raise BadEncodingError(entry, value) elif entry.format == Field.INTEGER: - assert isinstance(value, (int, long)) + assert isinstance(value, int) if length is None: raise FieldDataError(entry, 'Unable to encode integer field ' 'without explicit length') @@ -196,7 +196,7 @@ def encode_value(entry, value, length=None): try: return _encode_data(entry, value, length) - except DataError, ex: + except DataError as ex: raise FieldDataError(entry, ex) @@ -218,13 +218,13 @@ def _get_default(self, context): # to add leading nulls. try: length = self.entry.length.evaluate(context) - except UndecodedReferenceError, ex: + except UndecodedReferenceError as ex: # We don't know what length it should be. Just make # it a multiple of whole bytes. length = len(value) if length % 8: length = length + (8 - length % 8) - value = Data('\x00' * (length / 8 + 1), 0, length - len(value)) + value + value = Data('\x00' * (length // 8 + 1), 0, length - len(value)) + value break else: if self.is_hidden: @@ -232,7 +232,7 @@ def _get_default(self, context): value = Ranges([range]).get_default() try: length = self.entry.length.evaluate(context) - except UndecodedReferenceError, ex: + except UndecodedReferenceError as ex: # We don't know, and can't calculate, the length if value == 0: length = 0 @@ -241,7 +241,7 @@ def _get_default(self, context): if self.entry.format in [Field.HEX, Field.BINARY]: value = Data.from_int_big_endian(value, length) elif self.entry.format in [Field.TEXT]: - value = '\x00' * (length / 8 - 1) + chr(value) + value = '\x00' * (length // 8 - 1) + chr(value) else: # We don't have a default for this entry raise MissingFieldException(self.entry) @@ -251,7 +251,7 @@ def _get_data(self, value, context): try: length = self.entry.length.evaluate(context) return encode_value(self.entry, value, length) - except UndecodedReferenceError, ex: + except UndecodedReferenceError as ex: # We don't know how long this entry should be. if self.entry.format == self.entry.INTEGER: return _encode_unknown_variable_length_integer(self.entry, value, self._params) diff --git a/bdec/encode/sequence.py b/bdec/encode/sequence.py index bf628b9..7a4195b 100644 --- a/bdec/encode/sequence.py +++ b/bdec/encode/sequence.py @@ -52,6 +52,7 @@ from bdec.expression import UndecodedReferenceError from bdec.inspect.solver import solve from bdec.inspect.type import EntryValueType, EntryLengthType +from functools import reduce class CyclicEncodingError(DecodeError): def __init__(self, entry, loop): @@ -165,7 +166,7 @@ def _is_unknown_value(self, value): def _fixup_expression_value(self, value): # A sequence is always an integer; when we're encoding from xml the raw # value is a string, which means something else in constraints. - return int(value) + return self._sequence_int_value(value) def _fixup_value(self, value, context): """ @@ -185,12 +186,19 @@ def _fixup_value(self, value, context): return constraint.limit.evaluate(context) return value + def _sequence_int_value(self, value): + if hasattr(value, 'childNodes'): + text = ''.join(child.data for child in value.childNodes + if getattr(child, 'nodeType', None) == child.TEXT_NODE) + return int(text) + return int(value) + def _encode(self, query, value, context): if self.entry.value: # Update the context with the detected parameters if value is None: raise MissingValueError(self.entry) - self._solve(self.entry.value, int(value), context) + self._solve(self.entry.value, self._sequence_int_value(value), context) sequence_data = {} for child in self.order(): diff --git a/bdec/encode/sequenceof.py b/bdec/encode/sequenceof.py index 2a669d1..42ae547 100644 --- a/bdec/encode/sequenceof.py +++ b/bdec/encode/sequenceof.py @@ -74,5 +74,5 @@ def _encode(self, query, value, context): # Update the context with the detected parameters try: self._solve(self.entry.count, count, context) - except SolverError, ex: + except SolverError as ex: raise InvalidSequenceOfCount(self.entry, ex.expr, ex.expected) diff --git a/bdec/encode/test/testfield.py b/bdec/encode/test/testfield.py index cb70134..2cb845a 100644 --- a/bdec/encode/test/testfield.py +++ b/bdec/encode/test/testfield.py @@ -101,4 +101,4 @@ def test_unicode_field(self): Field('c', format=Field.TEXT, length=parse('${b:} * 8'), encoding='utf8') ]) self.assertEqual('\x0c\xe3\x83\xad\xe3\x82\xb0\xe3\x82\xa4\xe3\x83\xb3', - encode(a, {'c':u'ログイン'}).bytes()) + encode(a, {'c':'ログイン'}).bytes()) diff --git a/bdec/encode/test/testsequence.py b/bdec/encode/test/testsequence.py index 587124c..9ee5906 100644 --- a/bdec/encode/test/testsequence.py +++ b/bdec/encode/test/testsequence.py @@ -112,7 +112,7 @@ def test_cyclic_dependency_error(self): Field('payload', length=parse('${header:.length} * 8 - len{header:}'), format=Field.TEXT)]) try: encode(a, {'payload':'boom'}) - except CyclicEncodingError, ex: + except CyclicEncodingError as ex: self.assertTrue("'header:' -> 'payload' -> 'header:'" in str(ex), str(ex)) def test_length_reference(self): diff --git a/bdec/expression.py b/bdec/expression.py index 36fbb9b..2e20ed5 100644 --- a/bdec/expression.py +++ b/bdec/expression.py @@ -44,12 +44,13 @@ import bdec.data as dt import operator +from functools import reduce # A list of supported operators, in order of precedence _operators = [ [ ('*', operator.mul), - ('/', operator.div), + ('/', operator.floordiv), ('%', operator.mod), ], [ @@ -75,7 +76,7 @@ def __init__(self, name, context): def __str__(self): return "Missing context '%s' (have %s)" % (self.name, - ', '.join("'%s'" % k for k in self.context.keys())) + ', '.join("'%s'" % k for k in list(self.context.keys()))) class NullReferenceError(UndecodedReferenceError): def __str__(self): @@ -99,8 +100,10 @@ def evaluate(self, context): def __mul__(self, other): return ArithmeticExpression(operator.mul, self, other) - def __div__(self, other): - return ArithmeticExpression(operator.div, self, other) + def __truediv__(self, other): + return ArithmeticExpression(operator.floordiv, self, other) + + __div__ = __truediv__ def __mod__(self, other): return ArithmeticExpression(operator.mod, self, other) @@ -159,7 +162,7 @@ def evaluate(self, context): # In python -ve / +ve will round towards minus infinity, so we only # need to handle the round up case. - result = numerator / denominator + result = numerator // denominator if numerator % denominator and self.should_round_up: result += 1 return result @@ -187,7 +190,7 @@ def __repr__(self): class ReferenceExpression(Expression): """A reference to a value or length of another entry.""" def __init__(self, name): - assert isinstance(name, basestring) + assert isinstance(name, str) self.name = name def param_name(self): @@ -288,7 +291,7 @@ def parse(text): complete = _int_expression() + StringEnd() try: return complete.parseString(text)[0] - except ParseException, ex: + except ParseException as ex: raise ExpressionError(ex) # Legacy name for parse function compile = parse @@ -354,6 +357,6 @@ def _collapse_bool(s,l,t): complete = bool_expr + StringEnd() try: return complete.parseString(text)[0] - except ParseException, ex: + except ParseException as ex: raise ExpressionError(ex) diff --git a/bdec/inspect/chooser.py b/bdec/inspect/chooser.py index b5154b0..3bbab44 100644 --- a/bdec/inspect/chooser.py +++ b/bdec/inspect/chooser.py @@ -54,6 +54,7 @@ import bdec.sequence as seq import bdec.sequenceof as sof from collections import defaultdict +from functools import reduce class _UnknownData: """ @@ -155,7 +156,7 @@ def __eq__(self, other): return False return self.data == other.data - def next(self): + def __next__(self): """Return an iterator to _ProtocolStream items""" return self._next(0) @@ -201,7 +202,7 @@ def query(context, entry, i, name): def _can_differentiate(lookup, fallback): """Test to see if a lookup differentiates itself from other options.""" current_entries = None - for value, entries in lookup.iteritems(): + for value, entries in lookup.items(): entry_set = set(entries) if current_entries is None: current_entries = entry_set @@ -276,14 +277,14 @@ def _differentiate(entries): for entry, option in options[:]: if len(option.data) == 0: - next = list((entry, next) for next in option.next()) + next_options = list((entry, item) for item in next(option)) options.remove((entry, option)) - for item in next: + for item in next_options: if item not in options: # We found a unique item! options.append(item) - if len(next) == 0: + if len(next_options) == 0: have_new_success = True finished.add(entry) @@ -320,7 +321,7 @@ def choose(self, data): copy = data.copy() for offset, length, lookup, undistinguished, finished, possible_failure in self._cache: lookup_entries = set() - for keyed_entries in lookup.values(): + for keyed_entries in list(lookup.values()): lookup_entries.update(keyed_entries) assert set(self._entries) == lookup_entries | undistinguished | \ diff --git a/bdec/inspect/param.py b/bdec/inspect/param.py index cc5c493..8e33ba6 100644 --- a/bdec/inspect/param.py +++ b/bdec/inspect/param.py @@ -61,6 +61,10 @@ import bdec.expression as expr from bdec.inspect.type import VariableType, IntegerType, MultiSourceType, \ EntryType, EntryValueType, EntryLengthType, ShouldEndType +from functools import cmp_to_key +# cmp shim removed (Python 2 builtin no longer exists). +# The single call site below now uses a key function directly. + MAGIC_UNKNOWN_NAME = 'magic unknown param' @@ -248,7 +252,7 @@ def get_type(self): if len(self.types) > 1: type = MultiSourceType(self.types) elif len(self.types) == 1: - type = iter(self.types).next() + type = next(iter(self.types)) else: raise _FailedToResolveError(self.reference.name) return type @@ -302,7 +306,7 @@ def __init__(self, entries): self._populate_child_input_parameter_type(entry, name, param_type, visited) should_have_failed = False - for entry, references in unreferenced_entries.iteritems(): + for entry, references in unreferenced_entries.items(): for param in self._params[entry]: if not param.types: should_have_failed = True @@ -311,7 +315,7 @@ def __init__(self, entries): # We only want top-level entries as this provides the # most context to the user (and if a child entry is # failing, so to will the top level entry). - name = iter(references).next().name + name = next(iter(references)).name stack = self._find_child_using_param(entry, name) raise UnknownReferenceError(stack[0], name, stack[1:]) assert not should_have_failed, 'Found a parameter with an unknown type, ' \ @@ -456,7 +460,7 @@ def _populate_child_input_parameter_type(self, entry, name, param_type, visited) def is_output_param_used(self, entry, child, param): assert param.direction == Param.OUT - return param.name in self._local_child_param_name[entry][child].values() + return param.name in list(self._local_child_param_name[entry][child].values()) def _get_local_reference(self, entry, child, param): """Get the local of a parameter used by a child entry. @@ -586,7 +590,7 @@ def get_locals(self, entry): if _VariableParam(local, child_param.direction, None) not in params: locals.setdefault(local.param_name(), set()).add(child_param.get_type()) result = [] - for name, types in locals.items(): + for name, types in list(locals.items()): if len(types) == 1: type = types.pop() else: @@ -603,7 +607,7 @@ def compare_references(left, right): # It doesn't really matter what order we use for parameters, but it # has to be consistent. As such we order by name, and put value types # before length types. - result = cmp(left.reference.name, right.reference.name) + result = (left.reference.name > right.reference.name) - (left.reference.name < right.reference.name) if result: return result if isinstance(left.reference, expr.ValueResult) and \ @@ -614,7 +618,7 @@ def compare_references(left, right): return 1 return 0 - params.sort(cmp=compare_references) + params.sort(key=cmp_to_key(compare_references)) return params def get_params(self, entry): @@ -623,7 +627,7 @@ def get_params(self, entry): """ try: return [param.get_param() for param in self._get_params(entry)] - except _FailedToResolveError, ex: + except _FailedToResolveError as ex: raise UnknownReferenceError(entry, ex.name) def get_passed_variables(self, entry, child): diff --git a/bdec/inspect/range.py b/bdec/inspect/range.py index b3da082..96e2554 100644 --- a/bdec/inspect/range.py +++ b/bdec/inspect/range.py @@ -50,6 +50,21 @@ import decimal class _Infinite: + def __eq__(self, other): + return isinstance(other, _Infinite) + + def __lt__(self, other): + return False + + def __gt__(self, other): + return not isinstance(other, _Infinite) + + def __le__(self, other): + return isinstance(other, _Infinite) + + def __ge__(self, other): + return True + def __mul__(self, other): if other == 0: return 0 @@ -67,6 +82,21 @@ def __cmp__(self, other): class _MinusInfinite: + def __eq__(self, other): + return isinstance(other, _MinusInfinite) + + def __lt__(self, other): + return not isinstance(other, _MinusInfinite) + + def __gt__(self, other): + return False + + def __le__(self, other): + return True + + def __ge__(self, other): + return isinstance(other, _MinusInfinite) + def __mul__(self, other): if other == 0: return 0 @@ -105,8 +135,12 @@ def __init__(self, min=None, max=None): def intersect(self, other): """Return the intesection of self and other.""" result = Range() - # max of None and X will always return X - result.min = max(self.min, other.min) + if self.min is None: + result.min = other.min + elif other.min is None: + result.min = self.min + else: + result.min = max(self.min, other.min) if self.max is not None: if other.max is not None: result.max = min(self.max, other.max) @@ -119,8 +153,10 @@ def intersect(self, other): def union(self, other): """Return the outer bounds of self and other.""" result = Range() - # min of None and X will always return None - result.min = min(self.min, other.min) + if self.min is None or other.min is None: + result.min = None + else: + result.min = min(self.min, other.min) if self.max is not None and other.max is not None: result.max = max(self.max, other.max) return result @@ -169,11 +205,11 @@ def __mul__(self, other): values.append(maxx * maxy) values.sort() - min = values[0] if isinstance(values[0], (int, long)) else None - max = values[3] if isinstance(values[3], (int, long)) else None + min = values[0] if isinstance(values[0], int) else None + max = values[3] if isinstance(values[3], int) else None return Range(min, max) - def __div__(self, other): + def __truediv__(self, other): # Instead of implementing division, we'll just figure out the inverse, # and multiply the two together. if other.min == 0: @@ -204,6 +240,9 @@ def __div__(self, other): max = int(values[3]) if isinstance(values[3], decimal.Decimal) else None return Range(min, max) + __div__ = __truediv__ + __floordiv__ = __truediv__ + def __mod__(self, other): if other.max is not None: # FIXME: This assumes 'a % b' returns a postive integer; this isn't @@ -236,7 +275,7 @@ def __or__(self, other): maximum = max(self.max, other.max) return Range(minimum, maximum) - def __nonzero__(self): + def __bool__(self): if self.min is not None and self.max is not None: if self.min > self.max: # This is an empty span... @@ -286,7 +325,7 @@ def remove(self, range): subsequent_ranges = self._ranges[i:] del self._ranges[i:] for j, r in enumerate(subsequent_ranges): - if r.min < range.min: + if range.min is not None and (r.min is None or r.min < range.min): # This range starts before the range we are removing, # so add the start non-overlapping section. self._ranges.append(Range(r.min, range.min - 1)) diff --git a/bdec/inspect/solver.py b/bdec/inspect/solver.py index 9af061b..129e12a 100644 --- a/bdec/inspect/solver.py +++ b/bdec/inspect/solver.py @@ -91,7 +91,7 @@ def _break_into_parts(entry, expression, input_params): # We need to add / subtract the common components constant = ArithmeticExpression(expression.op, lconst, rconst) result = left - for ref, expr in right.items(): + for ref, expr in list(right.items()): existing = result.get(ref, Constant(0)) result[ref] = ArithmeticExpression(expression.op, existing, expr) elif left and right: @@ -105,30 +105,30 @@ def _break_into_parts(entry, expression, input_params): # f(y) = (left(params) + kl) * (right(params) + kr) # where left(params) or right(params) is zero. So the result will be # f(y) = kr * left(params) + kl * right(params) + kl * kr - for ref, expr in left.items(): + for ref, expr in list(left.items()): result[ref] = expr * rconst - for ref, expr in right.items(): + for ref, expr in list(right.items()): result[ref] = expr * lconst constant = lconst * rconst elif expression.op == operator.lshift: if right: # Don't support shifting by a non-constant raise SolverError(entry, expression, 'Shifting by a non constant not supported') - for ref, expr in left.items(): + for ref, expr in list(left.items()): result[ref] = expr << rconst constant = lconst << rconst - elif expression.op == operator.div: + elif expression.op == operator.floordiv: if right: raise SolverError(entry, expression, 'Dividing by a non-constant not supported') if len(left) > 1: raise SolverError(entr, expression, 'Dividing two unknowns not supported.') - for ref, expr in left.items(): + for ref, expr in list(left.items()): result[ref] = expr / rconst constant = lconst / rconst elif expression.op == operator.mod: if right: raise SolverError(entry, expression, 'Modding by a non-constant not supported') - for ref, expr in left.items(): + for ref, expr in list(left.items()): result[ref] = expr % rconst constant = lconst % rconst else: @@ -169,11 +169,11 @@ def _invert(result_expr, entry, expression, params, input_params, remainder_rang if right.op == operator.mul: if is_right_const: # left = right * k -> left / k = right - left = ArithmeticExpression(operator.div, left, right.right) + left = ArithmeticExpression(operator.floordiv, left, right.right) right = right.left else: # left = k * right -> left / k = right - left = ArithmeticExpression(operator.div, left, right.left) + left = ArithmeticExpression(operator.floordiv, left, right.left) right = right.right # To correctly handle solving signed integers, eg: @@ -257,7 +257,7 @@ def influence(component): # be really big. result = 1e1024 return result - variables = sorted(components.items(), key=influence, reverse=True) + variables = sorted(list(components.items()), key=influence, reverse=True) result_params = [] for i, (ref, expr) in enumerate(variables): # Get the range of the remaining references (required so we know what @@ -284,7 +284,7 @@ def solve(expression, entry, params, context, value): # 'solve result' variable which we will reference in the inverted # expressions. solve_result = ValueResult('solve result') - constant, variables = solve_expression(solve_result, expression, entry, params, context.keys()) + constant, variables = solve_expression(solve_result, expression, entry, params, list(context.keys())) result = {} original_value = value value -= constant.evaluate(context) diff --git a/bdec/inspect/test/testchooser.py b/bdec/inspect/test/testchooser.py index 017464e..4ef0c40 100644 --- a/bdec/inspect/test/testchooser.py +++ b/bdec/inspect/test/testchooser.py @@ -33,7 +33,7 @@ def test_field(self): a = fld.Field('a', 8) stream = chsr._ProtocolStream(a) self.assertEqual(a, stream.entry) - self.assertEqual([], stream.next()) + self.assertEqual([], next(stream)) def test_sequence(self): blah = seq.Sequence('blah', [fld.Field('a', 8), fld.Field('b', 8)]) @@ -41,17 +41,17 @@ def test_sequence(self): self.assertEqual(0, len(stream.data)) # Now we should move to 'a' - next = stream.next() - self.assertEqual(1, len(next)) - self.assertEqual('a', next[0].entry.name) + next_items = next(stream) + self.assertEqual(1, len(next_items)) + self.assertEqual('a', next_items[0].entry.name) # Now we should move to 'b' - next = next[0].next() - self.assertEqual(1, len(next)) - self.assertEqual('b', next[0].entry.name) + next_items = next(next_items[0]) + self.assertEqual(1, len(next_items)) + self.assertEqual('b', next_items[0].entry.name) # And then we should be done. - self.assertEqual(0, len(next[0].next())) + self.assertEqual(0, len(next(next_items[0]))) def test_choice(self): blah = chc.Choice('blah', [fld.Field('a', 8), fld.Field('b', 8)]) @@ -59,13 +59,13 @@ def test_choice(self): self.assertEqual(0, len(stream.data)) # Now we should move to 'a' or 'b' - next = stream.next() - self.assertEqual(2, len(next)) - self.assertEqual('a', next[0].entry.name) + next_items = next(stream) + self.assertEqual(2, len(next_items)) + self.assertEqual('a', next_items[0].entry.name) # Now both options should finish - self.assertEqual(0, len(next[0].next())) - self.assertEqual(0, len(next[1].next())) + self.assertEqual(0, len(next(next_items[0]))) + self.assertEqual(0, len(next(next_items[1]))) class TestChooser(unittest.TestCase): diff --git a/bdec/inspect/test/testparam.py b/bdec/inspect/test/testparam.py index 67ac060..cb6754a 100644 --- a/bdec/inspect/test/testparam.py +++ b/bdec/inspect/test/testparam.py @@ -137,7 +137,7 @@ def test_sequence_value(self): spec = Sequence('blah', [length, data]) vars = ExpressionParameters([spec]) - self.assertEquals([], vars.get_params(spec)) + self.assertEqual([], vars.get_params(spec)) self.assertTrue(vars.is_value_referenced(lower)) self.assertFalse(vars.is_value_referenced(ignored)) self.assertTrue(vars.is_value_referenced(upper)) diff --git a/bdec/inspect/test/testsolver.py b/bdec/inspect/test/testsolver.py index 9efde88..8fda5fa 100644 --- a/bdec/inspect/test/testsolver.py +++ b/bdec/inspect/test/testsolver.py @@ -42,7 +42,7 @@ def _solve(entry, child, value, context=None): ent = entry.children[child].entry if child is not None else entry result = solve(ent.value, ent, ExpressionParameters([entry]), context, value) - return dict((str(c), v) for c,v in result.items()) + return dict((str(c), v) for c,v in list(result.items())) class TestSolver (unittest.TestCase): diff --git a/bdec/inspect/type.py b/bdec/inspect/type.py index d989452..ad3672d 100644 --- a/bdec/inspect/type.py +++ b/bdec/inspect/type.py @@ -55,6 +55,7 @@ import bdec.field as fld from bdec.inspect.range import Range import bdec.sequence as seq +from functools import reduce def _delayed_range(delayed, entry, parameters, entries_stack): @@ -159,6 +160,9 @@ def is_reference_match(self, expression): class ShouldEndType(IntegerType): """Parameter used to pass the 'should end' up to the parent.""" + def __hash__(self): + return hash(ShouldEndType) + def __eq__(self, other): return isinstance(other, ShouldEndType) @@ -277,6 +281,9 @@ def __init__(self, sources): assert isinstance(source, VariableType) self.sources = sources + def __hash__(self): + return hash(tuple(self.sources)) + def __eq__(self, other): if not isinstance(other, MultiSourceType): return False diff --git a/bdec/output/instance.py b/bdec/output/instance.py index 7f0eaf3..c2c485f 100644 --- a/bdec/output/instance.py +++ b/bdec/output/instance.py @@ -51,6 +51,7 @@ import bdec.output import bdec.sequence as seq import bdec.sequenceof as sof +from functools import reduce def escape(name): return name.replace(' ', '_') @@ -67,7 +68,7 @@ def __getattr__(self, name): raise AttributeError(name) def __repr__(self): - result = unicode(self._children) + result = str(self._children) if self._value is not None: result = '%i %s' % (self._value, result) return result diff --git a/bdec/output/test/testinstance.py b/bdec/output/test/testinstance.py index 900348e..3acd74b 100644 --- a/bdec/output/test/testinstance.py +++ b/bdec/output/test/testinstance.py @@ -178,5 +178,5 @@ def test_sequenceof_to_string(self): def test_sequence_with_hidden_field_to_string(self): a = seq.Sequence('a', [fld.Field('length:', 8), fld.Field('b', format=fld.Field.TEXT, length=parse('${length:} * 8'))]) data = inst.decode(a, dt.Data('\x03cat')) - self.assertEqual(u"{'b': u'cat'}", unicode(data)) + self.assertEqual("{'b': 'cat'}", str(data)) diff --git a/bdec/output/xmlout.py b/bdec/output/xmlout.py index 18018e8..3da5f8f 100644 --- a/bdec/output/xmlout.py +++ b/bdec/output/xmlout.py @@ -45,7 +45,7 @@ import logging import operator import string -import StringIO +import io import xml.dom.minidom import xml.sax.saxutils import xml.sax.xmlreader @@ -58,6 +58,7 @@ import bdec.field as fld from bdec.sequence import Sequence import bdec.sequenceof as sof +from functools import reduce def escape_name(name): if not name: @@ -105,7 +106,7 @@ def _has_expected_value(entry): return True return False -def to_file(items, output, encoding="utf-8", verbose=False): +def to_open(items, output, encoding="utf-8", verbose=False): handler = _XMLGenerator(output, encoding) offset = 0 is_first = True @@ -143,7 +144,7 @@ def to_file(items, output, encoding="utf-8", verbose=False): if has_children: _print_whitespace(handler, offset) - text = xml_strip(unicode(value)) + text = xml_strip(str(value)) handler.characters(text) if verbose and data: @@ -157,8 +158,8 @@ def to_file(items, output, encoding="utf-8", verbose=False): handler.ignorableWhitespace('\n') def to_string(items, verbose=False): - buffer = StringIO.StringIO() - to_file(items, buffer, verbose=verbose) + buffer = io.StringIO() + to_open(items, buffer, verbose=verbose) return buffer.getvalue() class _SequenceOfEntry: @@ -227,8 +228,8 @@ def encode(protocol, xmldata): Returns an iterator to data objects representing the encoded structure. """ - if isinstance(xmldata, basestring): - xmldata = StringIO.StringIO(xmldata) + if isinstance(xmldata, str): + xmldata = io.StringIO(xmldata) document = xml.dom.minidom.parse(xmldata) return reduce(operator.add, protocol.encode(_query_element, document), Data()) diff --git a/bdec/parsing/__init__.py b/bdec/parsing/__init__.py index e1be1ea..0a3ee40 100644 --- a/bdec/parsing/__init__.py +++ b/bdec/parsing/__init__.py @@ -17,7 +17,7 @@ # . import inspect -from string import ascii_letters as alphas, digits as nums, hexdigits as hexnums, printable as printables, lower, upper +from string import ascii_letters as alphas, digits as nums, hexdigits as hexnums, printable as printables, ascii_lowercase as lower, ascii_uppercase as upper from bdec import DecodeError from bdec.choice import Choice @@ -29,6 +29,7 @@ from bdec.sequence import Sequence from bdec.sequenceof import SequenceOf from bdec.spec.references import ReferencedEntry +from functools import reduce alphanums = alphas + nums @@ -42,7 +43,7 @@ def __init__(self, filename, text, offset, ex): @property def _lines(self): - return self._text[:self._offset / 8].splitlines() or [''] + return self._text[:self._offset // 8].splitlines() or [''] @property def lineno(self): @@ -157,7 +158,7 @@ def setParseAction(self, fn): return self.addParseAction(fn) def addParseAction(self, fn): - num_args = len(inspect.getargspec(fn)[0]) + num_args = len(inspect.getfullargspec(fn).args) if num_args == 3: action = lambda t:fn('', 0, t) elif num_args == 2: @@ -169,7 +170,7 @@ def addParseAction(self, fn): return self def parseString(self, text): - if isinstance(text, unicode): + if isinstance(text, str): text = text.encode('ascii') token_stack = [[]] @@ -225,7 +226,7 @@ def _decode(self, text, filename): if not is_starting: offset += len(data) yield is_starting, name, entry, data, value - except DecodeError, ex: + except DecodeError as ex: raise ParseException(filename, text, offset, ex) def __add__(self, other): @@ -287,7 +288,7 @@ def OneOrMore(element): class Literal(ParserElement): def __init__(self, text): ParserElement.__init__(self) - assert isinstance(text, basestring), 'Literal must be a string! Is %s' % (repr(text)) + assert isinstance(text, str), 'Literal must be a string! Is %s' % (repr(text)) assert text, "Literal text entries shouldn't be empty!" self.text = text @@ -529,9 +530,9 @@ def srange(text): def _get_caseless_action(text): if text[0].isupper(): - action = upper + action = str.upper else: - action = lower + action = str.lower return lambda t:action(t[0]) def CaselessLiteral(text): diff --git a/bdec/parsing/test/testparsing.py b/bdec/parsing/test/testparsing.py index 2b4726c..098acdd 100644 --- a/bdec/parsing/test/testparsing.py +++ b/bdec/parsing/test/testparsing.py @@ -57,13 +57,13 @@ def test_ignore(self): comment = '//' + CharsNotIn('\n') + '\n' words = ZeroOrMore(Word(alphas)) + StringEnd() words.ignore(comment) - self.assertEquals(['run', 'dog', 'run'], list(words.parseString('''run // Ignore me + self.assertEqual(['run', 'dog', 'run'], list(words.parseString('''run // Ignore me // Ignore me too dog run'''))) # Test that an entry 'in the middle' can have an ignore entry a = Suppress('start:') + words - self.assertEquals(['run', 'dog', 'run'], list(a.parseString('''start: run // Ignore me + self.assertEqual(['run', 'dog', 'run'], list(a.parseString('''start: run // Ignore me dog //ignore me // ignore me too run'''))) @@ -133,8 +133,8 @@ def test_action_returns_object(self): def test_word_body_chars(self): a = Word(alphas, alphas + nums) + StringEnd() - self.assertEquals(['Dab0c'], list(a.parseString(' Dab0c '))) - self.assertEquals(['Aabc123'], list(a.parseString(' Aabc123 '))) + self.assertEqual(['Dab0c'], list(a.parseString(' Dab0c '))) + self.assertEqual(['Aabc123'], list(a.parseString(' Aabc123 '))) self.assertRaises(ParseException, a.parseString, '0abcd') self.assertRaises(ParseException, a.parseString, '/AbCd') self.assertRaises(ParseException, a.parseString, 'AbCd/') @@ -156,7 +156,7 @@ def test_one_of(self): def test_parse_results(self): def to_int(tokens): - self.assertEquals(ParseResults, tokens.__class__) + self.assertEqual(ParseResults, tokens.__class__) return int(tokens[0]) number = Word(srange('[0-9]')).setParseAction(to_int)('number') expr = (number('a') + number('b'))('c') + StringEnd() diff --git a/bdec/spec/__init__.py b/bdec/spec/__init__.py index 7c7026a..28e9a8c 100644 --- a/bdec/spec/__init__.py +++ b/bdec/spec/__init__.py @@ -43,7 +43,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os.path -from StringIO import StringIO +from io import StringIO class LoadError(Exception): """Base class for all loading errors.""" @@ -110,7 +110,7 @@ def _validate_parameters(entries, lookup): # Validate all of the parameters in use prm.CompoundParameters([prm.EndEntryParameters(entries), prm.ExpressionParameters(entries)]) - except prm.BadReferenceError, ex: + except prm.BadReferenceError as ex: context = [(lookup[e] + (str(e),)) for e in ex.context if e in lookup] raise ReferenceError(ex, ex.entry, lookup, context) @@ -129,7 +129,7 @@ def _resolve(decoder, references, lookup, should_remove_unused): common = references.resolve() if isinstance(decoder, ReferencedEntry): decoder = references.resolve_reference(decoder) - except MissingReferenceError, ex: + except MissingReferenceError as ex: raise ReferenceError(ex, ex.reference, lookup) if should_remove_unused: @@ -192,7 +192,7 @@ def load_specs(specs, main_name=None, should_remove_unused=False): if contents is None: contents = open(filename, 'r') - elif isinstance(contents, basestring): + elif isinstance(contents, str): contents = StringIO(contents) if format is None: diff --git a/bdec/spec/asn1.py b/bdec/spec/asn1.py index fa6e248..97a3b87 100644 --- a/bdec/spec/asn1.py +++ b/bdec/spec/asn1.py @@ -57,6 +57,7 @@ from pyparsing import Word, nums, alphanums, StringEnd, \ ParseException, Optional, Combine, oneOf, alphas,\ QuotedString, empty, lineno, SkipTo, ParserElement +from functools import reduce ParserElement.enablePackrat() @@ -123,7 +124,7 @@ def not_implemented_handler(name): def _handler(text, location, tokens): raise NotImplementedError(name, tokens, filename, lineno(location, text)) return _handler - for name, entry in parsers.items(): + for name, entry in list(parsers.items()): entry.setParseAction(not_implemented_handler(name)) self._common_entries = {} self._constants = {} @@ -219,7 +220,7 @@ def allow_entries_with_name(t, l, tokens): def _load_ebnf(self): # Load the xml spec that we will use for doing the decoding. asn1_filename = os.path.join(os.path.dirname(__file__), '..', '..', 'specs', 'asn1.ber.xml') - generic_spec, lookup = xmlspec.load(asn1_filename, file(asn1_filename, 'r'), self._references) + generic_spec, lookup = xmlspec.load(asn1_filename, open(asn1_filename, 'r'), self._references) table = { 'bstring' : Combine("'" + Word('01') + "'B"), @@ -243,7 +244,7 @@ def _load_ebnf(self): parser = parsers['ModuleDefinition'] + StringEnd() parser.ignore('--' + SkipTo('\n')) - return parser, dict((name, entry) for name, entry in parsers.items() if name not in table), lookup + return parser, dict((name, entry) for name, entry in list(parsers.items()) if name not in table), lookup def _create_named_numeric_list(self, s, l, t): value = 0 @@ -261,7 +262,7 @@ def _create_enumeration(self, s, l, t): value = 0 options = [] for token in t[0]['items']: - if not isinstance(token, basestring): + if not isinstance(token, str): name = token['name'] value = int(token['value']) else: @@ -331,11 +332,11 @@ def load(self, text): """Load a bdec specification from an asn.1 document.""" try: name, modules = self._parser.parseString(text)[0] - except ParseException, ex: + except ParseException as ex: raise Asn1ParseError(ex, self.filename, ex.lineno) common = dict((entry.name, entry) for entry in modules) common.update(self._common_entries) - for module in common.values(): + for module in list(common.values()): self._references.add_common(module) return modules[0], self._source_lookup diff --git a/bdec/spec/integer.py b/bdec/spec/integer.py index e8c604d..8a58132 100644 --- a/bdec/spec/integer.py +++ b/bdec/spec/integer.py @@ -54,6 +54,7 @@ from bdec.expression import compile, Constant, ArithmeticExpression, UndecodedReferenceError from bdec.field import Field from bdec.sequence import Sequence +from functools import reduce class IntegerError(Exception): pass @@ -116,7 +117,7 @@ def signed_litte_endian(self, length_expr): result = self.common[name] except KeyError: children = [] - num_bytes = length / 8 + num_bytes = length // 8 for i in range(num_bytes - 1): children.append(Field('byte %i:' % i, 8)) children.append(Field('signed:', 1)) diff --git a/bdec/spec/protobuffer.py b/bdec/spec/protobuffer.py index 1f03423..5c1bf9a 100644 --- a/bdec/spec/protobuffer.py +++ b/bdec/spec/protobuffer.py @@ -194,12 +194,12 @@ def load(filename, contents, references): """ # Load the parts of protocol buffers implemented in bdec proto_filename = os.path.join(os.path.dirname(__file__), '..', '..', 'specs', 'protobuffer.xml') - generic_spec, lookup = bdec.spec.xmlspec.load(proto_filename, file(proto_filename, 'r'), references) + generic_spec, lookup = bdec.spec.xmlspec.load(proto_filename, open(proto_filename, 'r'), references) parser = _Parser(references) try: entries = parser.parse(contents.read()) - except ParseException, ex: + except ParseException as ex: raise Error(filename, ex.lineno, ex.col, ex) return entries[-1], lookup diff --git a/bdec/spec/references.py b/bdec/spec/references.py index 89ecd3a..289c856 100644 --- a/bdec/spec/references.py +++ b/bdec/spec/references.py @@ -168,6 +168,6 @@ def resolve(self): assert isinstance(entry, Entry) reference.resolve(entry) - for entry in lookup.values(): + for entry in list(lookup.values()): assert isinstance(entry, Entry), "%s isn't an entry! This shouldn't be in common." % entry return [lookup[c.name] for c in self._common] diff --git a/bdec/spec/test/testxmlspec.py b/bdec/spec/test/testxmlspec.py index b829fb0..22b72cd 100644 --- a/bdec/spec/test/testxmlspec.py +++ b/bdec/spec/test/testxmlspec.py @@ -61,6 +61,7 @@ from bdec.spec import load_specs, ReferenceError, UnspecifiedMainEntry, LoadError import bdec.spec.xmlspec as xml from bdec.test.decoders import assert_xml_equivalent +from functools import reduce def loads(text): return load_specs([('', text, 'xml')]) @@ -200,8 +201,8 @@ def test_failure_when_referencing_sequenceof(self): try: loads(text) raise Exception('Exception not thrown!') - except bdec.spec.LoadError, ex: - print str(ex) + except bdec.spec.LoadError as ex: + print(str(ex)) self.assertTrue("Can't reference " in str(ex)) def test_expression_references_sub_field(self): @@ -618,7 +619,7 @@ def test_cannot_use_end_sequenceof_in_reference(self): """ try: loads(text) - except xml.XmlError, ex: + except xml.XmlError as ex: self.assertTrue("end-sequenceof cannot be used within a referenced item" in str(ex)) def test_break_in_break(self): @@ -641,13 +642,13 @@ def test_break_in_break(self): data = dt.Data("chicken\x00bob\x00\00") list(decoder.decode(data)) - self.assertEquals(0, len(data)) + self.assertEqual(0, len(data)) data = dt.Data("chicken\x00bob\x00") try: list(decoder.decode(dt.Data("chicken\x00bob\x00"))) - except fld.FieldDataError, ex: - self.assertEquals(dt.NotEnoughDataError, type(ex.error)) + except fld.FieldDataError as ex: + self.assertEqual(dt.NotEnoughDataError, type(ex.error)) def test_nameless_entry(self): text = """ @@ -829,7 +830,7 @@ def test_optional_entry(self): ''' a = loads(text)[0] - print xml.save(a) + print(xml.save(a)) # Test decoding without a footer list(a.decode(dt.Data('\x00\x00'))) @@ -876,7 +877,7 @@ def test_expression_references_unknown_entry(self): ''' try: loads(text) - except ReferenceError, ex: + except ReferenceError as ex: self.assertEqual("[4]: binary 'b' references " "unknown entry 'c'!\n" " [3]: sequence 'a'" , str(ex)) @@ -890,7 +891,7 @@ def test_unknown_reference(self): ''' try: loads(text) - except ReferenceError, ex: + except ReferenceError as ex: self.assertEqual("[4]: Reference to unknown entry 'missing'!" , str(ex)) def test_conditional_reference(self): @@ -910,7 +911,7 @@ def test_conditional_reference(self): spec = loads(text)[0] data = dt.Data('\x00') list(spec.decode(data)) - self.assertEquals(0, len(data)) + self.assertEqual(0, len(data)) self.assertRaises(bdec.DecodeError, list, spec.decode(dt.Data('\x01'))) data = dt.Data('\x01a') list(spec.decode(data)) @@ -955,8 +956,8 @@ def test_reporting_of_unknown_parameters(self): try: loads(text) assert 0, "Whoops, specification didn't fail!" - except ReferenceError, ex: - self.assertEquals("[5]: binary 'a' references unknown " + except ReferenceError as ex: + self.assertEqual("[5]: binary 'a' references unknown " "entry 'unknown:'!\n" " [6]: sequence 'b'\n" " [9]: sequence 'c'", @@ -972,8 +973,8 @@ def test_error_location_of_internal_entry(self): try: loads(text) assert 0, "Whoops, specification didn't fail!" - except ReferenceError, ex: - self.assertEquals("[3]: sequence 'not present:' references " + except ReferenceError as ex: + self.assertEqual("[3]: sequence 'not present:' references " "unknown entry 'unknown'!\n" " [3]: choice 'optional a'", str(ex)) @@ -983,8 +984,8 @@ def test_bad_field_value_error(self): try: loads(text) self.fail('Whoops, should have failed to load spec!') - except xml.XmlError, ex: - self.assertEquals("[1]: binary 'a' - Invalid binary text 'bad'", str(ex)) + except xml.XmlError as ex: + self.assertEqual("[1]: binary 'a' - Invalid binary text 'bad'", str(ex)) def test_cross_specification_references(self): # Test that we can reference entries between specifications @@ -1003,8 +1004,8 @@ def test_cross_specification_references(self): spec, common, lookup = load_specs([('', a, 'xml'), ('', b, 'xml')]) data = dt.Data('\x0a') items = list(spec.decode(data)) - self.assertEquals(4, len(items)) - self.assertEquals(10, items[-2][-1]) + self.assertEqual(4, len(items)) + self.assertEqual(10, items[-2][-1]) def test_error_with_multiple_decoders(self): # We should get an error when multiple specifications have the top @@ -1020,7 +1021,7 @@ def test_error_with_multiple_decoders(self): try: load_specs([('', a, 'xml'), ('', b, 'xml')]) self.fail("Should have failed to load the spec, but didn't!") - except UnspecifiedMainEntry, ex: + except UnspecifiedMainEntry as ex: self.assertEqual('Multiple top level protocols available! Choose one of:\n a\n b', str(ex)) def test_no_main_decoder(self): @@ -1028,7 +1029,7 @@ def test_no_main_decoder(self): try: load_specs([('', a, 'xml')]) self.fail("Should have failed to load the spec, but didn't!") - except UnspecifiedMainEntry, ex: + except UnspecifiedMainEntry as ex: self.assertEqual('No top level protocol present! Choose one of ' 'the common entries to be the main:\n a', str(ex)) @@ -1055,7 +1056,7 @@ def test_unknown_main_decoder(self): # The user specifies a decoder that doesn't exist.... load_specs([('', a, 'xml')], 'missing') self.fail('Oh dang. Should have failed.') - except ReferenceError, ex: + except ReferenceError as ex: self.assertEqual("unknown[0]: Reference to unknown entry 'missing'!", str(ex)) def test_remove_unused(self): @@ -1101,7 +1102,7 @@ def test_duplicate_common(self): ''' try: loads(text) - except LoadError, ex: + except LoadError as ex: self.assertEqual("[6]: Duplicate common entry 'a'", str(ex)) def test_variable_length_with_expected_value(self): diff --git a/bdec/spec/xmlspec.py b/bdec/spec/xmlspec.py index 36fd07e..6f325ac 100644 --- a/bdec/spec/xmlspec.py +++ b/bdec/spec/xmlspec.py @@ -43,7 +43,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import operator -import StringIO +import io import xml.sax from xml.sax import saxutils @@ -167,25 +167,25 @@ def endElement(self, name): children = self._children[-1] length = None - if attrs.has_key('length'): + if 'length' in attrs: length = self._parse_expression(attrs['length']) entry_name = "" - if attrs.has_key('name'): + if 'name' in attrs: entry_name = attrs['name'] entry = self._handlers[name](attrs, children, entry_name, length, breaks) # Check for value constraints constraints = [] - if attrs.has_key('min'): + if 'min' in attrs: minimum = self._parse_expression(attrs['min']) constraints.append(Minimum(minimum)) - if attrs.has_key('max'): + if 'max' in attrs: maximum = self._parse_expression(attrs['max']) constraints.append(Maximum(maximum)) - if attrs.has_key('expected'): + if 'expected' in attrs: expected = self._parse_expression(attrs['expected']) constraints.append(Equals(expected)) - if name == 'field' and not isinstance(entry, fld.Field) and attrs.has_key('value'): + if name == 'field' and not isinstance(entry, fld.Field) and 'value' in attrs: # Fields that are long integers can be internally converted to # references to Sequences; we need to handle the expected values. expected = self._parse_expression(attrs['value']) @@ -204,7 +204,7 @@ def endElement(self, name): self._children.pop() - if attrs.has_key('if'): + if 'if' in attrs: # This is a 'conditional' entry; only present if the expression in # 'if' is true. To decode this, we create a choice with a 'not # present' option; this option attempts to decode first, with the @@ -212,7 +212,7 @@ def endElement(self, name): try: not_present = exp.parse_conditional_inverse(attrs['if']) not_present.name = 'not present:' - except exp.ExpressionError, ex: + except exp.ExpressionError as ex: raise XmlExpressionError(ex, self._filename, self.locator) assert isinstance(not_present, ent.Entry) @@ -244,7 +244,7 @@ def _common(self, attributes, children, name, length, breaks): for entry in children: try: self._references.add_common(entry) - except DuplicateCommonError, ex: + except DuplicateCommonError as ex: raise self._error(ex) def _protocol(self, attributes, children, name, length, breaks): @@ -253,13 +253,13 @@ def _protocol(self, attributes, children, name, length, breaks): self.decoder = children[0] else: self.decoder = None - for entry in self._integers.common.values(): + for entry in list(self._integers.common.values()): self._references.add_common(entry) def _parse_expression(self, text): try: return exp.compile(text) - except exp.ExpressionError, ex: + except exp.ExpressionError as ex: raise XmlExpressionError(ex, self._filename, self.locator) def _reference(self, attributes, children, name, length, breaks): @@ -277,7 +277,7 @@ def _field(self, attributes, children, name, length, breaks): if length is None: raise self._error("Field entries required a 'length' attribute") - if attributes.has_key('type'): + if 'type' in attributes: lookup = { "binary" : fld.Field.BINARY, "hex" : fld.Field.HEX, @@ -290,7 +290,7 @@ def _field(self, attributes, children, name, length, breaks): format = lookup[attributes['type']] encoding = None - if attributes.has_key('encoding'): + if 'encoding' in attributes: encoding = attributes['encoding'] if format is fld.Field.INTEGER: _integer_encodings = [fld.Field.LITTLE_ENDIAN, fld.Field.BIG_ENDIAN] @@ -307,13 +307,13 @@ def _field(self, attributes, children, name, length, breaks): else: assert encoding == fld.Field.LITTLE_ENDIAN integer = self._integers.signed_litte_endian(length) - except IntegerError, error: + except IntegerError as error: raise self._error(str(error)) return self._references.get_common(name, integer.name) # We'll create the field, then use it to create the expected value. result = fld.Field(name, length, format, encoding) - if attributes.has_key('value'): + if 'value' in attributes: # Get the correct object type by encoding, then decoding, the text # value. expected_text = attributes['value'] @@ -329,7 +329,7 @@ def _field(self, attributes, children, name, length, breaks): # We'll align it to whole bytes, just in case. expected_length += 4 assert expected_length % 8 == 0 - data = dt.Data('\x00' * (expected_length / 8 + 8)) + data = dt.Data('\x00' * (expected_length // 8 + 8)) data += dt.Data.from_hex(expected_text[2:]) unused = data.pop(len(data) - expected_length) if len(unused) and int(unused) != 0: @@ -340,14 +340,14 @@ def _field(self, attributes, children, name, length, breaks): # native format. try: value = convert_value_context(result, expected_text, {}) - except fld.FieldDataError, ex: + except fld.FieldDataError as ex: raise self._error(ex) result.constraints.append(Equals(value)) return result def _sequence(self, attributes, children, name, length, breaks): value = None - if attributes.has_key('value'): + if 'value' in attributes: # A sequence can have a value derived from its children... value = self._parse_expression(attributes['value']) return seq.Sequence(name, children, value, length) @@ -363,7 +363,7 @@ def _sequenceof(self, attributes, children, name, length, breaks): # Default to being a greedy sequenceof, unless we have a length specified count = None - if attributes.has_key('count'): + if 'count' in attributes: count = self._parse_expression(attributes['count']) result = sof.SequenceOf(name, children[0], count, length, breaks) return result @@ -384,7 +384,7 @@ def load(filename, specfile, references): parser.setContentHandler(handler) try: parser.parse(specfile) - except xml.sax.SAXParseException, ex: + except xml.sax.SAXParseException as ex: # The sax parse exception object can operate as a locator raise XmlError(ex.args[0], filename, ex) return (handler.decoder, handler.lookup) @@ -468,7 +468,7 @@ def _write_entry(gen, entry, common, end_entry): attributes.append(('name', entry.name)) name = _handlers[type(entry)](entry, attributes) if entry.length is not None: - attributes.append(('length', str(entry.length))) + attributes.append(('length', str(entry.length))) for constraint in entry.constraints: if isinstance(constraint, Minimum): attributes.append(('min', str(constraint.limit))) @@ -526,7 +526,7 @@ def dump(spec, common, output): def dumps(spec, common=[]): """Save a specification in the xml format.""" - output = StringIO.StringIO() + output = io.StringIO() dump(spec, common, output) return output.getvalue() save = dumps diff --git a/bdec/templates/c/settings.py b/bdec/templates/c/settings.py index 25b2bad..9a72b4b 100644 --- a/bdec/templates/c/settings.py +++ b/bdec/templates/c/settings.py @@ -94,19 +94,19 @@ def escaped_type(entry): # as possible to their unescaped name. names = esc_names(common_names, esc_name) names += esc_names(embedded_names, esc_name, names) - _escaped_types.update(zip(entries, names)) + _escaped_types.update(list(zip(entries, names))) return _escaped_types[entry] def _int_types(): - possible = [(name, -1 << (info[0] - 1), (1 << (info[0] - 1)) - 1) for name, info in signed_types.items()] - possible.extend((name, 0, (1 << info[0]) - 1) for name, info in unsigned_types.items()) + possible = [(name, -1 << (info[0] - 1), (1 << (info[0] - 1)) - 1) for name, info in list(signed_types.items())] + possible.extend((name, 0, (1 << info[0]) - 1) for name, info in list(unsigned_types.items())) possible.sort(key=lambda a:a[2]) for name, minimum, maximum in possible: yield name, minimum, maximum def _biggest(types): """ Return the biggest possible type.""" - return reduce(lambda a, b: a if a[1][0] > b[1][0] else b, types.items())[0] + return reduce(lambda a, b: a if a[1][0] > b[1][0] else b, list(types.items()))[0] def _integer_type(type): """Choose an appropriate integral type for the given type.""" @@ -366,7 +366,7 @@ def encode_passed_params(parent, i, result_name): return _call_params(parent, i, result_name, encode_params) _OPERATORS = { - operator.__div__ : '/', + operator.__floordiv__ : '/', operator.__mod__ : '%', operator.__mul__ : '*', operator.__sub__ : '-', @@ -443,7 +443,7 @@ def enum_value(parent, child_index): offsets = {} for e in iter_entries(): if isinstance(e, chc.Choice): - offsets[e] = range(len(options), len(options) + len(e.children)) + offsets[e] = list(range(len(options), len(options) + len(e.children))) options.extend(c.name for c in e.children) names = esc_names(options, constant) for e in iter_entries(): diff --git a/bdec/test/decoders.py b/bdec/test/decoders.py index 321cd35..fd19aff 100644 --- a/bdec/test/decoders.py +++ b/bdec/test/decoders.py @@ -49,7 +49,7 @@ See the create_decoder_classes function. """ -from ConfigParser import NoOptionError, NoSectionError +from configparser import NoOptionError, NoSectionError import glob import itertools import os @@ -58,7 +58,7 @@ import shutil import subprocess import stat -import StringIO +import io import time import xml.etree.ElementTree @@ -96,7 +96,7 @@ def _is_xml_text_equal(a, b): return a.strip() == b.strip() def _get_elem_text(a, event): - attribs = ' '.join('%s="%s"' % (name, value) for name, value in a.attrib.items()) + attribs = ' '.join('%s="%s"' % (name, value) for name, value in list(a.attrib.items())) text = a.text or "" prefix = '' if event == 'start': @@ -104,9 +104,9 @@ def _get_elem_text(a, event): return "%s%s" % (prefix, text.strip(), a.tag) def assert_xml_equivalent(expected, actual): - a = xml.etree.ElementTree.iterparse(StringIO.StringIO(expected), ['start', 'end']) - b = xml.etree.ElementTree.iterparse(StringIO.StringIO(actual), ['start', 'end']) - for (a_event, a_elem), (b_event, b_elem) in itertools.izip(a, b): + a = xml.etree.ElementTree.iterparse(io.StringIO(expected), ['start', 'end']) + b = xml.etree.ElementTree.iterparse(io.StringIO(actual), ['start', 'end']) + for (a_event, a_elem), (b_event, b_elem) in zip(a, b): if a_event != b_event or a_elem.tag != b_elem.tag or \ a_elem.attrib != b_elem.attrib or \ (a_event == 'end' and not _is_xml_text_equal(a_elem, b_elem)): @@ -126,7 +126,7 @@ def _find_executable(name): def _get_valgrind(): path = _find_executable('valgrind') if path is None: - print 'Failed to find valgrind! Code will not be tested with valgrind.' + print('Failed to find valgrind! Code will not be tested with valgrind.') return path _template_cache = {} @@ -263,9 +263,9 @@ def _validate_xml(spec, data, xmltext): there may be differences in whitespace, and some fields can be represented in multiple ways (eg: 5.0 vs 5.00000 vs 5). """ - xml_entries = xml.etree.ElementTree.iterparse(StringIO.StringIO(xmltext), ['start', 'end']) + xml_entries = xml.etree.ElementTree.iterparse(io.StringIO(xmltext), ['start', 'end']) child_tail=None - for (is_starting, name, entry, data, expected), (a_event, a_elem) in itertools.izip(_decode_visible(spec, data), xml_entries): + for (is_starting, name, entry, data, expected), (a_event, a_elem) in zip(_decode_visible(spec, data), xml_entries): if is_starting and a_event != 'start': raise Exception ("Expected '%s' to be starting, but got '%s' ending" % (name, a_elem.tag)) @@ -309,7 +309,7 @@ def _validate_xml(spec, data, xmltext): # be represented in xml (eg: a string with a binary # character). Encode and decode the expected value to see if # matches now (being escaped itself...) - expected_text = xmlout.xml_strip(unicode(expected)) + expected_text = xmlout.xml_strip(str(expected)) escaped_expected = convert_value(entry, expected_text, len(data)) constraint = Equals(escaped_expected) constraint.check(entry, actual, {}) @@ -330,7 +330,7 @@ def _check_encoded_data(spec, sourcefile, actual, actual_xml, require_exact_enco try: regenerated_xml = xmlout.to_string(spec.decode(dt.Data(actual))) assert_xml_equivalent(actual_xml, regenerated_xml) - except Exception, ex: + except Exception as ex: raise Exception('Re-decoding of encoded data failed: %s' % str(ex)) if require_exact_encoding: @@ -354,7 +354,7 @@ class _CompiledDecoder(object): TEST_DIR = os.path.join(os.path.dirname(__file__), 'temp') VALGRIND = _get_valgrind() - def decode_file(self, spec, common, data, should_check_encoding=True, require_exact_encoding=False): + def decode_open(self, spec, common, data, should_check_encoding=True, require_exact_encoding=False): """Return a tuple containing the exit code and the decoded xml.""" generate(spec, common, self, should_check_encoding) encode_filename = os.path.join(self.TEST_DIR, 'encoded.bin') if should_check_encoding else None @@ -368,7 +368,7 @@ def decode_file(self, spec, common, data, should_check_encoding=True, require_ex try: _validate_xml(spec, dt.Data(data), xml) - except bdec.DecodeError, ex: + except bdec.DecodeError as ex: raise Exception("Compiled decoder succeeded, but should have failed with: %s" % str(ex)) if should_check_encoding: _check_encoded_data(spec, data, open(encode_filename, 'rb').read(), xml, require_exact_encoding) @@ -378,11 +378,11 @@ class _PythonDecoder: """Use the builtin python decoder for the tests.""" NAME = 'Python' - def decode_file(self, spec, common, sourcefile, should_check_encoding=True, require_exact_encoding=False): + def decode_open(self, spec, common, sourcefile, should_check_encoding=True, require_exact_encoding=False): data = dt.Data(sourcefile) try: xml = xmlout.to_string(spec.decode(data)) - except bdec.DecodeError, ex: + except bdec.DecodeError as ex: raise ExecuteError(3, ex) if should_check_encoding: @@ -395,6 +395,8 @@ def _decoder(name, template, compiler): return type('%sDecode' % name, (_CompiledDecoder, ), {'NAME':name, 'LANGUAGE':template, 'COMPILER':compiler.split(' ')})() +_CDecoder = _decoder('C', 'c', 'gcc -Wall -Werror -g -Wno-long-long -pedantic -o decode') + def create_decoder_classes(base_classes, module): """ Return a dictionary of classes derived from unittest.TestCase. @@ -435,9 +437,9 @@ class _BaseRegressionTest: def _test_failure(self, spec, common, spec_filename, data_filename, should_encode): datafile = open(data_filename, 'rb') try: - xml = self.decoder.decode_file(spec, common, datafile, should_encode) + xml = self.decoder.decode_open(spec, common, datafile, should_encode) raise Exception("'%s' should have failed to decode '%s', but succeeded with output:\n%s" % (spec_filename, data_filename, xml)) - except ExecuteError, ex: + except ExecuteError as ex: if ex.exit_code != 3: # It should have been a decode error... raise @@ -447,10 +449,10 @@ def _test_success(self, spec, common, spec_filename, data_filename, expected_xml if os.path.splitext(data_filename)[1] == ".gz": # As gzip'ed files seek extremely poorly, we'll read the file completely into memory. import gzip - datafile = StringIO.StringIO(gzip.GzipFile(data_filename, 'rb').read()) + datafile = io.StringIO(gzip.GzipFile(data_filename, 'rb').read().decode('latin1')) else: datafile = open(data_filename, 'rb') - xml = self.decoder.decode_file(spec, common, datafile, should_encode, require_exact_encoding) + xml = self.decoder.decode_open(spec, common, datafile, should_encode, require_exact_encoding) datafile.close() if expected_xml: assert_xml_equivalent(expected_xml, xml) @@ -471,7 +473,7 @@ def _test_spec(self, test_path, spec_filename, entry_name, successes, failures, self._get(config, 'default', test_path.lower()) if skip == 'decoding-broken': - print 'Skipping test.' + print('Skipping test.') return elif skip == 'encoding-broken': should_encode = False @@ -489,7 +491,7 @@ def _test_spec(self, test_path, spec_filename, entry_name, successes, failures, expected_xml = None expected_filename = '%s.expected.xml' % os.path.splitext(data_filename)[0] if os.path.exists(expected_filename): - xml_file = file(expected_filename, 'r') + xml_file = open(expected_filename, 'r') expected_xml = xml_file.read() xml_file.close() if entry_name is not None: @@ -526,7 +528,7 @@ def create_classes(name, tests, config): config_filename -- The path to the fixme config file. """ clsname = name[0].upper() + name[1:] - cls = type(clsname, (object, _BaseRegressionTest,), {}) + cls = type(clsname, (_BaseRegressionTest,), {}) for test_name, spec_filename, entry, successes, failures in tests: cls.add_method(test_name, '%s/%s' % (name, test_name), spec_filename, entry, successes, failures, config) return create_decoder_classes([(cls, clsname)], __name__) diff --git a/bdec/test/testchoice.py b/bdec/test/testchoice.py index a87c23d..8763771 100644 --- a/bdec/test/testchoice.py +++ b/bdec/test/testchoice.py @@ -55,6 +55,7 @@ import bdec.field as fld import bdec.sequence as seq import bdec.expression as expr +from functools import reduce def get_best_guess(entry, data): @@ -63,8 +64,8 @@ def get_best_guess(entry, data): try: for is_starting, name, entry, entry_data, value in entry.decode(data): results.append((is_starting, entry)) - except ConstraintError, ex: - pass + except ConstraintError as caught: + ex = caught assert ex is not None return ex.entry, results diff --git a/bdec/test/testconstraints.py b/bdec/test/testconstraints.py index 2d08caf..2693bca 100644 --- a/bdec/test/testconstraints.py +++ b/bdec/test/testconstraints.py @@ -32,7 +32,7 @@ def test_minimum(self): try: min.check(field, 7, {}) raise Exception('Minimum constraint failed!') - except DecodeError, ex: + except DecodeError as ex: expected = "Expected ${a} >= 8; got 7" self.assertEqual(expected, str(ex)) @@ -44,7 +44,7 @@ def test_maximum(self): try: min.check(field, 9, {}) raise Exception('Maximum constraint failed!') - except DecodeError, ex: + except DecodeError as ex: expected = "Expected ${a} <= 8; got 9" self.assertEqual(expected, str(ex)) @@ -55,6 +55,6 @@ def test_equals(self): try: min.check(field, 'dog', {}) raise Exception('Maximum constraint failed!') - except DecodeError, ex: + except DecodeError as ex: expected = "Expected ${a} == cat; got dog" self.assertEqual(expected, str(ex)) diff --git a/bdec/test/testdata.py b/bdec/test/testdata.py index 6859b71..7ecb167 100644 --- a/bdec/test/testdata.py +++ b/bdec/test/testdata.py @@ -45,13 +45,14 @@ #!/usr/bin/env python import operator -import StringIO +import io import unittest import bdec import bdec.data as dt +from functools import reduce -class NonSeekable(StringIO.StringIO): +class NonSeekable(io.StringIO): def seek(self, *args): raise IOError() def tell(self): @@ -179,7 +180,7 @@ def test_bad_encoding(self): self.assertRaises(dt.BadTextEncodingError, dt.Data.text, data, "ascii") def test_file_buffer(self): - buffer = StringIO.StringIO() + buffer = io.StringIO() buffer.write('\x04abcd') data = dt.Data(buffer) self.assertEqual(4, int(data.pop(8))) @@ -189,15 +190,16 @@ def test_not_enough_data(self): # There was a bug in the size of available data we were popping; check # the sizes reported in the exception. data = dt.Data('abcd', 8, 48) + ex = None try: data.text('ascii') self.fail('NotEnoughDataError not thrown!') - except dt.NotEnoughDataError, ex: - pass + except dt.NotEnoughDataError as caught: + ex = caught self.assertEqual(40, ex.requested) self.assertEqual(24, ex.available) - def test_non_seeking_file(self): + def test_non_seeking_open(self): file = NonSeekable('abcdef') data = dt.Data(file) self.assertEqual('abc', data.pop(24).text('ascii')) @@ -210,7 +212,7 @@ def test_invalid_binary_text(self): try: dt.Data.from_binary_text('abcd') self.fail('Whoops, from_binary_test should have failed!') - except dt.InvalidBinaryTextError, ex: + except dt.InvalidBinaryTextError as ex: self.assertEqual("Invalid binary text 'abcd'", str(ex)) def test_large_add(self): @@ -221,7 +223,7 @@ def test_add_unknown_length(self): try: a = dt.Data('', 0, 4) + dt.Data('b') self.fail('Should have thrown NotEnoughDataError...') - except dt.NotEnoughDataError, ex: + except dt.NotEnoughDataError as ex: self.assertEqual('Asked for 4 bits, but only have 0 bits available!', str(ex)) def test_len_not_enough_data(self): diff --git a/bdec/test/testexpression.py b/bdec/test/testexpression.py index 1825817..42b8cae 100644 --- a/bdec/test/testexpression.py +++ b/bdec/test/testexpression.py @@ -33,12 +33,12 @@ def bool(text,context={}): # If the expression needs context to decode, it'll need to be able to # reference other entries, so create them here... children = [] - for name, value in context.items(): + for name, value in list(context.items()): children.append(sequence.Sequence(name, [], value=exp.Constant(value))) children.append(conditional) list(sequence.Sequence('', children).decode(Data())) return False - except DecodeError,ex: + except DecodeError as ex: return True class TestExpression(unittest.TestCase): @@ -79,12 +79,12 @@ def test_bimdas(self): def test_named_reference(self): a = exp.compile('${bob}') self.assertTrue(isinstance(a, exp.ValueResult)) - self.assertEquals('bob', a.name) + self.assertEqual('bob', a.name) def test_length_lookup(self): a = exp.compile('len{bob}') self.assertTrue(isinstance(a, exp.LengthResult)) - self.assertEquals('bob', a.name) + self.assertEqual('bob', a.name) def test_hex(self): self.assertEqual(5, eval("0x5")) diff --git a/bdec/test/testfield.py b/bdec/test/testfield.py index aca3e9a..2314897 100644 --- a/bdec/test/testfield.py +++ b/bdec/test/testfield.py @@ -52,6 +52,7 @@ from bdec.expression import ValueResult import bdec.data as dt import bdec.field as fld +from functools import reduce def query(context, entry, i, name): return context @@ -98,7 +99,7 @@ def test_binary_type(self): def test_binary_type_to_unicode(self): actual = self._get_decode_value("017a", 12, fld.Field.BINARY) - self.assertEqual(u"0000 00010111", unicode(actual)) + self.assertEqual("0000 00010111", str(actual)) def test_hexstring_type(self): actual = self._get_decode_value("017a", 12, fld.Field.HEX) @@ -133,20 +134,20 @@ def test_good_expected_data(self): def test_encode(self): field = fld.Field("bob", 8, format=fld.Field.INTEGER) result = field.encode(query, 0x3f) - self.assertEqual(0x3f, int(result.next())) + self.assertEqual(0x3f, int(next(result))) def test_encoded_size_matches_expected_size(self): # When we specify a size for a field, what we actually encode should match it. text = fld.Field("bob", 48, format=fld.Field.TEXT) - self.assertEqual("rabbit", text.encode(query, "rabbit").next().bytes()) + self.assertEqual("rabbit", next(text.encode(query, "rabbit")).bytes()) self.assertRaises(DataLengthError, list, text.encode(query, "boxfish")) binary = fld.Field("bob", 8, format=fld.Field.BINARY) - self.assertEqual("\x39", binary.encode(query, "00111001").next().bytes()) + self.assertEqual("\x39", next(binary.encode(query, "00111001")).bytes()) self.assertRaises(DataLengthError, list, binary.encode(query, "1011")) hex = fld.Field("bob", 8, format=fld.Field.HEX) - self.assertEqual("\xe7", hex.encode(query, "e7").next().bytes()) + self.assertEqual("\xe7", next(hex.encode(query, "e7")).bytes()) self.assertRaises(DataLengthError, list, hex.encode(query, "ecd")) def test_string_conversion(self): @@ -160,7 +161,7 @@ def test_bad_format_error(self): def test_encode_of_field_with_expected_value_fails_when_given_bad_data(self): field = fld.Field("bob", 8, constraints=[Equals(dt.Data('c'))]) - self.assertRaises(ConstraintError, field.encode(query, dt.Data("d")).next) + self.assertRaises(ConstraintError, field.encode(query, dt.Data("d")).__next__) def test_encode_of_field_with_expected_value_succeeds_with_missing_data(self): """ @@ -172,7 +173,7 @@ def test_encode_of_field_with_expected_value_succeeds_with_missing_data(self): field = fld.Field("bob", 8, constraints=[Equals(dt.Data("c"))]) def no_data_query(obj, entry, i, name): return '' - self.assertEqual("c", field.encode(no_data_query, None).next().bytes()) + self.assertEqual("c", next(field.encode(no_data_query, None)).bytes()) def test_range(self): field = fld.Field("bob", 8, min=8, max=15) @@ -185,7 +186,7 @@ def test_range(self): try: list(field.decode(dt.Data("\x07"))) self.fail("Exception not thrown!") - except fld.BadRangeError, ex: + except fld.BadRangeError as ex: text = str(ex) def test_range(self): diff --git a/bdec/test/testsequence.py b/bdec/test/testsequence.py index 9c73fbb..b41c49a 100644 --- a/bdec/test/testsequence.py +++ b/bdec/test/testsequence.py @@ -51,6 +51,7 @@ import bdec.expression as expr import bdec.field as fld import bdec.sequence as seq +from functools import reduce class TestSequence(unittest.TestCase): def test_simple_sequence(self): diff --git a/bdec/tools/compile.py b/bdec/tools/compile.py index 0d31b7c..2f14654 100644 --- a/bdec/tools/compile.py +++ b/bdec/tools/compile.py @@ -52,32 +52,32 @@ def usage(program): - print 'Compile bdec specifications into language specific decoders.' - print 'Usage:' - print ' %s [options] [spec_filename] ...' % program - print - print 'Arguments:' - print ' spec_filename -- The filename of the specification to be compiled.' - print - print 'Options:' - print ' -h, --help Print this help.' - print ' -d Directory to save the generated source code. Defaults' - print ' to %s.' % os.getcwd() - print ' --encoder Generate an encoder as well as a decoder.' - print ' --main= Specify the entry to be use as the default decoder.' - print ' --remove-unused Remove any entries that are not referenced from the' - print ' main entry.' - print ' --template= Set the template to compile. If there is a directory' - print ' with the specified name, it will be used as the' - print ' template directory. Otherwise it will use the internal' - print ' template with the specified name. If not specified a' - print ' C language decoder will be compiled.' - print ' -V Print the version of the bdec compiler.' + print('Compile bdec specifications into language specific decoders.') + print('Usage:') + print(' %s [options] [spec_filename] ...' % program) + print() + print('Arguments:') + print(' spec_filename -- The filename of the specification to be compiled.') + print() + print('Options:') + print(' -h, --help Print this help.') + print(' -d Directory to save the generated source code. Defaults') + print(' to %s.' % os.getcwd()) + print(' --encoder Generate an encoder as well as a decoder.') + print(' --main= Specify the entry to be use as the default decoder.') + print(' --remove-unused Remove any entries that are not referenced from the') + print(' main entry.') + print(' --template= Set the template to compile. If there is a directory') + print(' with the specified name, it will be used as the') + print(' template directory. Otherwise it will use the internal') + print(' template with the specified name. If not specified a') + print(' C language decoder will be compiled.') + print(' -V Print the version of the bdec compiler.') def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'd:hV', ['encoder', 'help', 'main=', 'remove-unused', 'template=']) - except getopt.GetoptError, ex: + except getopt.GetoptError as ex: sys.exit("%s.\nRun '%s -h' for correct usage." % (ex, sys.argv[0])) main_spec = None @@ -98,7 +98,7 @@ def main(): elif opt == '--main': main_spec = arg elif opt == '-V': - print bdec.__version__ + print(bdec.__version__) sys.exit(0) elif opt == '--remove-unused': should_remove_unused = True @@ -114,7 +114,7 @@ def main(): try: spec, common, lookup = load_specs([(s, None, None) for s in args], main_spec, should_remove_unused) - except bdec.spec.LoadError, ex: + except bdec.spec.LoadError as ex: sys.exit(str(ex)) if template_dir is None: diff --git a/bdec/tools/decode.py b/bdec/tools/decode.py index 455562e..47edc1a 100644 --- a/bdec/tools/decode.py +++ b/bdec/tools/decode.py @@ -55,24 +55,24 @@ from bdec.spec.xmlspec import dumps def usage(program): - print 'Decode standard input to xml given a bdec specification.' - print 'Usage:' - print ' %s [options] ' % program - print - print 'Arguments:' - print ' spec_filename -- The filename of the specification to be compiled.' - print - print 'Options:' - print ' -f Decode from filename instead of stdin.' - print ' -h, --help Print this help.' - print ' -l Log status messages.' - print ' --main= Specify the entry to be used as the decoder.' - print ' -q Quiet output. Only errors will be printed to stderr.' - print ' --remove-unused Remove any entries that are not referenced from the main' - print ' entry.' - print ' -S Print an xml representation of the specification.' - print ' --verbose Include hidden entries and raw data in the decoded output.' - print ' -V Print the version of the bdec compiler.' + print('Decode standard input to xml given a bdec specification.') + print('Usage:') + print(' %s [options] ' % program) + print() + print('Arguments:') + print(' spec_filename -- The filename of the specification to be compiled.') + print() + print('Options:') + print(' -f Decode from filename instead of stdin.') + print(' -h, --help Print this help.') + print(' -l Log status messages.') + print(' --main= Specify the entry to be used as the decoder.') + print(' -q Quiet output. Only errors will be printed to stderr.') + print(' --remove-unused Remove any entries that are not referenced from the main') + print(' entry.') + print(' -S Print an xml representation of the specification.') + print(' --verbose Include hidden entries and raw data in the decoded output.') + print(' -V Print the version of the bdec compiler.') def _parse_args(): verbose = 1 @@ -82,7 +82,7 @@ def _parse_args(): should_print_spec = False try: opts, args = getopt.getopt(sys.argv[1:], 'f:hlqSV', ['help', 'main=', 'remove-unused', 'verbose']) - except getopt.GetoptError, ex: + except getopt.GetoptError as ex: sys.exit("%s\nSee '%s -h' for correct usage." % (ex, sys.argv[0])) for opt, arg in opts: if opt == '-f': @@ -103,7 +103,7 @@ def _parse_args(): elif opt == '-S': should_print_spec = True elif opt == '-V': - print bdec.__version__ + print(bdec.__version__) sys.exit(0) else: assert 0, 'Unhandled option %s!' % opt @@ -118,11 +118,11 @@ def main(): main_spec, specs, binary, verbose, should_remove_unused, should_print_spec = _parse_args() try: decoder, common, lookup = load_specs([(s, None, None) for s in specs], main_spec, should_remove_unused) - except bdec.spec.LoadError, ex: + except bdec.spec.LoadError as ex: sys.exit(str(ex)) if should_print_spec: - print dumps(decoder, common) + print(dumps(decoder, common)) return data = dt.Data(binary) @@ -131,8 +131,8 @@ def main(): for item in decoder.decode(data): pass else: - xmlout.to_file(decoder.decode(data), sys.stdout, verbose=(verbose==2)) - except bdec.DecodeError, ex: + xmlout.to_open(decoder.decode(data), sys.stdout, verbose=(verbose==2)) + except bdec.DecodeError as ex: try: (filename, line_number, column_number) = lookup[ex.entry] except KeyError: @@ -140,7 +140,7 @@ def main(): # We include an extra new line, as the xml is unlikely to have finished # on a new line (issue164). - print + print() sys.exit("%s[%i]: %s" % (filename, line_number, str(ex))) try: diff --git a/bdec/tools/encode.py b/bdec/tools/encode.py index 4598756..54e3c9b 100644 --- a/bdec/tools/encode.py +++ b/bdec/tools/encode.py @@ -72,17 +72,17 @@ def main(): try: protocol, common, lookup = load_specs([(spec, None, None) for spec in args], options.main, options.remove_unused) - except bdec.spec.LoadError, ex: + except bdec.spec.LoadError as ex: sys.exit(str(ex)) if options.filename: - xml = file(options.filename, 'rb').read() + xml = open(options.filename, 'rb').read() else: xml = sys.stdin.read() try: binary = xmlout.encode(protocol, xml).bytes() - except bdec.DecodeError, ex: + except bdec.DecodeError as ex: try: (filename, line_number, column_number) = lookup[ex.entry] except KeyError: diff --git a/docs/source/conf.py b/docs/source/conf.py index d406c63..6635795 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,7 +3,7 @@ # bdec documentation build configuration file, created by # sphinx-quickstart on Thu Apr 3 23:01:59 2008. # -# This file is execfile()d with the current directory set to its containing dir. +# This file is executed with the current directory set to its containing dir. # # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). diff --git a/regression/test.py b/regression/test.py index f9c4959..ebc1395 100644 --- a/regression/test.py +++ b/regression/test.py @@ -1,6 +1,6 @@ from collections import defaultdict -from ConfigParser import ConfigParser +from configparser import ConfigParser import glob import os.path import re @@ -45,7 +45,7 @@ def __str__(self): else: assert os.path.splitext(filename) not in ['swp'], \ "Unknown regression file '%s'!" % path - for name, test in tests.items(): + for name, test in list(tests.items()): entry = None if '-' in name: assert test.filename is None @@ -63,12 +63,14 @@ def _create_test_cases(): return -- A dictionary of name to test class. """ - config = ConfigParser() + config = ConfigParser(strict=False) config.read(os.path.join(os.path.dirname(__file__), 'fixme.cfg')) result = {} regression_dir = os.path.dirname(__file__) for name in os.listdir(regression_dir): + if name.startswith('__'): + continue path = os.path.join(regression_dir, name) if os.path.isdir(path): result.update(create_classes(name, _find_regression_tests(name, path), config)) diff --git a/specs/msword/python/decode.py b/specs/msword/python/decode.py index 82f26b6..0a1ac8e 100755 --- a/specs/msword/python/decode.py +++ b/specs/msword/python/decode.py @@ -15,12 +15,12 @@ data = Data(open(sys.argv[1], 'rb')) spec, common, lookup = load_msword_spec() try: - to_file(decode(data, common), sys.stdout) - except DecodeError, ex: + to_open(decode(data, common), sys.stdout) + except DecodeError as ex: try: (filename, line_number, column_number) = lookup[ex.entry] except KeyError: (filename, line_number, column_number) = ('unknown', 0, 0) - print + print() sys.exit("%s[%i]: %s" % (filename, line_number, str(ex))) diff --git a/specs/test/__init__.py b/specs/test/__init__.py index c54e85a..ca59f64 100644 --- a/specs/test/__init__.py +++ b/specs/test/__init__.py @@ -4,7 +4,7 @@ It attempts to load each example specification, and will attempt to decode all of the supplied sample files. """ -from ConfigParser import ConfigParser +from configparser import ConfigParser import glob import os.path @@ -17,7 +17,7 @@ def _create_decode_classes(): testdir = os.path.dirname(__file__) path = os.path.join(testdir, '..') result = {} - config = ConfigParser() + config = ConfigParser(strict=False) config.read(os.path.join(os.path.dirname(__file__), 'fixme.cfg')) for filename in glob.glob(os.path.join(path, '*.xml')): # Create a test class per specification, which each sample file being diff --git a/tools/release.py b/tools/release.py index a079760..195e9dc 100755 --- a/tools/release.py +++ b/tools/release.py @@ -4,7 +4,7 @@ import datetime import getpass -import httplib +import http.client import json import os.path import re @@ -19,26 +19,26 @@ import bdec def usage(): - print "A script to simplify deploying new releases." - print - print "Usage: %s" % sys.argv[0] - print - print "This script will;" - print " 1. Check that all source files have copyright statements." - print " 2. Get the changelog from the README." - print " 3. If the changelog version matches the bdec version, will prompt " - print " to update the website documentation." - print " 4. If the changelog version doesn't match the bdec version, it will;" - print " a) Update the bdec/__init__.py version." - print " b) Insert today's date into the README changelog." - print " c) Update the website bdec documentation." - print " d) Create a new tarball in the website 'files' folder." - print " e) Prompt if the user wants to upload and notify others of this" - print " release. If so, all changes will be automatically committed," - print " the website will have the changes uploaded, and a new tag will" - print " be created in source control." - print " f) Prompt if user wants freshmeat & pypi to be notified. If so," - print " README changlog will be used." + print("A script to simplify deploying new releases.") + print() + print("Usage: %s" % sys.argv[0]) + print() + print("This script will;") + print(" 1. Check that all source files have copyright statements.") + print(" 2. Get the changelog from the README.") + print(" 3. If the changelog version matches the bdec version, will prompt ") + print(" to update the website documentation.") + print(" 4. If the changelog version doesn't match the bdec version, it will;") + print(" a) Update the bdec/__init__.py version.") + print(" b) Insert today's date into the README changelog.") + print(" c) Update the website bdec documentation.") + print(" d) Create a new tarball in the website 'files' folder.") + print(" e) Prompt if the user wants to upload and notify others of this") + print(" release. If so, all changes will be automatically committed,") + print(" the website will have the changes uploaded, and a new tag will") + print(" be created in source control.") + print(" f) Prompt if user wants freshmeat & pypi to be notified. If so,") + print(" README changlog will be used.") _README = os.path.join(root_path, 'README.rst') _CHANGELOG = os.path.join(root_path, 'CHANGELOG') @@ -48,7 +48,7 @@ def usage(): freshmeat_pass = os.path.join(website_dir, 'freshmeat.txt') def _check_copyright_statements(subdirs): - print 'Checking for copyright notices in source files...' + print('Checking for copyright notices in source files...') is_missing_copyright = False for subdir in subdirs: for dir, subdirs, filenames in os.walk(os.path.join(root_path, subdir)): @@ -60,13 +60,13 @@ def _check_copyright_statements(subdirs): if 'Copyright' in line: break else: - print "'%s' doesn't include copyright information!" % filename + print("'%s' doesn't include copyright information!" % filename) is_missing_copyright = True if is_missing_copyright: sys.exit('Copyright issues must be resolved.') def _read_changelog(): - readme = file(_CHANGELOG, 'r') + readme = open(_CHANGELOG, 'r') contents = readme.read() readme.close() return contents @@ -155,7 +155,7 @@ def _create_changelog_html(): _generate_html(contents) os.rename('index.html', 'changelog.html') -def _create_index_file(version): +def _create_index_open(version): # Create a temporary file that contains a modified readme version, date, notes = get_changelog()[0] notes = '\n '.join(notes.splitlines()) @@ -173,12 +173,12 @@ def _create_index_file(version): _generate_html(contents) def _create_pdf(version, target): - print 'Generating pdf documentation...' + print('Generating pdf documentation...') os.chdir(os.path.join(root_path, 'docs')) command = 'PYTHONPATH=%s sphinx-build -c tempdir -b latex -a source tempdir' % (root_path) if not os.path.exists('tempdir'): os.mkdir('tempdir') - conf = file('tempdir/conf.py', 'w') + conf = open('tempdir/conf.py', 'w') conf.write('''latex_documents=[('index', 'bdec-%s.tex', 'Bdec binary specifications', 'Henry Ludemann', 'manual', True)] release='%s' ''' % (version, version)) conf.close() @@ -190,19 +190,19 @@ def _create_pdf(version, target): shutil.rmtree('tempdir') def generate_website_files(version): - print 'Updating project index...' + print('Updating project index...') pdf_file = os.path.join(project_dir, 'files', 'bdec-%s.pdf' % version) - if not os.path.exists(pdf_file) or raw_input("Pdf documentation '%s' exists! Regenerate? [y]" % pdf_file) in ('', 'Y', 'y'): + if not os.path.exists(pdf_file) or input("Pdf documentation '%s' exists! Regenerate? [y]" % pdf_file) in ('', 'Y', 'y'): _create_pdf(version, pdf_file) else: - print 'Not regenerating pdf...' + print('Not regenerating pdf...') os.chdir(project_dir) _create_changelog_html() - _create_index_file(version) + _create_index_open(version) - print 'Updating project documentation...' + print('Updating project documentation...') os.chdir(os.path.join(root_path, 'docs')) html_doc_dir = os.path.join(project_dir, 'docs') command = 'PYTHONPATH=%s sphinx-build -a source tempdir' % (root_path) @@ -225,9 +225,9 @@ def create_release_tarball(version): os.chdir(root_path) destination = os.path.join(project_dir, 'files', 'bdec-%s.tar.gz' % version) if os.path.exists(destination): - text = raw_input("Archive '%s' exists! Overwrite? [n]" % destination) + text = input("Archive '%s' exists! Overwrite? [n]" % destination) if text != 'y': - print 'Not updated archiving...' + print('Not updated archiving...') return command = 'git archive --format=tar --prefix=bdec-%s/ HEAD | gzip > %s' % (version, destination) @@ -246,25 +246,25 @@ def tag_changes(version): if git.wait() != 0: sys.exit('Failed to read git tags!') if tag not in tags: - text = raw_input("Create new tag '%s'? [y]" % tag) + text = input("Create new tag '%s'? [y]" % tag) else: - text = raw_input("Tag '%s' exists! Overwrite? [y]" % tag) + text = input("Tag '%s' exists! Overwrite? [y]" % tag) if text.strip() and text != 'y': - print 'Not tagged.' + print('Not tagged.') elif os.system('git tag -f "%s"' % tag) != 0: sys.exit('Failed to tag!') def _edit_message(message): filename = 'message.edit' - data = file(filename, 'w') + data = open(filename, 'w') data.write(message) data.close() if os.system('vi %s' % filename) != 0: sys.exit('Stopping due to edit message failure!') - data = file(filename, 'r') + data = open(filename, 'r') message = data.read() data.close() os.remove(filename) @@ -274,13 +274,13 @@ def commit_website(version): os.chdir(project_dir) if os.system('git diff HEAD') != 0: sys.exit('Stopped after reviewing changes.') - text = raw_input('Commit website changes? [y]') + text = input('Commit website changes? [y]') if text and text != 'y': - print 'Not committed.' + print('Not committed.') return False # Commit the website changes - data = file('.commitmsg', 'w') + data = open('.commitmsg', 'w') data.write('Updated bdec project to version %s' % version) data.close() if os.system('git commit --file .commitmsg --edit') != 0: @@ -308,10 +308,10 @@ def send_email(version, changelog): os.remove('.emailmsg') try: - user = raw_input('Enter gmail username:') + user = input('Enter gmail username:') password = getpass.getpass() - print 'Sending email...' + print('Sending email...') smtp = smtplib.SMTP('smtp.gmail.com', 587) smtp.ehlo() smtp.starttls() @@ -319,18 +319,18 @@ def send_email(version, changelog): smtp.login(user, password) smtp.sendmail('henry@protocollogic.com', to_addr, message) smtp.quit() - except smtplib.SMTPAuthenticationError, ex: - print 'Authenticion error!', ex + except smtplib.SMTPAuthenticationError as ex: + print('Authenticion error!', ex) def _get_freshmeat_auth_code(): if os.path.exists(freshmeat_pass): - print 'Reading freshmeat credentials from', freshmeat_pass - data = file(freshmeat_pass, 'r') + print('Reading freshmeat credentials from', freshmeat_pass) + data = open(freshmeat_pass, 'r') result = data.read() data.close() else: - result = raw_input('Enter freshmeat auth token (from user page):') - data = file(freshmeat_pass, 'w') + result = input('Enter freshmeat auth token (from user page):') + data = open(freshmeat_pass, 'w') data.write(result) data.close() result = result.strip() @@ -351,7 +351,7 @@ def _get_tags(connection, freshmeat_auth): data = response.read() conn.close() if response.status >= 400: - print response.status, response.reason + print(response.status, response.reason) sys.exit('Failed to query freshmeat tags! (%s)' % data) tags = set() for release in json.loads(data): @@ -361,26 +361,26 @@ def _get_tags(connection, freshmeat_auth): tags = dict(enumerate(tags)) # Ask the user what tags they want - print 'Tags are:' - for id, text in tags.iteritems(): - print '%i - %s' % (id, text) + print('Tags are:') + for id, text in tags.items(): + print('%i - %s' % (id, text)) ids = [1,3] - text = raw_input('What is the release focus? [1,3] ') + text = input('What is the release focus? [1,3] ') if text.strip(): ids = [int(id.strip()) for id in text.split(',')] return ', '.join(tags[id] for id in ids) def notify(version, changelog, freshmeat_auth=_get_freshmeat_auth_code, - connection=httplib.HTTPConnection, system=os.system, confirm=raw_input, + connection=http.client.HTTPConnection, system=os.system, confirm=input, tag_list=_get_tags): # This is a fresmeat limit MAX_CHARS = 600 short_message = shorten_changelog(changelog) while len(short_message) > 600: - print 'Changelog is too long (must be less then %i characters, is %i)' % (MAX_CHARS, len(short_message)) - text = raw_input('Edit changelog for submission? [y]') + print('Changelog is too long (must be less then %i characters, is %i)' % (MAX_CHARS, len(short_message))) + text = input('Edit changelog for submission? [y]') if text and text != 'y': - print 'Servers not notified of release.' + print('Servers not notified of release.') return short_message = _edit_message(short_message) @@ -388,12 +388,12 @@ def notify(version, changelog, freshmeat_auth=_get_freshmeat_auth_code, # Notify freshmeat if confirm('Should freshmeat be notified? [y]') in ['', 'y', 'Y']: release = { - 'auth_code': freshmeat_auth(), 'release':{ 'tag_list':tag_list(connection, freshmeat_auth), 'version':str(version), 'changelog':short_message }, + 'auth_code': freshmeat_auth(), } headers = {"Content-type": "application/json"} conn = connection("freshmeat.net") @@ -402,12 +402,12 @@ def notify(version, changelog, freshmeat_auth=_get_freshmeat_auth_code, data = response.read() conn.close() if response.status >= 400: - print response.status, response.reason + print(response.status, response.reason) sys.exit('Failed to submit to freshmeat! (%s)' % data) - print "Created release; %s %s\n%s" % (response.status, response.reason, data) + print("Created release; %s %s\n%s" % (response.status, response.reason, data)) else: - print 'Not notifying freshmeat.' + print('Not notifying freshmeat.') # Notify the python package index if confirm('Should pypi be notified? [y]') in ['', 'y', 'Y']: @@ -416,19 +416,19 @@ def notify(version, changelog, freshmeat_auth=_get_freshmeat_auth_code, if system(command) != 0: sys.exit('Failed to update python package index!') else: - print 'Not notifying pypi.' + print('Not notifying pypi.') if confirm('Send an email to the bdec mailing list? [y]') in ['', 'y', 'Y']: send_email(version, changelog) def upload(): - print "Uploading to the server..." + print("Uploading to the server...") while 1: os.chdir(project_dir) command = "../../google_appengine/appcfg.py update ../" if os.system(command) == 0: break - text = raw_input('Failed to upload to the server! Try again? [y]') + text = input('Failed to upload to the server! Try again? [y]') if text.strip() and text != 'y': sys.exit('Not uploaded.') @@ -436,7 +436,7 @@ def has_modifications(): process = subprocess.Popen(['git', 'status', '--porcelain'], stdout=subprocess.PIPE) output = process.stdout.read() - print output + print(output) assert process.wait() == 0, 'Failed to query git status.' return len(output.strip()) != 0 @@ -451,16 +451,16 @@ def main(): if version != bdec.__version__: sys.exit("Version mismatch! Changelog version is '%s', bdec version is '%s'" % (version, bdec.__version__)) - print "Preparing new bdec release", version - print "Changes are;" - print shorten_changelog(changelog) - print + print("Preparing new bdec release", version) + print("Changes are;") + print(shorten_changelog(changelog)) + print() generate_website_files(version) create_release_tarball(version) os.chdir(root_path) - if has_modifications() and raw_input('Source tree has changes! Stop? [y]') != 'n': + if has_modifications() and input('Source tree has changes! Stop? [y]') != 'n': sys.exit('Stopping due to changes in the source tree.') os.chdir(root_path) diff --git a/tools/test/testrelease.py b/tools/test/testrelease.py index aa122cd..f41a1c8 100644 --- a/tools/test/testrelease.py +++ b/tools/test/testrelease.py @@ -1,6 +1,6 @@ import os.path -import StringIO +import io import unittest import bdec @@ -52,7 +52,7 @@ def __init__(self, domain): def request(self, actual, path, json, headers): self.json = json def getresponse(self): - response = StringIO.StringIO() + response = io.StringIO() response.status = 200 response.reason = '' return response