diff --git a/ibflex/Types.py b/ibflex/Types.py index 6ab7555..fb8f4b2 100644 --- a/ibflex/Types.py +++ b/ibflex/Types.py @@ -479,6 +479,7 @@ class EquitySummaryByReportDateInBase(FlexElement): marginFinancingChargeAccrualsShort: Optional[decimal.Decimal] = None cryptoLong: Optional[decimal.Decimal] = None cryptoShort: Optional[decimal.Decimal] = None + liteSurchargeAccruals: Optional[decimal.Decimal] = None @dataclass(frozen=True) @@ -709,6 +710,8 @@ class CashReportCurrency(FlexElement): salesTaxYTD: Optional[decimal.Decimal] = None salesTaxPaxos: Optional[decimal.Decimal] = None otherIncome: Optional[decimal.Decimal] = None + otherIncomeMTD: Optional[decimal.Decimal] = None + otherIncomeYTD: Optional[decimal.Decimal] = None otherIncomeSec: Optional[decimal.Decimal] = None otherIncomeCom: Optional[decimal.Decimal] = None otherFeesMTD: Optional[decimal.Decimal] = None @@ -1144,7 +1147,8 @@ class Trade(FlexElement): subCategory: Optional[str] = None issuerCountryCode: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None + positionActionID: Optional[str] = None @dataclass(frozen=True) @@ -1296,7 +1300,8 @@ class Lot(FlexElement): issuerCountryCode: Optional[str] = None relatedTradeID: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None + positionActionID: Optional[str] = None @dataclass(frozen=True) @@ -1385,6 +1390,9 @@ class SymbolSummary(FlexElement): tradeID: Optional[str] = None orderID: Optional[decimal.Decimal] = None execID: Optional[str] = None + ibExecID: Optional[str] = None + extExecID: Optional[str] = None + exchOrderId: Optional[str] = None brokerageOrderID: Optional[str] = None orderReference: Optional[str] = None volatilityOrderLink: Optional[str] = None @@ -1392,12 +1400,19 @@ class SymbolSummary(FlexElement): origTradePrice: Optional[decimal.Decimal] = None origTradeDate: Optional[datetime.date] = None origTradeID: Optional[str] = None + transactionID: Optional[str] = None # Despite the name, `orderTime` actually contains date/time data. orderTime: Optional[datetime.datetime] = None + openDateTime: Optional[datetime.datetime] = None + holdingPeriodDateTime: Optional[datetime.datetime] = None dateTime: Optional[datetime.datetime] = None reportDate: Optional[datetime.date] = None settleDate: Optional[datetime.date] = None + settleDateTarget: Optional[datetime.date] = None # expected date of ownership transfer + taxes: Optional[decimal.Decimal] = None tradeDate: Optional[datetime.date] = None + tradePrice: Optional[decimal.Decimal] = None + tradeMoney: Optional[decimal.Decimal] = None # TradeMoney = Proceeds + Fees + Commissions exchange: Optional[str] = None buySell: Optional[enums.BuySell] = None quantity: Optional[decimal.Decimal] = None @@ -1427,6 +1442,40 @@ class SymbolSummary(FlexElement): relatedTradeID: Optional[str] = None origTransactionID: Optional[str] = None relatedTransactionID: Optional[str] = None + positionActionID: Optional[str] = None + changeInPrice: Optional[decimal.Decimal] = None + changeInQuantity: Optional[decimal.Decimal] = None + closePrice: Optional[decimal.Decimal] = None + commodityType: Optional[str] = None + cost: Optional[decimal.Decimal] = None + deliveryType: Optional[str] = None + exchOrderId: Optional[str] = None + extExecID: Optional[str] = None + fifoPnlRealized: Optional[decimal.Decimal] = None + fineness: Optional[decimal.Decimal] = None + holdingPeriodDateTime: Optional[datetime.datetime] = None + ibCommission: Optional[decimal.Decimal] = None + ibCommissionCurrency: Optional[str] = None + ibExecID: Optional[str] = None + ibOrderID: Optional[str] = None + initialInvestment: Optional[bool] = None + mtmPnl: Optional[decimal.Decimal] = None + netCash: Optional[decimal.Decimal] = None + netCashInBase: Optional[decimal.Decimal] = None + notes: Optional[str] = None + openCloseIndicator: Optional[enums.OpenClose] = None + openDateTime: Optional[datetime.datetime] = None + origOrderID: Optional[str] = None + rtn: Optional[str] = None + serialNumber: Optional[str] = None + settleDateTarget: Optional[datetime.date] = None + taxes: Optional[decimal.Decimal] = None + tradeMoney: Optional[decimal.Decimal] = None + tradePrice: Optional[decimal.Decimal] = None + transactionID: Optional[str] = None + weight: Optional[str] = None + whenRealized: Optional[datetime.datetime] = None + whenReopened: Optional[datetime.datetime] = None @dataclass(frozen=True) @@ -1535,7 +1584,8 @@ class AssetSummary(FlexElement): origTransactionID: Optional[str] = None relatedTransactionID: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None + positionActionID: Optional[str] = None @dataclass(frozen=True) @@ -1638,12 +1688,13 @@ class Order(FlexElement): origTransactionID: Optional[str] = None relatedTransactionID: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None serialNumber: Optional[str] = None deliveryType: Optional[str] = None commodityType: Optional[str] = None fineness: Optional[decimal.Decimal] = None weight: Optional[str] = None + positionActionID: Optional[str] = None @dataclass(frozen=True) @@ -1796,7 +1847,7 @@ class OptionEAE(FlexElement): origTransactionID: Optional[str] = None relatedTransactionID: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None serialNumber: Optional[str] = None deliveryType: Optional[str] = None commodityType: Optional[str] = None @@ -2122,7 +2173,12 @@ class Transfer(FlexElement): commodityType: Optional[str] = None fineness: Optional[decimal.Decimal] = None weight: Optional[str] = None - + figi: Optional[str] = None + settleDate: Optional[datetime.date] = None + issuerCountryCode: Optional[str] = None + levelOfDetail: Optional[str] = None + positionInstructionID: Optional[str] = None + positionInstructionSetID: Optional[str] = None @dataclass(frozen=True) class UnsettledTransfer(FlexElement): @@ -2245,6 +2301,11 @@ class CorporateAction(FlexElement): commodityType: Optional[str] = None fineness: Optional[decimal.Decimal] = None weight: Optional[str] = None + figi: Optional[str] = None + issuerCountryCode: Optional[str] = None + costBasis: Optional[decimal.Decimal] = None + + @dataclass(frozen=True) @@ -2480,7 +2541,7 @@ class SecurityInfo(FlexElement): origTransactionID: Optional[str] = None relatedTransactionID: Optional[str] = None rtn: Optional[str] = None - initialInvestment: Optional[decimal.Decimal] = None + initialInvestment: Optional[bool] = None serialNumber: Optional[str] = None deliveryType: Optional[str] = None commodityType: Optional[str] = None diff --git a/ibflex/__init__.py b/ibflex/__init__.py index 684a0d7..b6dcd6b 100644 --- a/ibflex/__init__.py +++ b/ibflex/__init__.py @@ -4,6 +4,8 @@ from .Types import * from . import parser from .parser import parse +from .parser import enable_unknown_attribute_tolerance +from .parser import disable_unknown_attribute_tolerance from . import utils from . import client diff --git a/ibflex/client.py b/ibflex/client.py index f203e8b..041f53a 100755 --- a/ibflex/client.py +++ b/ibflex/client.py @@ -136,8 +136,6 @@ def request_statement( """First part of the 2-step download process. """ url = url or REQUEST_URL - ### AKE FIX - url = 'https://ndcdyn.interactivebrokers.com/portal.flexweb/api/v1/flexQuery' response = submit_request(url, token, query=query_id) stmt_access = parse_stmt_response(response) if isinstance(stmt_access, StatementError): diff --git a/ibflex/parser.py b/ibflex/parser.py index af6b270..f582187 100755 --- a/ibflex/parser.py +++ b/ibflex/parser.py @@ -21,6 +21,56 @@ class FlexParserError(Exception): """ Error experienced while parsing Flex XML data. """ +############################################################################### +# UNKNOWN ATTRIBUTE TOLERANCE +############################################################################### +_UNKNOWN_ATTRIBUTE_TOLERANCE = False + + +def enable_unknown_attribute_tolerance(): + """Enable tolerance for unknown XML attributes in IB Flex data. + + When enabled, unknown attributes and element types in the XML data are + silently ignored instead of raising FlexParserError. This is useful when + Interactive Brokers adds new fields to their exports that are not yet + defined in the Types module. + + This function is only available in the forked version of ibflex. + Attempting to call it on the original upstream package will raise + AttributeError (the method does not exist there), providing a clear + signal that the feature is not supported. + + Default: off (strict mode - unknown attributes raise errors). + + See also: disable_unknown_attribute_tolerance() + """ + global _UNKNOWN_ATTRIBUTE_TOLERANCE + _UNKNOWN_ATTRIBUTE_TOLERANCE = True + + +def disable_unknown_attribute_tolerance(): + """Disable tolerance for unknown XML attributes (the default behavior). + + After calling this, unknown attributes in XML data will raise + FlexParserError as usual. + + See also: enable_unknown_attribute_tolerance() + """ + global _UNKNOWN_ATTRIBUTE_TOLERANCE + _UNKNOWN_ATTRIBUTE_TOLERANCE = False + + +def _get_known_attributes(Class): + """Get all known attribute names for a FlexElement subclass, + including inherited attributes from base classes. + """ + attrs = set() + for klass in Class.__mro__: + if hasattr(klass, '__annotations__'): + attrs.update(klass.__annotations__.keys()) + return attrs + + DataType = Union[ None, str, int, bool, decimal.Decimal, datetime.date, datetime.time, datetime.datetime, enums.EnumType, Tuple[str, ...], Tuple[enums.Code, ...] @@ -99,22 +149,40 @@ def parse_element_container(elem: ET.Element) -> Tuple[Types.FlexElement, ...]: return tuple(itertools.chain.from_iterable(fxlots)) instances = tuple(parse_data_element(child) for child in elem) + if _UNKNOWN_ATTRIBUTE_TOLERANCE: + instances = tuple(inst for inst in instances if inst is not None) return instances def parse_data_element( elem: ET.Element -) -> Types.FlexElement: +) -> Optional[Types.FlexElement]: """Parse an XML data element into a Types.FlexElement subclass instance. + + Returns None if unknown_attribute_tolerance is enabled and the element + type is not recognized. """ # Look up XML element's matching FlexElement subclass in ibflex.Types. - Class = getattr(Types, elem.tag) + try: + Class = getattr(Types, elem.tag) + except AttributeError: + if _UNKNOWN_ATTRIBUTE_TOLERANCE: + return None + raise + + # When tolerance is enabled, pre-compute known attributes and filter + known = _get_known_attributes(Class) if _UNKNOWN_ATTRIBUTE_TOLERANCE else None # Parse element attributes + if known is not None: + attrib_items = [(k, v) for k, v in elem.attrib.items() if k in known] + else: + attrib_items = list(elem.attrib.items()) + try: attrs = dict( parse_element_attr(Class, k, v) - for k, v in elem.attrib.items() + for k, v in attrib_items ) except KeyError as exc: msg = f"{Class.__name__} has no attribute " + str(exc) @@ -125,6 +193,12 @@ def parse_data_element( contained_elements = {child.tag: parse_element(child) for child in elem} if contained_elements: assert elem.tag in ("FlexQueryResponse", "FlexStatement") + if _UNKNOWN_ATTRIBUTE_TOLERANCE: + # Filter out unknown or unparseable contained elements + contained_elements = { + k: v for k, v in contained_elements.items() + if k in known and v is not None + } attrs.update(contained_elements) try: @@ -170,9 +244,11 @@ def parse_element_attr( # INPUT VALUE PREP FUNCTIONS FOR DATA CONVERTERS # These are just implementation details for converters and don't need testing. ############################################################################### -def prep_date(value: str) -> Tuple[int, int, int]: +def prep_date(value: str) -> Optional[Tuple[int, int, int]]: """Returns a tuple of (year, month, day). """ + if value == "MULTI": + return None # Summaries have MULTI as date value. date_format = DATE_FORMATS[len(value)][value.count('/')] return datetime.datetime.strptime(value, date_format).timetuple()[:3] @@ -184,9 +260,11 @@ def prep_time(value: str) -> Tuple[int, int, int]: return datetime.datetime.strptime(value, time_format).timetuple()[3:6] -def prep_datetime(value: str) -> Tuple[int, ...]: +def prep_datetime(value: str) -> Optional[Tuple[int, ...]]: """Returns a tuple of (year, month, day, hour, minute, second). """ + if value == "MULTI": + return None # Summaries have MULTI as date value. # HACK - some old data has ", " separator instead of ",". value = value.replace(", ", ",") @@ -328,8 +406,8 @@ def optional_convert(value): convert_string = make_optional(make_converter(str, prep=utils.identity_func)) convert_int = make_converter(int, prep=utils.identity_func) -# IB sends "Y"/"N" for True/False -convert_bool = make_converter(bool, prep=lambda x: {"Y": True, "N": False}[x]) +# IB sends "Y"/"N" or "Yes"/"No" for True/False +convert_bool = make_converter(bool, prep=lambda x: {"Y": True, "N": False, "Yes": True, "No": False}[x]) # IB sends numeric data with place delimiters (commas) convert_decimal = make_converter( decimal.Decimal, @@ -463,6 +541,7 @@ def convert_enum(Type, value): "CNH", # RMB traded in HK "BASE_SUMMARY", # Fake currency code used in IB NAV/Performance reports "", # Lot element allows blank currency ?! + "RUS", # Russian-related currency code used by IBKR ) diff --git a/tests/test_parser.py b/tests/test_parser.py index 92fab3f..26b20cc 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -333,9 +333,11 @@ def testConvertInt(self): parser.convert_int("") def testConvertBool(self): - """ Legal boolean values are 'Y'/'N' """ + """ Legal boolean values are 'Y'/'N' or 'Yes'/'No' """ self.assertEqual(parser.convert_bool("Y"), True) self.assertEqual(parser.convert_bool("N"), False) + self.assertEqual(parser.convert_bool("Yes"), True) + self.assertEqual(parser.convert_bool("No"), False) # Empty string raises FlexParserError. with self.assertRaises(parser.FlexParserError): @@ -536,5 +538,239 @@ def testMakeOptional(self): ) +class UnknownAttributeToleranceTestCase(unittest.TestCase): + """Tests for the unknown attribute tolerance feature. + + This feature allows the parser to silently ignore unknown XML attributes + and element types, which is useful when Interactive Brokers adds new fields + to their exports. + """ + + def setUp(self): + """Ensure tolerance is off before each test.""" + parser.disable_unknown_attribute_tolerance() + + def tearDown(self): + """Ensure tolerance is off after each test.""" + parser.disable_unknown_attribute_tolerance() + + def test_tolerance_default_off(self): + """Tolerance is off by default.""" + self.assertFalse(parser._UNKNOWN_ATTRIBUTE_TOLERANCE) + + def test_enable_disable(self): + """enable/disable functions toggle the flag correctly.""" + self.assertFalse(parser._UNKNOWN_ATTRIBUTE_TOLERANCE) + parser.enable_unknown_attribute_tolerance() + self.assertTrue(parser._UNKNOWN_ATTRIBUTE_TOLERANCE) + parser.disable_unknown_attribute_tolerance() + self.assertFalse(parser._UNKNOWN_ATTRIBUTE_TOLERANCE) + + def test_tolerance_functions_accessible_from_package(self): + """The tolerance functions are accessible from the top-level ibflex package. + + On the original upstream ibflex package these functions do not exist, + so calling ibflex.enable_unknown_attribute_tolerance() would raise + AttributeError, providing a clear version guard. + """ + import ibflex + self.assertTrue(hasattr(ibflex, 'enable_unknown_attribute_tolerance')) + self.assertTrue(hasattr(ibflex, 'disable_unknown_attribute_tolerance')) + # Verify they are callable + self.assertTrue(callable(ibflex.enable_unknown_attribute_tolerance)) + self.assertTrue(callable(ibflex.disable_unknown_attribute_tolerance)) + + def test_unknown_attr_raises_without_tolerance(self): + """Without tolerance, unknown XML attributes raise FlexParserError.""" + # AccountInformation with an unknown attribute "newIBField" + elem = ET.fromstring( + '' + ) + with self.assertRaises(parser.FlexParserError): + parser.parse_data_element(elem) + + def test_unknown_attr_ignored_with_tolerance(self): + """With tolerance enabled, unknown XML attributes are silently ignored.""" + parser.enable_unknown_attribute_tolerance() + + elem = ET.fromstring( + '' + ) + instance = parser.parse_data_element(elem) + self.assertIsInstance(instance, Types.AccountInformation) + self.assertEqual(instance.accountId, "U123456") + self.assertEqual(instance.currency, "USD") + # Unknown attributes are not present on the parsed object + self.assertFalse(hasattr(instance, 'newIBField')) + self.assertFalse(hasattr(instance, 'anotherNewField')) + + def test_known_attrs_still_parsed_with_tolerance(self): + """With tolerance, known attributes are still correctly parsed.""" + parser.enable_unknown_attribute_tolerance() + + elem = ET.fromstring( + '' + ) + instance = parser.parse_data_element(elem) + self.assertIsInstance(instance, Types.AccountInformation) + self.assertEqual(instance.accountId, "U123456") + self.assertEqual(instance.acctAlias, "test") + self.assertEqual(instance.currency, "USD") + self.assertEqual(instance.name, "Test User") + import datetime + self.assertEqual(instance.dateOpened, datetime.date(2020, 1, 15)) + + def test_disable_restores_strict_behavior(self): + """After disabling tolerance, unknown attributes raise errors again.""" + parser.enable_unknown_attribute_tolerance() + + elem = ET.fromstring( + '' + ) + # Should succeed with tolerance on + instance = parser.parse_data_element(elem) + self.assertIsInstance(instance, Types.AccountInformation) + + parser.disable_unknown_attribute_tolerance() + + # Should fail with tolerance off + with self.assertRaises(parser.FlexParserError): + parser.parse_data_element(elem) + + def test_unknown_element_type_raises_without_tolerance(self): + """Without tolerance, unknown XML element types raise AttributeError.""" + elem = ET.fromstring('') + with self.assertRaises(AttributeError): + parser.parse_data_element(elem) + + def test_unknown_element_type_returns_none_with_tolerance(self): + """With tolerance, unknown element types return None.""" + parser.enable_unknown_attribute_tolerance() + + elem = ET.fromstring('') + result = parser.parse_data_element(elem) + self.assertIsNone(result) + + def test_unknown_elements_filtered_in_container(self): + """With tolerance, unknown element types are filtered out of containers.""" + parser.enable_unknown_attribute_tolerance() + + container = ET.Element("Trades") + # Add a known Trade element + ET.SubElement(container, "Trade", attrib={ + "accountId": "U123456", + "currency": "USD", + "fxRateToBase": "1", + "assetCategory": "STK", + "symbol": "AAPL", + "description": "APPLE INC", + "conid": "265598", + "tradeID": "123", + "reportDate": "2020-01-15", + "tradeDate": "2020-01-15", + "quantity": "100", + "tradePrice": "150.00", + "tradeMoney": "15000.00", + "proceeds": "-15000.00", + "taxes": "0", + "ibCommission": "-1.00", + "ibCommissionCurrency": "USD", + "netCash": "-15001.00", + "buySell": "BUY", + }) + # Add an unknown element type + ET.SubElement(container, "BrandNewTradeType", attrib={ + "unknownField": "value" + }) + + result = parser.parse_element_container(container) + # Only the known Trade should be in the result + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], Types.Trade) + + def test_full_parse_with_unknown_attrs(self): + """Full round-trip: parse a FlexQueryResponse with unknown attributes.""" + parser.enable_unknown_attribute_tolerance() + + xml_data = ( + '' + '' + '' + '' + '' + ) + response = parser.parse(xml_data.encode()) + self.assertIsInstance(response, Types.FlexQueryResponse) + self.assertEqual(response.queryName, "test") + self.assertEqual(len(response.FlexStatements), 1) + stmt = response.FlexStatements[0] + self.assertEqual(stmt.accountId, "U123456") + + def test_full_parse_unknown_attrs_fails_without_tolerance(self): + """Without tolerance, unknown attributes in full parse raise errors.""" + xml_data = ( + '' + '' + '' + '' + '' + ) + with self.assertRaises(parser.FlexParserError): + parser.parse(xml_data.encode()) + + def test_unknown_contained_element_in_statement(self): + """With tolerance, unknown contained elements in FlexStatement are ignored.""" + parser.enable_unknown_attribute_tolerance() + + xml_data = ( + '' + '' + '' + '' + '' + '' + '' + '' + '' + ) + response = parser.parse(xml_data.encode()) + self.assertIsInstance(response, Types.FlexQueryResponse) + self.assertEqual(len(response.FlexStatements), 1) + self.assertEqual(response.FlexStatements[0].accountId, "U123456") + + def test_multiple_unknown_attrs_on_trade(self): + """Unknown attributes on Trade elements are ignored with tolerance.""" + parser.enable_unknown_attribute_tolerance() + + elem = ET.fromstring( + '' + ) + instance = parser.parse_data_element(elem) + self.assertIsInstance(instance, Types.Trade) + self.assertEqual(instance.symbol, "AAPL") + self.assertEqual(instance.tradeID, "123") + self.assertFalse(hasattr(instance, 'newField1')) + + if __name__ == '__main__': unittest.main(verbosity=3) diff --git a/tests/test_types.py b/tests/test_types.py index bf53cf4..175eaa5 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1985,5 +1985,49 @@ def testParse(self): self.assertEqual(instance.tradeID, None) +class TradeInitialInvestmentTestCase(unittest.TestCase): + """Test case for Trade.initialInvestment as boolean field. + + Tests the fix for https://github.com/vroonhof/opensteuerauszug/issues/106 + where initialInvestment="Yes" was causing parsing errors. + """ + data = ET.fromstring( + ('') + ) + + def testParse(self): + instance = parser.parse_data_element(self.data) + self.assertIsInstance(instance, Types.Trade) + self.assertEqual(instance.accountId, "U123456") + self.assertEqual(instance.currency, "USD") + self.assertEqual(instance.assetCategory, enums.AssetClass.STOCK) + self.assertEqual(instance.symbol, "TEST") + self.assertEqual(instance.initialInvestment, True) + self.assertEqual(instance.quantity, decimal.Decimal("100")) + + +class EquitySummaryLiteSurchargeAccrualsTestCase(unittest.TestCase): + """Test case for EquitySummaryByReportDateInBase.liteSurchargeAccruals field. + + Tests the fix for https://github.com/vroonhof/opensteuerauszug/issues/106 + where liteSurchargeAccruals attribute was missing. + """ + data = ET.fromstring( + ('') + ) + + def testParse(self): + instance = parser.parse_data_element(self.data) + self.assertIsInstance(instance, Types.EquitySummaryByReportDateInBase) + self.assertEqual(instance.accountId, "U123456") + self.assertEqual(instance.reportDate, datetime.date(2024, 1, 1)) + self.assertEqual(instance.cash, decimal.Decimal("1000.00")) + self.assertEqual(instance.total, decimal.Decimal("1000.00")) + self.assertEqual(instance.liteSurchargeAccruals, decimal.Decimal("5.50")) + + if __name__ == '__main__': unittest.main(verbosity=3)