Add an opt-in method to enable tolerance against new attribibutes

ibflex/Types.py

@@ -479,6 +479,7 @@ class EquitySummaryByReportDateInBase(FlexElement):
     marginFinancingChargeAccrualsShort: Optional[decimal.Decimal] = None
     cryptoLong: Optional[decimal.Decimal] = None
     cryptoShort: Optional[decimal.Decimal] = None
+    liteSurchargeAccruals: Optional[decimal.Decimal] = None
 
 
 @dataclass(frozen=True)
@@ -709,6 +710,8 @@ class CashReportCurrency(FlexElement):
     salesTaxYTD: Optional[decimal.Decimal] = None
     salesTaxPaxos: Optional[decimal.Decimal] = None
     otherIncome: Optional[decimal.Decimal] = None
+    otherIncomeMTD: Optional[decimal.Decimal] = None
+    otherIncomeYTD: Optional[decimal.Decimal] = None
     otherIncomeSec: Optional[decimal.Decimal] = None
     otherIncomeCom: Optional[decimal.Decimal] = None
     otherFeesMTD: Optional[decimal.Decimal] = None
@@ -1144,7 +1147,8 @@ class Trade(FlexElement):
     subCategory: Optional[str] = None
     issuerCountryCode: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
+    positionActionID: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -1296,7 +1300,8 @@ class Lot(FlexElement):
     issuerCountryCode: Optional[str] = None
     relatedTradeID: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
+    positionActionID: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -1385,19 +1390,29 @@ class SymbolSummary(FlexElement):
     tradeID: Optional[str] = None
     orderID: Optional[decimal.Decimal] = None
     execID: Optional[str] = None
+    ibExecID: Optional[str] = None
+    extExecID: Optional[str] = None
+    exchOrderId: Optional[str] = None
     brokerageOrderID: Optional[str] = None
     orderReference: Optional[str] = None
     volatilityOrderLink: Optional[str] = None
     clearingFirmID: Optional[str] = None
     origTradePrice: Optional[decimal.Decimal] = None
     origTradeDate: Optional[datetime.date] = None
     origTradeID: Optional[str] = None
+    transactionID: Optional[str] = None
     #  Despite the name, `orderTime` actually contains date/time data.
     orderTime: Optional[datetime.datetime] = None
+    openDateTime: Optional[datetime.datetime] = None
+    holdingPeriodDateTime: Optional[datetime.datetime] = None
     dateTime: Optional[datetime.datetime] = None
     reportDate: Optional[datetime.date] = None
     settleDate: Optional[datetime.date] = None
+    settleDateTarget: Optional[datetime.date] = None        # expected date of ownership transfer
+    taxes: Optional[decimal.Decimal] = None
     tradeDate: Optional[datetime.date] = None
+    tradePrice: Optional[decimal.Decimal] = None    
+    tradeMoney: Optional[decimal.Decimal] = None            # TradeMoney = Proceeds + Fees + Commissions
     exchange: Optional[str] = None
     buySell: Optional[enums.BuySell] = None
     quantity: Optional[decimal.Decimal] = None
@@ -1427,6 +1442,40 @@ class SymbolSummary(FlexElement):
     relatedTradeID: Optional[str] = None
     origTransactionID: Optional[str] = None
     relatedTransactionID: Optional[str] = None
+    positionActionID: Optional[str] = None
+    changeInPrice: Optional[decimal.Decimal] = None
+    changeInQuantity: Optional[decimal.Decimal] = None
+    closePrice: Optional[decimal.Decimal] = None
+    commodityType: Optional[str] = None
+    cost: Optional[decimal.Decimal] = None
+    deliveryType: Optional[str] = None
+    exchOrderId: Optional[str] = None
+    extExecID: Optional[str] = None
+    fifoPnlRealized: Optional[decimal.Decimal] = None
+    fineness: Optional[decimal.Decimal] = None
+    holdingPeriodDateTime: Optional[datetime.datetime] = None
+    ibCommission: Optional[decimal.Decimal] = None
+    ibCommissionCurrency: Optional[str] = None
+    ibExecID: Optional[str] = None
+    ibOrderID: Optional[str] = None
+    initialInvestment: Optional[bool] = None
+    mtmPnl: Optional[decimal.Decimal] = None
+    netCash: Optional[decimal.Decimal] = None
+    netCashInBase: Optional[decimal.Decimal] = None
+    notes: Optional[str] = None
+    openCloseIndicator: Optional[enums.OpenClose] = None
+    openDateTime: Optional[datetime.datetime] = None
+    origOrderID: Optional[str] = None
+    rtn: Optional[str] = None
+    serialNumber: Optional[str] = None
+    settleDateTarget: Optional[datetime.date] = None
+    taxes: Optional[decimal.Decimal] = None
+    tradeMoney: Optional[decimal.Decimal] = None
+    tradePrice: Optional[decimal.Decimal] = None
+    transactionID: Optional[str] = None
+    weight: Optional[str] = None
+    whenRealized: Optional[datetime.datetime] = None
+    whenReopened: Optional[datetime.datetime] = None
 
 
 @dataclass(frozen=True)
@@ -1535,7 +1584,8 @@ class AssetSummary(FlexElement):
     origTransactionID: Optional[str] = None
     relatedTransactionID: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
+    positionActionID: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -1638,12 +1688,13 @@ class Order(FlexElement):
     origTransactionID: Optional[str] = None
     relatedTransactionID: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
     serialNumber: Optional[str] = None
     deliveryType: Optional[str] = None
     commodityType: Optional[str] = None
     fineness: Optional[decimal.Decimal] = None
     weight: Optional[str] = None
+    positionActionID: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -1796,7 +1847,7 @@ class OptionEAE(FlexElement):
     origTransactionID: Optional[str] = None
     relatedTransactionID: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
     serialNumber: Optional[str] = None
     deliveryType: Optional[str] = None
     commodityType: Optional[str] = None
@@ -2122,7 +2173,12 @@ class Transfer(FlexElement):
     commodityType: Optional[str] = None
     fineness: Optional[decimal.Decimal] = None
     weight: Optional[str] = None
-
+    figi: Optional[str] = None
+    settleDate: Optional[datetime.date] = None
+    issuerCountryCode: Optional[str] = None
+    levelOfDetail: Optional[str] = None
+    positionInstructionID: Optional[str] = None
+    positionInstructionSetID: Optional[str] = None
 
 @dataclass(frozen=True)
 class UnsettledTransfer(FlexElement):
@@ -2245,6 +2301,11 @@ class CorporateAction(FlexElement):
     commodityType: Optional[str] = None
     fineness: Optional[decimal.Decimal] = None
     weight: Optional[str] = None
+    figi: Optional[str] = None
+    issuerCountryCode: Optional[str] = None
+    costBasis: Optional[decimal.Decimal] = None
+
+
 
 
 @dataclass(frozen=True)
@@ -2480,7 +2541,7 @@ class SecurityInfo(FlexElement):
     origTransactionID: Optional[str] = None
     relatedTransactionID: Optional[str] = None
     rtn: Optional[str] = None
-    initialInvestment: Optional[decimal.Decimal] = None
+    initialInvestment: Optional[bool] = None
     serialNumber: Optional[str] = None
     deliveryType: Optional[str] = None
     commodityType: Optional[str] = None

ibflex/__init__.py

@@ -4,6 +4,8 @@
 from .Types import *
 from . import parser
 from .parser import parse
+from .parser import enable_unknown_attribute_tolerance
+from .parser import disable_unknown_attribute_tolerance
 from . import utils
 from . import client
 

ibflex/client.py

@@ -136,8 +136,6 @@ def request_statement(
     """First part of the 2-step download process.
     """
     url = url or REQUEST_URL
-    ### AKE FIX
-    url = 'https://ndcdyn.interactivebrokers.com/portal.flexweb/api/v1/flexQuery'
     response = submit_request(url, token, query=query_id)
     stmt_access = parse_stmt_response(response)
     if isinstance(stmt_access, StatementError):

ibflex/parser.py

@@ -21,6 +21,56 @@ class FlexParserError(Exception):
     """ Error experienced while parsing Flex XML data. """
 
 
+###############################################################################
+#  UNKNOWN ATTRIBUTE TOLERANCE
+###############################################################################
+_UNKNOWN_ATTRIBUTE_TOLERANCE = False
+
+
+def enable_unknown_attribute_tolerance():
+    """Enable tolerance for unknown XML attributes in IB Flex data.
+
+    When enabled, unknown attributes and element types in the XML data are
+    silently ignored instead of raising FlexParserError. This is useful when
+    Interactive Brokers adds new fields to their exports that are not yet
+    defined in the Types module.
+
+    This function is only available in the forked version of ibflex.
+    Attempting to call it on the original upstream package will raise
+    AttributeError (the method does not exist there), providing a clear
+    signal that the feature is not supported.
+
+    Default: off (strict mode - unknown attributes raise errors).
+
+    See also: disable_unknown_attribute_tolerance()
+    """
+    global _UNKNOWN_ATTRIBUTE_TOLERANCE
+    _UNKNOWN_ATTRIBUTE_TOLERANCE = True
+
+
+def disable_unknown_attribute_tolerance():
+    """Disable tolerance for unknown XML attributes (the default behavior).
+
+    After calling this, unknown attributes in XML data will raise
+    FlexParserError as usual.
+
+    See also: enable_unknown_attribute_tolerance()
+    """
+    global _UNKNOWN_ATTRIBUTE_TOLERANCE
+    _UNKNOWN_ATTRIBUTE_TOLERANCE = False
+
+
+def _get_known_attributes(Class):
+    """Get all known attribute names for a FlexElement subclass,
+    including inherited attributes from base classes.
+    """
+    attrs = set()
+    for klass in Class.__mro__:
+        if hasattr(klass, '__annotations__'):
+            attrs.update(klass.__annotations__.keys())
+    return attrs
+
+
 DataType = Union[
     None, str, int, bool, decimal.Decimal, datetime.date, datetime.time,
     datetime.datetime, enums.EnumType, Tuple[str, ...], Tuple[enums.Code, ...]
@@ -99,22 +149,40 @@ def parse_element_container(elem: ET.Element) -> Tuple[Types.FlexElement, ...]:
         return tuple(itertools.chain.from_iterable(fxlots))
 
     instances = tuple(parse_data_element(child) for child in elem)
+    if _UNKNOWN_ATTRIBUTE_TOLERANCE:
+        instances = tuple(inst for inst in instances if inst is not None)
     return instances
 
 
 def parse_data_element(
     elem: ET.Element
-) -> Types.FlexElement:
+) -> Optional[Types.FlexElement]:
     """Parse an XML data element into a Types.FlexElement subclass instance.
+
+    Returns None if unknown_attribute_tolerance is enabled and the element
+    type is not recognized.
     """
     #  Look up XML element's matching FlexElement subclass in ibflex.Types.
-    Class = getattr(Types, elem.tag)
+    try:
+        Class = getattr(Types, elem.tag)
+    except AttributeError:
+        if _UNKNOWN_ATTRIBUTE_TOLERANCE:
+            return None
+        raise
+
+    #  When tolerance is enabled, pre-compute known attributes and filter
+    known = _get_known_attributes(Class) if _UNKNOWN_ATTRIBUTE_TOLERANCE else None
 
     #  Parse element attributes
+    if known is not None:
+        attrib_items = [(k, v) for k, v in elem.attrib.items() if k in known]
+    else:
+        attrib_items = list(elem.attrib.items())
+
     try:
         attrs = dict(
             parse_element_attr(Class, k, v)
-            for k, v in elem.attrib.items()
+            for k, v in attrib_items
         )
     except KeyError as exc:
         msg = f"{Class.__name__} has no attribute " + str(exc)
@@ -125,6 +193,12 @@ def parse_data_element(
     contained_elements = {child.tag: parse_element(child) for child in elem}
     if contained_elements:
         assert elem.tag in ("FlexQueryResponse", "FlexStatement")
+        if _UNKNOWN_ATTRIBUTE_TOLERANCE:
+            #  Filter out unknown or unparseable contained elements
+            contained_elements = {
+                k: v for k, v in contained_elements.items()
+                if k in known and v is not None
+            }
         attrs.update(contained_elements)
 
     try:
@@ -170,9 +244,11 @@ def parse_element_attr(
 #  INPUT VALUE PREP FUNCTIONS FOR DATA CONVERTERS
 #  These are just implementation details for converters and don't need testing.
 ###############################################################################
-def prep_date(value: str) -> Tuple[int, int, int]:
+def prep_date(value: str) -> Optional[Tuple[int, int, int]]:
     """Returns a tuple of (year, month, day).
     """
+    if value == "MULTI":
+        return None  # Summaries have MULTI as date value.
     date_format = DATE_FORMATS[len(value)][value.count('/')]
     return datetime.datetime.strptime(value, date_format).timetuple()[:3]
 
@@ -184,9 +260,11 @@ def prep_time(value: str) -> Tuple[int, int, int]:
     return datetime.datetime.strptime(value, time_format).timetuple()[3:6]
 
 
-def prep_datetime(value: str) -> Tuple[int, ...]:
+def prep_datetime(value: str) -> Optional[Tuple[int, ...]]:
     """Returns a tuple of (year, month, day, hour, minute, second).
     """
+    if value == "MULTI":
+        return None  # Summaries have MULTI as date value.
     #  HACK - some old data has ", " separator instead of ",".
     value = value.replace(", ", ",")
 
@@ -328,8 +406,8 @@ def optional_convert(value):
 
 convert_string = make_optional(make_converter(str, prep=utils.identity_func))
 convert_int = make_converter(int, prep=utils.identity_func)
-# IB sends "Y"/"N" for True/False
-convert_bool = make_converter(bool, prep=lambda x: {"Y": True, "N": False}[x])
+# IB sends "Y"/"N" or "Yes"/"No" for True/False
+convert_bool = make_converter(bool, prep=lambda x: {"Y": True, "N": False, "Yes": True, "No": False}[x])
 # IB sends numeric data with place delimiters (commas)
 convert_decimal = make_converter(
     decimal.Decimal,
@@ -463,6 +541,7 @@ def convert_enum(Type, value):
     "CNH",           # RMB traded in HK
     "BASE_SUMMARY",  # Fake currency code used in IB NAV/Performance reports
     "",              # Lot element allows blank currency ?!
+    "RUS",           # Russian-related currency code used by IBKR
 )