Skip to content

Comments

fix: dialect.has_table() fixed#32

Merged
nwoolmer merged 1 commit intomainfrom
jh_has_table
May 12, 2025
Merged

fix: dialect.has_table() fixed#32
nwoolmer merged 1 commit intomainfrom
jh_has_table

Conversation

@jerrinot
Copy link
Contributor

@jerrinot jerrinot commented May 12, 2025

Tt was missing the last **kw argument. For this reason Pandas/Polars could not use QuestDB as a target.

This now works:

import polars as pl
from datetime import datetime, timezone

data = {
    "event_time": [
        datetime(2024, 1, 1, 10, 0, 0, microsecond=1, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 5, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 10, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 15, tzinfo=timezone.utc),
    ],
    "sensor_id": ["sensor_alpha", "sensor_beta", "sensor_alpha", "sensor_gamma"],
    "temperature": [22.5, 23.1, 22.7, 25.0],
    "humidity": [45.0, 47.5, 45.5, 42.1],
    "status": ["normal", "normal", "warning", "normal"],
}
df = pl.DataFrame(data)
df = df.with_columns(pl.col("event_time").cast(pl.Datetime(time_unit="us")))

QUESTDB_PG_URI = "questdb://admin:quest@localhost:8812/qdb"
TABLE_NAME = "sensor_readings_polars"

print("Polars DataFrame to be written:")
print(df)

df.write_database(
    table_name=TABLE_NAME,
    connection=QUESTDB_PG_URI,
    if_table_exists="append",
    engine="sqlalchemy"
)
print(f"\nSuccessfully wrote DataFrame to QuestDB table: '{TABLE_NAME}'")

Writting frames via psycopg2 is far from ideal from a performance point of view, but it's better than failing with:

Traceback (most recent call last):
  File "/home/jara/devel/tmp/polars_test/test.py", line 25, in <module>
    df.write_database(
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/polars/dataframe/frame.py", line 4278, in write_database
    res: int | None = self.to_pandas(
                      ^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/util/_decorators.py", line 333, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/core/generic.py", line 3087, in to_sql
    return sql.to_sql(
           ^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 842, in to_sql
    return pandas_sql.to_sql(
           ^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 2008, in to_sql
    table = self.prep_table(
            ^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 1912, in prep_table
    table.create()
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 984, in create
    if self.exists():
       ^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 970, in exists
    return self.pd_sql.has_table(self.name, self.schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 2041, in has_table
    return insp.has_table(name, schema or self.meta.schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/sqlalchemy/engine/reflection.py", line 439, in has_table
    return self.dialect.has_table(
           ^^^^^^^^^^^^^^^^^^^^^^^
TypeError: QuestDBDialect.has_table() got an unexpected keyword argument 'info_cache'

Tt was missing the last **kw argument. For this reason Pandas/Polars could not use
QuestDB as a target.

This now works:
```python
import polars as pl
from datetime import datetime, timezone

data = {
    "event_time": [
        datetime(2024, 1, 1, 10, 0, 0, microsecond=1, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 5, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 10, tzinfo=timezone.utc),
        datetime(2024, 1, 1, 10, 0, 15, tzinfo=timezone.utc),
    ],
    "sensor_id": ["sensor_alpha", "sensor_beta", "sensor_alpha", "sensor_gamma"],
    "temperature": [22.5, 23.1, 22.7, 25.0],
    "humidity": [45.0, 47.5, 45.5, 42.1],
    "status": ["normal", "normal", "warning", "normal"],
}
df = pl.DataFrame(data)
df = df.with_columns(pl.col("event_time").cast(pl.Datetime(time_unit="us")))

QUESTDB_PG_URI = "questdb://admin:quest@localhost:8812/qdb"
TABLE_NAME = "sensor_readings_polars"

print("Polars DataFrame to be written:")
print(df)

df.write_database(
    table_name=TABLE_NAME,
    connection=QUESTDB_PG_URI,
    if_table_exists="append",
    engine="sqlalchemy"
)
print(f"\nSuccessfully wrote DataFrame to QuestDB table: '{TABLE_NAME}'")
```

It's far from ideal from performance point of view, but it's better than failing with:
```
Traceback (most recent call last):
  File "/home/jara/devel/tmp/polars_test/test.py", line 25, in <module>
    df.write_database(
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/polars/dataframe/frame.py", line 4278, in write_database
    res: int | None = self.to_pandas(
                      ^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/util/_decorators.py", line 333, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/core/generic.py", line 3087, in to_sql
    return sql.to_sql(
           ^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 842, in to_sql
    return pandas_sql.to_sql(
           ^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 2008, in to_sql
    table = self.prep_table(
            ^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 1912, in prep_table
    table.create()
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 984, in create
    if self.exists():
       ^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 970, in exists
    return self.pd_sql.has_table(self.name, self.schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/pandas/io/sql.py", line 2041, in has_table
    return insp.has_table(name, schema or self.meta.schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jara/devel/tmp/polars_test/.venv/lib/python3.11/site-packages/sqlalchemy/engine/reflection.py", line 439, in has_table
    return self.dialect.has_table(
           ^^^^^^^^^^^^^^^^^^^^^^^
TypeError: QuestDBDialect.has_table() got an unexpected keyword argument 'info_cache'
```
@nwoolmer nwoolmer merged commit 781b7a5 into main May 12, 2025
3 checks passed
@nwoolmer nwoolmer deleted the jh_has_table branch May 12, 2025 12:18
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants