Skip to content

Commit 911c745

Browse files
izeigermanizeigerman
authored
Fix!: Use LONGTEXT data type when storing large blobs of text in MySQL (#1673)
1 parent 5d655d4 commit 911c745

28 files changed

+308
-238
lines changed

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"requests",
4747
"rich[jupyter]",
4848
"ruamel.yaml",
49-
"sqlglot~=19.0.3",
49+
"sqlglot~=19.1.1",
5050
],
5151
extras_require={
5252
"bigquery": [

‎sqlmesh/core/state_sync/engine_adapter.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
from sqlmesh.utils import major_minor, random_id
5858
from sqlmesh.utils.date import TimeLike, now_timestamp, time_like_to_str
5959
from sqlmesh.utils.errors import SQLMeshError
60+
from sqlmesh.utils.migration import blob_text_type, index_text_type
6061

6162
logger = logging.getLogger(__name__)
6263

@@ -103,17 +104,20 @@ def __init__(
103104
self.plan_dags_table = exp.table_("_plan_dags", db=self.schema)
104105
self.versions_table = exp.table_("_versions", db=self.schema)
105106

107+
index_type = index_text_type(self.engine_adapter.dialect)
108+
blob_type = blob_text_type(self.engine_adapter.dialect)
109+
106110
self._snapshot_columns_to_types = {
107-
"name": exp.DataType.build("text"),
108-
"identifier": exp.DataType.build("text"),
109-
"version": exp.DataType.build("text"),
110-
"snapshot": exp.DataType.build("text"),
111-
"kind_name": exp.DataType.build("text"),
111+
"name": exp.DataType.build(index_type),
112+
"identifier": exp.DataType.build(index_type),
113+
"version": exp.DataType.build(index_type),
114+
"snapshot": exp.DataType.build(blob_type),
115+
"kind_name": exp.DataType.build(index_type),
112116
}
113117

114118
self._environment_columns_to_types = {
115-
"name": exp.DataType.build("text"),
116-
"snapshots": exp.DataType.build("text"),
119+
"name": exp.DataType.build(index_type),
120+
"snapshots": exp.DataType.build(blob_type),
117121
"start_at": exp.DataType.build("text"),
118122
"end_at": exp.DataType.build("text"),
119123
"plan_id": exp.DataType.build("text"),
@@ -125,17 +129,17 @@ def __init__(
125129
}
126130

127131
self._seed_columns_to_types = {
128-
"name": exp.DataType.build("text"),
129-
"identifier": exp.DataType.build("text"),
130-
"content": exp.DataType.build("text"),
132+
"name": exp.DataType.build(index_type),
133+
"identifier": exp.DataType.build(index_type),
134+
"content": exp.DataType.build(blob_type),
131135
}
132136

133137
self._interval_columns_to_types = {
134-
"id": exp.DataType.build("text"),
138+
"id": exp.DataType.build(index_type),
135139
"created_ts": exp.DataType.build("bigint"),
136-
"name": exp.DataType.build("text"),
137-
"identifier": exp.DataType.build("text"),
138-
"version": exp.DataType.build("text"),
140+
"name": exp.DataType.build(index_type),
141+
"identifier": exp.DataType.build(index_type),
142+
"version": exp.DataType.build(index_type),
139143
"start_ts": exp.DataType.build("bigint"),
140144
"end_ts": exp.DataType.build("bigint"),
141145
"is_dev": exp.DataType.build("boolean"),

‎sqlmesh/migrations/v0001_init.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ def migrate(state_sync): # type: ignore
2121
environments_table = f"{schema}.{environments_table}"
2222
versions_table = f"{schema}.{versions_table}"
2323

24-
text_type = index_text_type(engine_adapter.dialect)
24+
index_type = index_text_type(engine_adapter.dialect)
2525

2626
engine_adapter.create_state_table(
2727
snapshots_table,
2828
{
29-
"name": exp.DataType.build(text_type),
30-
"identifier": exp.DataType.build(text_type),
31-
"version": exp.DataType.build(text_type),
29+
"name": exp.DataType.build(index_type),
30+
"identifier": exp.DataType.build(index_type),
31+
"version": exp.DataType.build(index_type),
3232
"snapshot": exp.DataType.build("text"),
3333
},
3434
primary_key=("name", "identifier"),
@@ -39,7 +39,7 @@ def migrate(state_sync): # type: ignore
3939
engine_adapter.create_state_table(
4040
environments_table,
4141
{
42-
"name": exp.DataType.build(text_type),
42+
"name": exp.DataType.build(index_type),
4343
"snapshots": exp.DataType.build("text"),
4444
"start_at": exp.DataType.build("text"),
4545
"end_at": exp.DataType.build("text"),

‎sqlmesh/migrations/v0005_create_seed_table.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@ def migrate(state_sync): # type: ignore
1010
if state_sync.schema:
1111
seeds_table = f"{state_sync.schema}.{seeds_table}"
1212

13-
text_type = index_text_type(engine_adapter.dialect)
13+
index_type = index_text_type(engine_adapter.dialect)
1414

1515
engine_adapter.create_state_table(
1616
seeds_table,
1717
{
18-
"name": exp.DataType.build(text_type),
19-
"identifier": exp.DataType.build(text_type),
18+
"name": exp.DataType.build(index_type),
19+
"identifier": exp.DataType.build(index_type),
2020
"content": exp.DataType.build("text"),
2121
},
2222
primary_key=("name", "identifier"),

‎sqlmesh/migrations/v0007_env_table_info_to_kind.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,13 @@ def migrate(state_sync): # type: ignore
7979
if new_environments:
8080
engine_adapter.delete_from(environments_table, "TRUE")
8181

82-
text_type = index_text_type(engine_adapter.dialect)
82+
index_type = index_text_type(engine_adapter.dialect)
8383

8484
engine_adapter.insert_append(
8585
environments_table,
8686
pd.DataFrame(new_environments),
8787
columns_to_types={
88-
"name": exp.DataType.build(text_type),
88+
"name": exp.DataType.build(index_type),
8989
"snapshots": exp.DataType.build("text"),
9090
"start_at": exp.DataType.build("text"),
9191
"end_at": exp.DataType.build("text"),

‎sqlmesh/migrations/v0008_create_intervals_table.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@ def migrate(state_sync): # type: ignore
1010
if state_sync.schema:
1111
intervals_table = f"{state_sync.schema}.{intervals_table}"
1212

13-
text_type = index_text_type(engine_adapter.dialect)
13+
index_type = index_text_type(engine_adapter.dialect)
1414

1515
engine_adapter.create_state_table(
1616
intervals_table,
1717
{
18-
"id": exp.DataType.build(text_type),
18+
"id": exp.DataType.build(index_type),
1919
"created_ts": exp.DataType.build("bigint"),
20-
"name": exp.DataType.build(text_type),
21-
"identifier": exp.DataType.build(text_type),
22-
"version": exp.DataType.build(text_type),
20+
"name": exp.DataType.build(index_type),
21+
"identifier": exp.DataType.build(index_type),
22+
"version": exp.DataType.build(index_type),
2323
"start_ts": exp.DataType.build("bigint"),
2424
"end_ts": exp.DataType.build("bigint"),
2525
"is_dev": exp.DataType.build("boolean"),

‎sqlmesh/migrations/v0009_remove_pre_post_hooks.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,15 @@ def migrate(state_sync): # type: ignore
4646
if new_snapshots:
4747
engine_adapter.delete_from(snapshots_table, "TRUE")
4848

49-
text_type = index_text_type(engine_adapter.dialect)
49+
index_type = index_text_type(engine_adapter.dialect)
5050

5151
engine_adapter.insert_append(
5252
snapshots_table,
5353
pd.DataFrame(new_snapshots),
5454
columns_to_types={
55-
"name": exp.DataType.build(text_type),
56-
"identifier": exp.DataType.build(text_type),
57-
"version": exp.DataType.build(text_type),
55+
"name": exp.DataType.build(index_type),
56+
"identifier": exp.DataType.build(index_type),
57+
"version": exp.DataType.build(index_type),
5858
"snapshot": exp.DataType.build("text"),
5959
},
6060
contains_json=True,

‎sqlmesh/migrations/v0011_add_model_kind_name.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@ def migrate(state_sync): # type: ignore
1414
if schema:
1515
snapshots_table = f"{schema}.{snapshots_table}"
1616

17-
text_type = index_text_type(engine_adapter.dialect)
17+
index_type = index_text_type(engine_adapter.dialect)
1818

1919
alter_table_exp = exp.AlterTable(
2020
this=exp.to_table(snapshots_table),
2121
actions=[
2222
exp.ColumnDef(
2323
this=exp.to_column("kind_name"),
24-
kind=exp.DataType.build(text_type),
24+
kind=exp.DataType.build(index_type),
2525
)
2626
],
2727
)
@@ -51,11 +51,11 @@ def migrate(state_sync): # type: ignore
5151
snapshots_table,
5252
pd.DataFrame(new_snapshots),
5353
columns_to_types={
54-
"name": exp.DataType.build(text_type),
55-
"identifier": exp.DataType.build(text_type),
56-
"version": exp.DataType.build(text_type),
54+
"name": exp.DataType.build(index_type),
55+
"identifier": exp.DataType.build(index_type),
56+
"version": exp.DataType.build(index_type),
5757
"snapshot": exp.DataType.build("text"),
58-
"kind_name": exp.DataType.build(text_type),
58+
"kind_name": exp.DataType.build(index_type),
5959
},
6060
contains_json=True,
6161
)

‎sqlmesh/migrations/v0012_update_jinja_expressions.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,17 @@ def migrate(state_sync): # type: ignore
5050
if new_snapshots:
5151
engine_adapter.delete_from(snapshots_table, "TRUE")
5252

53-
text_type = index_text_type(engine_adapter.dialect)
53+
index_type = index_text_type(engine_adapter.dialect)
5454

5555
engine_adapter.insert_append(
5656
snapshots_table,
5757
pd.DataFrame(new_snapshots),
5858
columns_to_types={
59-
"name": exp.DataType.build(text_type),
60-
"identifier": exp.DataType.build(text_type),
61-
"version": exp.DataType.build(text_type),
59+
"name": exp.DataType.build(index_type),
60+
"identifier": exp.DataType.build(index_type),
61+
"version": exp.DataType.build(index_type),
6262
"snapshot": exp.DataType.build("text"),
63-
"kind_name": exp.DataType.build("text"),
63+
"kind_name": exp.DataType.build(index_type),
6464
},
6565
contains_json=True,
6666
)

‎sqlmesh/migrations/v0013_serde_using_model_dialects.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,17 @@ def migrate(state_sync): # type: ignore
4848
if new_snapshots:
4949
engine_adapter.delete_from(snapshots_table, "TRUE")
5050

51-
text_type = index_text_type(engine_adapter.dialect)
51+
index_type = index_text_type(engine_adapter.dialect)
5252

5353
engine_adapter.insert_append(
5454
snapshots_table,
5555
pd.DataFrame(new_snapshots),
5656
columns_to_types={
57-
"name": exp.DataType.build(text_type),
58-
"identifier": exp.DataType.build(text_type),
59-
"version": exp.DataType.build(text_type),
57+
"name": exp.DataType.build(index_type),
58+
"identifier": exp.DataType.build(index_type),
59+
"version": exp.DataType.build(index_type),
6060
"snapshot": exp.DataType.build("text"),
61-
"kind_name": exp.DataType.build("text"),
61+
"kind_name": exp.DataType.build(index_type),
6262
},
6363
contains_json=True,
6464
)

0 commit comments

Comments
 (0)