From 1491db1671a8f9655184995be36cc67751de4bf4 Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Tue, 15 Aug 2023 11:22:57 -0400 Subject: [PATCH 1/6] Support disk/embedded/remote store via libsql --- CHANGELOG.md | 4 ++ README.md | 9 ++- fastapi_cache/backends/__init__.py | 7 +++ fastapi_cache/backends/libsql.py | 99 ++++++++++++++++++++++++++++++ poetry.lock | 20 +++++- pyproject.toml | 4 +- 6 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 fastapi_cache/backends/libsql.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 179cd214..c2132228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang ## 0.2 +### 0.2.2 + +- Support `libsql` backend. + ### 0.2.1 - Fix picklecoder - Fix connection failure transparency and add logging diff --git a/README.md b/README.md index 4bac1398..7ebc9473 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,11 @@ ## Introduction `fastapi-cache` is a tool to cache FastAPI endpoint and function results, with -backends supporting Redis, Memcached, and Amazon DynamoDB. +backends supporting Redis, Memcached, libsql and Amazon DynamoDB. ## Features -- Supports `redis`, `memcache`, `dynamodb`, and `in-memory` backends. +- Supports `redis`, `memcache`, `dynamodb`, `libsql` and `in-memory` backends. - Easy integration with [FastAPI](https://fastapi.tiangolo.com/). - Support for HTTP cache headers like `ETag` and `Cache-Control`, as well as conditional `If-Match-None` requests. @@ -21,6 +21,7 @@ backends supporting Redis, Memcached, and Amazon DynamoDB. - `redis` when using `RedisBackend`. - `memcache` when using `MemcacheBackend`. - `aiobotocore` when using `DynamoBackend`. +- `libsql-client` when using `libsql` ## Install @@ -46,6 +47,10 @@ or > pip install "fastapi-cache2[dynamodb]" ``` +```shell +> pip install "fastapi-cache2[libsql]" +``` + ## Usage ### Quick Start diff --git a/fastapi_cache/backends/__init__.py b/fastapi_cache/backends/__init__.py index 23bd0a54..11cf5b1d 100644 --- a/fastapi_cache/backends/__init__.py +++ b/fastapi_cache/backends/__init__.py @@ -26,3 +26,10 @@ pass else: __all__ += ["redis"] + +try: + from fastapi_cache.backends import libsql +except ImportError: + pass +else: + __all__ += ["libsql"] diff --git a/fastapi_cache/backends/libsql.py b/fastapi_cache/backends/libsql.py new file mode 100644 index 00000000..12179af5 --- /dev/null +++ b/fastapi_cache/backends/libsql.py @@ -0,0 +1,99 @@ +import time +from typing import Optional, Tuple + +import libsql_client +from libsql_client import ResultSet + +from fastapi_cache.types import Backend + +EmptyResultSet = ResultSet( + columns=(), + rows=[], + rows_affected=0, + last_insert_rowid=0) + +class LibsqlBackend(Backend): + """ + libsql backend provider + + This backend requires a table name to be passed during initialization. The table + will be created if it does not exist. If the table does exists, it will be emptied during init + + Note that this backend does not fully support TTL. It will only delete outdated objects on get. + + Usage: + >> libsql_url = "file:local.db" + >> cache = LibsqlBackend(libsql_url=libsql_url, table_name="your-cache") + >> cache.create_and_flush() + >> FastAPICache.init(cache) + """ + + # client: libsql_client.Client + table_name: str + libsql_url: str + + def __init__(self, libsql_url: str, table_name: str): + self.libsql_url = libsql_url + self.table_name = table_name + + @property + def now(self) -> int: + return int(time.time()) + + async def _make_request(self, request: str) -> ResultSet: + # TODO: Exception handling. Return EmptyResultSet on error? + async with libsql_client.create_client(self.libsql_url) as client: + return await client.execute(request) + + + async def create_and_flush(self) -> None: + await self._make_request("CREATE TABLE IF NOT EXISTS `{}` " + "(key STRING PRIMARY KEY, value BLOB, expire INTEGER);" + .format(self.table_name)) + await self._make_request("DELETE FROM `{}`;".format(self.table_name)) + + return None + + async def _get(self, key: str) -> Tuple[int, Optional[bytes]]: + result_set = await self._make_request("SELECT * from `{}` WHERE key = \"{}\"" + .format(self.table_name,key)) + if len(result_set.rows) == 0: + return (0,None) + + value = result_set.rows[0]["value"] + ttl_ts = result_set.rows[0]["expire"] + + if not value: + return (0,None) + if ttl_ts < self.now: + await self._make_request("DELETE FROM `{}` WHERE key = \"{}\"" + .format(self.table_name, key)) + return (0, None) + + return(ttl_ts, value) # type: ignore[union-attr,no-any-return] + + async def get_with_ttl(self, key: str) -> Tuple[int, Optional[bytes]]: + return await self._get(key) + + async def get(self, key: str) -> Optional[bytes]: + _, value = await self._get(key) + return value + + async def set(self, key: str, value: bytes, expire: Optional[int] = None) -> None: + ttl = self.now + expire if expire else 0 + await self._make_request("INSERT OR REPLACE INTO `{}`(\"key\", \"value\", \"expire\") " + "VALUES('{}','{}',{});" + .format(self.table_name, key, value.decode("utf-8"), ttl)) + return None + + async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None) -> int: + + if namespace: + result_set = await self._make_request("DELETE FROM `{}` WHERE key = \"{}%\"" + .format(self.table_name, namespace)) + return result_set.rowcount # type: ignore + elif key: + result_set = await self._make_request("DELETE FROM `{}` WHERE key = \"{}\"" + .format(self.table_name, key)) + return result_set.rowcount # type: ignore + return 0 diff --git a/poetry.lock b/poetry.lock index 78b80c56..8ba4e9fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -988,6 +988,21 @@ completion = ["shtab"] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "libsql-client" +version = "0.3.0" +description = "Python SDK for libSQL" +optional = true +python-versions = ">=3.7,<4.0" +files = [ + {file = "libsql_client-0.3.0-py3-none-any.whl", hash = "sha256:b9edc4bc3b2c5f7e10a397e7a4e36451633a3ae4f2d7cec82c6767ccb9c34420"}, + {file = "libsql_client-0.3.0.tar.gz", hash = "sha256:8ed74e37601fc60498dfd70c5086252e8b8abb7974f7973be93d05dbdf589d05"}, +] + +[package.dependencies] +aiohttp = ">=3.0,<4.0" +typing-extensions = ">=4.5,<5.0" + [[package]] name = "markdown-it-py" version = "2.2.0" @@ -2487,12 +2502,13 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -all = ["aiobotocore", "aiomcache", "redis"] +all = ["aiobotocore", "aiomcache", "libsql-client", "redis"] dynamodb = ["aiobotocore"] +libsql = ["libsql-client"] memcache = ["aiomcache"] redis = ["redis"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "1d3f1bea1e9e956afd436a09a934a94f8f59dc223cc7d20b9e877f1962cfc06e" +content-hash = "1c91b7855d6ae5a943e7ca40d11b9eaa333cfaf9e558f30fad8a991acb16c01f" diff --git a/pyproject.toml b/pyproject.toml index cb834b0e..20f7045b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ pendulum = "*" aiobotocore = { version = ">=1.4.1,<3.0.0", optional = true } typing-extensions = { version = ">=4.1.0" } importlib-metadata = {version = "^6.6.0", python = "<3.8"} +libsql-client = { version = "^0.3.0", optional = true } [tool.poetry.group.linting] optional = true @@ -53,7 +54,8 @@ twine = { version = "^4.0.2", python = "^3.10" } redis = ["redis"] memcache = ["aiomcache"] dynamodb = ["aiobotocore"] -all = ["redis", "aiomcache", "aiobotocore"] +libsql = ["libsql-client"] +all = ["redis", "aiomcache", "aiobotocore", "libsql-client"] [tool.mypy] files = ["."] From 51e84268f73de903b2418af3f92f1726bb853daa Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Wed, 16 Aug 2023 10:48:09 -0400 Subject: [PATCH 2/6] * Switch to using parametrized queries where possible, to protect against sql injection * formatting etc., to pass linting --- fastapi_cache/backends/libsql.py | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/fastapi_cache/backends/libsql.py b/fastapi_cache/backends/libsql.py index 12179af5..196cd044 100644 --- a/fastapi_cache/backends/libsql.py +++ b/fastapi_cache/backends/libsql.py @@ -1,5 +1,5 @@ import time -from typing import Optional, Tuple +from typing import Any, Optional, Tuple import libsql_client from libsql_client import ResultSet @@ -8,17 +8,17 @@ EmptyResultSet = ResultSet( columns=(), - rows=[], - rows_affected=0, + rows=[], + rows_affected=0, last_insert_rowid=0) class LibsqlBackend(Backend): """ libsql backend provider - This backend requires a table name to be passed during initialization. The table + This backend requires a table name to be passed during initialization. The table will be created if it does not exist. If the table does exists, it will be emptied during init - + Note that this backend does not fully support TTL. It will only delete outdated objects on get. Usage: @@ -34,42 +34,42 @@ class LibsqlBackend(Backend): def __init__(self, libsql_url: str, table_name: str): self.libsql_url = libsql_url - self.table_name = table_name + #TODO: scrub table name for SQL injection. sqlite doesn't accept parameters for table names + self.table_name = table_name @property def now(self) -> int: return int(time.time()) - - async def _make_request(self, request: str) -> ResultSet: + + async def _make_request(self, request: str, params: Any = None) -> ResultSet: # TODO: Exception handling. Return EmptyResultSet on error? async with libsql_client.create_client(self.libsql_url) as client: - return await client.execute(request) + return await client.execute(request, params) async def create_and_flush(self) -> None: - await self._make_request("CREATE TABLE IF NOT EXISTS `{}` " - "(key STRING PRIMARY KEY, value BLOB, expire INTEGER);" - .format(self.table_name)) - await self._make_request("DELETE FROM `{}`;".format(self.table_name)) + await self._make_request(f"CREATE TABLE IF NOT EXISTS `{self.table_name}` " + "(key STRING PRIMARY KEY, value BLOB , expire INTEGER)") # noqa: S608 + await self._make_request(f"DELETE FROM `{self.table_name}`") # noqa: S608 return None async def _get(self, key: str) -> Tuple[int, Optional[bytes]]: - result_set = await self._make_request("SELECT * from `{}` WHERE key = \"{}\"" - .format(self.table_name,key)) + result_set = await self._make_request(f"SELECT * from `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + [key]) if len(result_set.rows) == 0: return (0,None) - + value = result_set.rows[0]["value"] ttl_ts = result_set.rows[0]["expire"] - + if not value: return (0,None) if ttl_ts < self.now: - await self._make_request("DELETE FROM `{}` WHERE key = \"{}\"" - .format(self.table_name, key)) + await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + [key]) return (0, None) - + return(ttl_ts, value) # type: ignore[union-attr,no-any-return] async def get_with_ttl(self, key: str) -> Tuple[int, Optional[bytes]]: @@ -81,19 +81,19 @@ async def get(self, key: str) -> Optional[bytes]: async def set(self, key: str, value: bytes, expire: Optional[int] = None) -> None: ttl = self.now + expire if expire else 0 - await self._make_request("INSERT OR REPLACE INTO `{}`(\"key\", \"value\", \"expire\") " - "VALUES('{}','{}',{});" - .format(self.table_name, key, value.decode("utf-8"), ttl)) + await self._make_request(f"INSERT OR REPLACE INTO `{self.table_name}`(\"key\", \"value\", \"expire\") " + "VALUES('?','?',?)", # noqa: S608 + [key, value.decode("utf-8"), ttl]) return None async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None) -> int: if namespace: - result_set = await self._make_request("DELETE FROM `{}` WHERE key = \"{}%\"" - .format(self.table_name, namespace)) + result_set = await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?%\"", # noqa: S608 + [namespace]) return result_set.rowcount # type: ignore elif key: - result_set = await self._make_request("DELETE FROM `{}` WHERE key = \"{}\"" - .format(self.table_name, key)) + result_set = await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + [key]) return result_set.rowcount # type: ignore return 0 From c26857a18b011b0681951e8c64db01927ff6b7b9 Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Wed, 16 Aug 2023 10:57:42 -0400 Subject: [PATCH 3/6] scrub table name for SQL injection. sqlite doesn't accept parameterized table names --- fastapi_cache/backends/libsql.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/fastapi_cache/backends/libsql.py b/fastapi_cache/backends/libsql.py index 196cd044..4c89ffbb 100644 --- a/fastapi_cache/backends/libsql.py +++ b/fastapi_cache/backends/libsql.py @@ -1,3 +1,4 @@ +import codecs import time from typing import Any, Optional, Tuple @@ -12,6 +13,21 @@ rows_affected=0, last_insert_rowid=0) +# see https://gist.github.com/jeremyBanks/1083518 +def quote_identifier(s:str, errors:str ="strict") -> str: + encodable = s.encode("utf-8", errors).decode("utf-8") + + nul_index = encodable.find("\x00") + + if nul_index >= 0: + error = UnicodeEncodeError("utf-8", encodable, nul_index, nul_index + 1, "NUL not allowed") + error_handler = codecs.lookup_error(errors) + replacement, _ = error_handler(error) + encodable = encodable.replace("\x00", replacement) # type: ignore + + return "\"" + encodable.replace("\"", "\"\"") + "\"" + + class LibsqlBackend(Backend): """ libsql backend provider @@ -34,8 +50,7 @@ class LibsqlBackend(Backend): def __init__(self, libsql_url: str, table_name: str): self.libsql_url = libsql_url - #TODO: scrub table name for SQL injection. sqlite doesn't accept parameters for table names - self.table_name = table_name + self.table_name = quote_identifier(table_name) @property def now(self) -> int: From b2f90a353d7d0dac40c0860f80e243c25a52e4cc Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Wed, 16 Aug 2023 11:00:08 -0400 Subject: [PATCH 4/6] revert CHANGELOG changes, since 2.2 upgrade isn't included in this pr --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2132228..179cd214 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,6 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang ## 0.2 -### 0.2.2 - -- Support `libsql` backend. - ### 0.2.1 - Fix picklecoder - Fix connection failure transparency and add logging From e7c0942fb3f60c46fce665a46a650a04e7400cbe Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Thu, 17 Aug 2023 09:45:25 -0400 Subject: [PATCH 5/6] Now that the queries are parametrized and the table name is sanitized, no longer need to escape throughout Now properly storing bytes instead of strings for request blobs --- fastapi_cache/backends/libsql.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fastapi_cache/backends/libsql.py b/fastapi_cache/backends/libsql.py index 4c89ffbb..6e33690c 100644 --- a/fastapi_cache/backends/libsql.py +++ b/fastapi_cache/backends/libsql.py @@ -63,14 +63,14 @@ async def _make_request(self, request: str, params: Any = None) -> ResultSet: async def create_and_flush(self) -> None: - await self._make_request(f"CREATE TABLE IF NOT EXISTS `{self.table_name}` " + await self._make_request(f"CREATE TABLE IF NOT EXISTS {self.table_name} " "(key STRING PRIMARY KEY, value BLOB , expire INTEGER)") # noqa: S608 - await self._make_request(f"DELETE FROM `{self.table_name}`") # noqa: S608 + await self._make_request(f"DELETE FROM {self.table_name}") # noqa: S608 return None async def _get(self, key: str) -> Tuple[int, Optional[bytes]]: - result_set = await self._make_request(f"SELECT * from `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + result_set = await self._make_request(f"SELECT * from {self.table_name} WHERE key = ?", # noqa: S608 [key]) if len(result_set.rows) == 0: return (0,None) @@ -81,7 +81,7 @@ async def _get(self, key: str) -> Tuple[int, Optional[bytes]]: if not value: return (0,None) if ttl_ts < self.now: - await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + await self._make_request(f"DELETE FROM {self.table_name} WHERE key = ?", # noqa: S608 [key]) return (0, None) @@ -96,19 +96,19 @@ async def get(self, key: str) -> Optional[bytes]: async def set(self, key: str, value: bytes, expire: Optional[int] = None) -> None: ttl = self.now + expire if expire else 0 - await self._make_request(f"INSERT OR REPLACE INTO `{self.table_name}`(\"key\", \"value\", \"expire\") " - "VALUES('?','?',?)", # noqa: S608 - [key, value.decode("utf-8"), ttl]) + await self._make_request(f"INSERT OR REPLACE INTO {self.table_name}(\"key\", \"value\", \"expire\") " + "VALUES(?,?,?)", # noqa: S608 + [key, value, ttl]) return None async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None) -> int: if namespace: - result_set = await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?%\"", # noqa: S608 - [namespace]) + result_set = await self._make_request(f"DELETE FROM {self.table_name} WHERE key = ?", # noqa: S608 + [namespace + '%']) return result_set.rowcount # type: ignore elif key: - result_set = await self._make_request(f"DELETE FROM `{self.table_name}` WHERE key = \"?\"", # noqa: S608 + result_set = await self._make_request(f"DELETE FROM {self.table_name} WHERE key = ?", # noqa: S608 [key]) return result_set.rowcount # type: ignore return 0 From 3405de6e0bc7ce987cf8579c976d7bd2cff953d0 Mon Sep 17 00:00:00 2001 From: Alex Ausch Date: Fri, 18 Aug 2023 08:33:56 -0400 Subject: [PATCH 6/6] Return rows affected instead of row countt when deleting --- fastapi_cache/backends/libsql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastapi_cache/backends/libsql.py b/fastapi_cache/backends/libsql.py index 6e33690c..62225581 100644 --- a/fastapi_cache/backends/libsql.py +++ b/fastapi_cache/backends/libsql.py @@ -106,9 +106,9 @@ async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None if namespace: result_set = await self._make_request(f"DELETE FROM {self.table_name} WHERE key = ?", # noqa: S608 [namespace + '%']) - return result_set.rowcount # type: ignore + return result_set.rows_affected # type: ignore elif key: result_set = await self._make_request(f"DELETE FROM {self.table_name} WHERE key = ?", # noqa: S608 [key]) - return result_set.rowcount # type: ignore + return result_set.rows_affected # type: ignore return 0