From d196da017742cb451734b97e14b64c97d153f393 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Sat, 20 Jun 2026 23:00:05 +0100 Subject: [PATCH 1/4] Support attaching SQLite files residing on S3 When DuckDB file on S3 is attached as read-only, the remote file system instance (`S3FileSystem` from `httpfs` extension) is used to perform the remote scans over that file. This does not work for SQLite files, as the file path is passed to `sqlite3_open_v2` that doesn't know about remote filesystems. As a remote `ATTACH` is supposed to be read-only, a possible workaround is to download the whole `.sqlite` file locally and then attach it as readonly. This PR implements the auto-download for remote SQLite files (`https://`, `s3://`, `abfss://` etc) into a system temp directory and open the file from there (file is deleted on `DETACH`). It is intended to be used with the files of reasonable size in scenarios, when manual download is not possible/convenient. For example: opening a remote DuckLake catalog (residing on S3) from a BI tool. Testing: new test is added that opens an SQLite file from GitHub; also tested manualy DuckLake catalogs on `s3://` and `abfss://`. Fixes: duckdb/ducklake#912 --- src/include/sqlite_db.hpp | 13 ++++++ src/include/sqlite_utils.hpp | 2 + src/include/storage/sqlite_catalog.hpp | 5 ++- src/sqlite_storage.cpp | 56 +++++++++++++++++++++++++- src/sqlite_utils.cpp | 51 +++++++++++++++++++++++ src/storage/sqlite_catalog.cpp | 14 ++++++- test/sql/storage/attach_remote.test | 24 +++++++++++ 7 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 test/sql/storage/attach_remote.test diff --git a/src/include/sqlite_db.hpp b/src/include/sqlite_db.hpp index 8c156b2..b41871e 100644 --- a/src/include/sqlite_db.hpp +++ b/src/include/sqlite_db.hpp @@ -15,6 +15,19 @@ namespace duckdb { class SQLiteStatement; struct IndexInfo; +struct SQLiteDBLocation { + string temp_directory; + string db_file_path; + + explicit SQLiteDBLocation(std::string db_file_path_p) + : db_file_path(std::move(db_file_path_p)) {} + + explicit SQLiteDBLocation(std::string temp_directory_p, + std::string db_file_path_p) + : temp_directory(std::move(temp_directory_p)), + db_file_path(std::move(db_file_path_p)) {} +}; + class SQLiteDB { public: SQLiteDB(); diff --git a/src/include/sqlite_utils.hpp b/src/include/sqlite_utils.hpp index 28414eb..b5fedd9 100644 --- a/src/include/sqlite_utils.hpp +++ b/src/include/sqlite_utils.hpp @@ -22,6 +22,8 @@ class SQLiteUtils { static string SanitizeIdentifier(const string &table_name); static LogicalType ToSQLiteType(const LogicalType &input); string ToSQLiteTypeAlias(const LogicalType &input); + static string GetSystemTempDirectory(FileSystem &local_file_system); + static string GenerateRandomDirName(const string &prefix); }; struct RowIdInfo { diff --git a/src/include/storage/sqlite_catalog.hpp b/src/include/storage/sqlite_catalog.hpp index 0bf1ca2..b936995 100644 --- a/src/include/storage/sqlite_catalog.hpp +++ b/src/include/storage/sqlite_catalog.hpp @@ -17,7 +17,7 @@ class SQLiteSchemaEntry; class SQLiteCatalog : public Catalog { public: - explicit SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options); + explicit SQLiteCatalog(AttachedDatabase &db_p, SQLiteDBLocation location, SQLiteOpenOptions options); ~SQLiteCatalog(); string path; @@ -76,6 +76,9 @@ class SQLiteCatalog : public Catalog { mutex in_memory_lock; //! Whether or not there is any active transaction on the in-memory database bool active_in_memory; + //! Temporary directory where the file resides, created by us (if not empty), + //! must be deleted on detach + string temporary_directory; }; } // namespace duckdb diff --git a/src/sqlite_storage.cpp b/src/sqlite_storage.cpp index 38255f6..f94c298 100644 --- a/src/sqlite_storage.cpp +++ b/src/sqlite_storage.cpp @@ -1,6 +1,7 @@ #include "duckdb.hpp" #include "sqlite3.h" +#include "sqlite_db.hpp" #include "sqlite_utils.hpp" #include "sqlite_storage.hpp" #include "storage/sqlite_catalog.hpp" @@ -12,6 +13,58 @@ namespace duckdb { +static string CopyDBFile(FileSystem &local_fs, unique_ptr fh, const string &sqlite_dir, const string &file_path, const string &attached_name) { + auto filename = local_fs.ExtractName(file_path); + if (filename.empty()) { + filename = attached_name + ".sqlite"; + } + auto dst_file_path = local_fs.JoinPath(sqlite_dir, filename); + auto dst = local_fs.OpenFile(dst_file_path, + FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE); + if (!dst) { + throw IOException("failed to open target file for writing: %s", dst_file_path); + } + std::vector buffer; + buffer.resize(4 * 1024 * 1024); // 4MB large buffer + while (true) { + int64_t read = fh->Read(buffer.data(), buffer.size()); // returns bytes read, 0 on EOF + if (read <= 0) { + break; + } + dst->Write(buffer.data(), read); + } + dst->Sync(); + return dst_file_path; +} + +static SQLiteDBLocation GetDBLocation(AttachedDatabase &db, ClientContext &context, const string &path, const string &attached_name, const SQLiteOpenOptions &options) { + if (options.access_mode != AccessMode::READ_ONLY) { + return SQLiteDBLocation(path); + } + auto &fs = FileSystem::Get(db); + auto fh = fs.OpenFile(path, FileOpenFlags::FILE_FLAGS_READ); + if (!fh || fh->OnDiskFile()) { + return SQLiteDBLocation(path); + } + auto &db_instance = DatabaseInstance::GetDatabase(context); + auto &local_fs = FileSystem::GetLocal(db_instance); + string sqlite_dir; + try { + auto sys_tmp_dir = SQLiteUtils::GetSystemTempDirectory(local_fs); + auto dirname = SQLiteUtils::GenerateRandomDirName("duckdb_sqlite"); + sqlite_dir = local_fs.JoinPath(sys_tmp_dir, dirname); + local_fs.CreateDirectory(sqlite_dir); + auto dst_file_path = CopyDBFile(local_fs, std::move(fh), sqlite_dir, path, attached_name); + return SQLiteDBLocation(sqlite_dir, dst_file_path); + } catch(const IOException &ex) { + if (!sqlite_dir.empty()) { + local_fs.RemoveDirectory(sqlite_dir); + } + ErrorData error_data(ex); + throw IOException("Unable to make a local copy of a remote file \"%s\": %s", path, error_data.RawMessage()); + } +} + static unique_ptr SQLiteAttach(optional_ptr storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AttachOptions &attach_options) { @@ -26,7 +79,8 @@ static unique_ptr SQLiteAttach(optional_ptr stora throw NotImplementedException("Unsupported parameter for SQLite Attach: %s", entry.first); } } - return make_uniq(db, info.path, std::move(options)); + auto location = GetDBLocation(db, context, info.path, name, options); + return make_uniq(db, std::move(location), std::move(options)); } static unique_ptr SQLiteCreateTransactionManager(optional_ptr storage_info, diff --git a/src/sqlite_utils.cpp b/src/sqlite_utils.cpp index 58e1269..983e218 100644 --- a/src/sqlite_utils.cpp +++ b/src/sqlite_utils.cpp @@ -113,4 +113,55 @@ LogicalType SQLiteUtils::TypeToLogicalType(const string &sqlite_type) { return LogicalType::VARCHAR; } +static void AppendEnvVarValue(vector &vec, const string &name) { + const char *val = std::getenv(name.c_str()); + if (val != nullptr && *val != '\0') { + string val_str(val); + vec.emplace_back(std::move(val_str)); + } +} + +static vector TempDirCandidates() { + vector res; +#ifdef _WIN32 + vector buf; + buf.resize(MAX_PATH + 2); + auto len = GetTempPath2W(static_cast(buf.size()), buf.data()); + if (len > 0) { // The maximum possible return value is MAX_PATH+1 (261). + buf.resize(len); + buf[len + 1] = '\0'; + string dirname = WindowsUtil::UnicodeToUTF8(buf); + res.emplace_back(std::move(dirname)); + } else { + AppendEnvVarValue(res, "TEMP"); + AppendEnvVarValue(res, "TMP"); + } +#else // !_WIN32 + AppendEnvVarValue(res, "TMPDIR"); + AppendEnvVarValue(res, "TMP"); + res.emplace_back("/tmp"); + res.emplace_back("/var/tmp"); + res.emplace_back("."); +#endif // _WIN32 + return res; +} + +string SQLiteUtils::GetSystemTempDirectory(FileSystem &fs) { + for (const auto &dir : TempDirCandidates()) { + if (fs.DirectoryExists(dir)) { + return fs.CanonicalizePath(dir); + } + } + throw IOException("Unable to determine the OS temporary directory, setting " + "TMP environment variable may resolve this"); +} + +string SQLiteUtils::GenerateRandomDirName(const string &prefix) { + auto now = std::chrono::system_clock::now().time_since_epoch().count(); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 100000); + return prefix + "_" + std::to_string(now) + "_" + std::to_string(dis(gen)); +} + } // namespace duckdb diff --git a/src/storage/sqlite_catalog.cpp b/src/storage/sqlite_catalog.cpp index b01a247..10eae6e 100644 --- a/src/storage/sqlite_catalog.cpp +++ b/src/storage/sqlite_catalog.cpp @@ -5,17 +5,27 @@ #include "storage/sqlite_schema_entry.hpp" #include "storage/sqlite_transaction.hpp" #include "duckdb/common/exception/transaction_exception.hpp" +#include "duckdb/common/local_file_system.hpp" namespace duckdb { -SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options_p) - : Catalog(db_p), path(path), options(std::move(options_p)), in_memory(path == ":memory:"), active_in_memory(false) { +SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, SQLiteDBLocation location, SQLiteOpenOptions options_p) + : Catalog(db_p), path(std::move(location.db_file_path)), options(std::move(options_p)), in_memory(path == ":memory:"), active_in_memory(false), + temporary_directory(std::move(location.temp_directory)) { if (InMemory()) { in_memory_db = SQLiteDB::Open(path, options, true); } } SQLiteCatalog::~SQLiteCatalog() { + try { + if (!temporary_directory.empty()) { + LocalFileSystem fs; + fs.RemoveDirectory(temporary_directory); + } + } catch (...) { + // suppress + } } void SQLiteCatalog::Initialize(bool load_builtin) { diff --git a/test/sql/storage/attach_remote.test b/test/sql/storage/attach_remote.test new file mode 100644 index 0000000..d27b638 --- /dev/null +++ b/test/sql/storage/attach_remote.test @@ -0,0 +1,24 @@ +# name: test/sql/storage/attach_remote.test +# description: test attaching remote DB (it is downloaded automatically) +# group: [storage] + +require sqlite_scanner + +require httpfs + +statement ok +ATTACH 'https://github.com/duckdb/sqlite_scanner/raw/main/data/db/sakila.db' + +statement ok +USE sakila + +query I +SELECT COUNT(*) FROM actor +---- +200 + +statement ok +USE memory + +statement ok +DETACH sakila From d1460714452b81e72757e81f0983932996e0f46b Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Sat, 20 Jun 2026 23:32:16 +0100 Subject: [PATCH 2/4] Missed includes fix --- src/sqlite_utils.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlite_utils.cpp b/src/sqlite_utils.cpp index 983e218..a2dd8c6 100644 --- a/src/sqlite_utils.cpp +++ b/src/sqlite_utils.cpp @@ -1,5 +1,8 @@ #include "sqlite_utils.hpp" +#include +#include + namespace duckdb { void SQLiteUtils::Check(int rc, sqlite3 *db) { From 33f83561d6ff4c8b20af015e154b240ac5550e89 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Sat, 20 Jun 2026 23:36:35 +0100 Subject: [PATCH 3/4] Disable HTTPFS test on main --- test/sql/storage/attach_remote.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sql/storage/attach_remote.test b/test/sql/storage/attach_remote.test index d27b638..d03f57f 100644 --- a/test/sql/storage/attach_remote.test +++ b/test/sql/storage/attach_remote.test @@ -2,6 +2,8 @@ # description: test attaching remote DB (it is downloaded automatically) # group: [storage] +require-env SQLITE_TEST_HTTPFS_AVAILABLE + require sqlite_scanner require httpfs From e2480864a7faa114945015c5d58cb8e35ed1b2f0 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Mon, 22 Jun 2026 10:26:04 +0100 Subject: [PATCH 4/4] Windows header fix --- src/sqlite_utils.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/sqlite_utils.cpp b/src/sqlite_utils.cpp index a2dd8c6..d87e293 100644 --- a/src/sqlite_utils.cpp +++ b/src/sqlite_utils.cpp @@ -3,6 +3,13 @@ #include #include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#include +#include "duckdb/common/windows_util.hpp" +#endif // _WIN32 + namespace duckdb { void SQLiteUtils::Check(int rc, sqlite3 *db) { @@ -131,9 +138,7 @@ static vector TempDirCandidates() { buf.resize(MAX_PATH + 2); auto len = GetTempPath2W(static_cast(buf.size()), buf.data()); if (len > 0) { // The maximum possible return value is MAX_PATH+1 (261). - buf.resize(len); - buf[len + 1] = '\0'; - string dirname = WindowsUtil::UnicodeToUTF8(buf); + string dirname = WindowsUtil::UnicodeToUTF8(buf.data()); res.emplace_back(std::move(dirname)); } else { AppendEnvVarValue(res, "TEMP");