Support attaching SQLite files residing on S3

src/include/sqlite_db.hpp

@@ -15,6 +15,19 @@ namespace duckdb {
 class SQLiteStatement;
 struct IndexInfo;
 
+struct SQLiteDBLocation {
+	string temp_directory;
+	string db_file_path;
+
+	explicit SQLiteDBLocation(std::string db_file_path_p)
+			: db_file_path(std::move(db_file_path_p)) {}
+
+	explicit SQLiteDBLocation(std::string temp_directory_p,
+														std::string db_file_path_p)
+			: temp_directory(std::move(temp_directory_p)),
+				db_file_path(std::move(db_file_path_p)) {}
+};
+
 class SQLiteDB {
 public:
 	SQLiteDB();

src/include/sqlite_utils.hpp

@@ -22,6 +22,8 @@ class SQLiteUtils {
 	static string SanitizeIdentifier(const string &table_name);
 	static LogicalType ToSQLiteType(const LogicalType &input);
 	string ToSQLiteTypeAlias(const LogicalType &input);
+  static string GetSystemTempDirectory(FileSystem &local_file_system);
+  static string GenerateRandomDirName(const string &prefix);
 };
 
 struct RowIdInfo {

src/include/storage/sqlite_catalog.hpp

@@ -17,7 +17,7 @@ class SQLiteSchemaEntry;
 
 class SQLiteCatalog : public Catalog {
 public:
-	explicit SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options);
+	explicit SQLiteCatalog(AttachedDatabase &db_p, SQLiteDBLocation location, SQLiteOpenOptions options);
 	~SQLiteCatalog();
 
 	string path;
@@ -76,6 +76,9 @@ class SQLiteCatalog : public Catalog {
 	mutex in_memory_lock;
 	//! Whether or not there is any active transaction on the in-memory database
 	bool active_in_memory;
+  //! Temporary directory where the file resides, created by us (if not empty),
+  //! must be deleted on detach
+  string temporary_directory;
 };
 
 } // namespace duckdb

src/sqlite_storage.cpp

@@ -1,6 +1,7 @@
 #include "duckdb.hpp"
 
 #include "sqlite3.h"
+#include "sqlite_db.hpp"
 #include "sqlite_utils.hpp"
 #include "sqlite_storage.hpp"
 #include "storage/sqlite_catalog.hpp"
@@ -12,6 +13,58 @@
 
 namespace duckdb {
 
+static string CopyDBFile(FileSystem &local_fs, unique_ptr<FileHandle> fh, const string &sqlite_dir, const string &file_path, const string &attached_name) {
+	auto filename = local_fs.ExtractName(file_path);
+	if (filename.empty()) {
+		filename = attached_name + ".sqlite";
+	}
+	auto dst_file_path = local_fs.JoinPath(sqlite_dir, filename);
+	auto dst = local_fs.OpenFile(dst_file_path,
+													FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE);
+	if (!dst) {
+			throw IOException("failed to open target file for writing: %s", dst_file_path);
+	}
+	std::vector<char> buffer;
+	buffer.resize(4 * 1024 * 1024); // 4MB large buffer
+	while (true) {
+			int64_t read = fh->Read(buffer.data(), buffer.size()); // returns bytes read, 0 on EOF
+			if (read <= 0) {
+				break;
+			}
+			dst->Write(buffer.data(), read);
+	}
+	dst->Sync();
+	return dst_file_path;
+}
+
+static SQLiteDBLocation GetDBLocation(AttachedDatabase &db, ClientContext &context, const string &path, const string &attached_name, const SQLiteOpenOptions &options) {
+	if (options.access_mode != AccessMode::READ_ONLY) {
+		return SQLiteDBLocation(path);
+	}
+	auto &fs = FileSystem::Get(db);
+	auto fh = fs.OpenFile(path, FileOpenFlags::FILE_FLAGS_READ);
+	if (!fh || fh->OnDiskFile()) {
+		return SQLiteDBLocation(path);
+	}
+	auto &db_instance = DatabaseInstance::GetDatabase(context);
+	auto &local_fs = FileSystem::GetLocal(db_instance);
+	string sqlite_dir;
+	try {
+		auto sys_tmp_dir = SQLiteUtils::GetSystemTempDirectory(local_fs);
+		auto dirname = SQLiteUtils::GenerateRandomDirName("duckdb_sqlite");
+		sqlite_dir = local_fs.JoinPath(sys_tmp_dir, dirname);
+		local_fs.CreateDirectory(sqlite_dir);
+		auto dst_file_path = CopyDBFile(local_fs, std::move(fh), sqlite_dir, path, attached_name);
+		return SQLiteDBLocation(sqlite_dir, dst_file_path);
+	} catch(const IOException &ex) {
+		if (!sqlite_dir.empty()) {
+			local_fs.RemoveDirectory(sqlite_dir);
+		}
+		ErrorData error_data(ex);
+		throw IOException("Unable to make a local copy of a remote file \"%s\": %s", path, error_data.RawMessage());
+	}
+}
+
 static unique_ptr<Catalog> SQLiteAttach(optional_ptr<StorageExtensionInfo> storage_info, ClientContext &context,
                                         AttachedDatabase &db, const string &name, AttachInfo &info,
                                         AttachOptions &attach_options) {
@@ -26,7 +79,8 @@ static unique_ptr<Catalog> SQLiteAttach(optional_ptr<StorageExtensionInfo> stora
 			throw NotImplementedException("Unsupported parameter for SQLite Attach: %s", entry.first);
 		}
 	}
-	return make_uniq<SQLiteCatalog>(db, info.path, std::move(options));
+	auto location = GetDBLocation(db, context, info.path, name, options);
+	return make_uniq<SQLiteCatalog>(db, std::move(location), std::move(options));
 }
 
 static unique_ptr<TransactionManager> SQLiteCreateTransactionManager(optional_ptr<StorageExtensionInfo> storage_info,

src/sqlite_utils.cpp

@@ -1,5 +1,15 @@
 #include "sqlite_utils.hpp"
 
+#include <chrono>
+#include <random>
+
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <fileapi.h>
+#include "duckdb/common/windows_util.hpp"
+#endif // _WIN32
+
 namespace duckdb {
 
 void SQLiteUtils::Check(int rc, sqlite3 *db) {
@@ -113,4 +123,53 @@ LogicalType SQLiteUtils::TypeToLogicalType(const string &sqlite_type) {
 	return LogicalType::VARCHAR;
 }
 
+static void AppendEnvVarValue(vector<string> &vec, const string &name) {
+	const char *val = std::getenv(name.c_str());
+	if (val != nullptr && *val != '\0') {
+		string val_str(val);
+		vec.emplace_back(std::move(val_str));
+	}
+}
+
+static vector<string> TempDirCandidates() {
+	vector<string> res;
+#ifdef _WIN32
+	vector<wchar_t> buf;
+	buf.resize(MAX_PATH + 2);
+	auto len = GetTempPath2W(static_cast<DWORD>(buf.size()), buf.data());
+	if (len > 0) { // The maximum possible return value is MAX_PATH+1 (261).
+		string dirname = WindowsUtil::UnicodeToUTF8(buf.data());
+		res.emplace_back(std::move(dirname));
+	} else {
+		AppendEnvVarValue(res, "TEMP");
+		AppendEnvVarValue(res, "TMP");
+	}
+#else  // !_WIN32
+	AppendEnvVarValue(res, "TMPDIR");
+	AppendEnvVarValue(res, "TMP");
+	res.emplace_back("/tmp");
+	res.emplace_back("/var/tmp");
+	res.emplace_back(".");
+#endif // _WIN32
+	return res;
+}
+
+string SQLiteUtils::GetSystemTempDirectory(FileSystem &fs) {
+	for (const auto &dir : TempDirCandidates()) {
+		if (fs.DirectoryExists(dir)) {
+			return fs.CanonicalizePath(dir);
+		}
+	}
+	throw IOException("Unable to determine the OS temporary directory, setting "
+										"TMP environment variable may resolve this");
+}
+
+string SQLiteUtils::GenerateRandomDirName(const string &prefix) {
+	auto now = std::chrono::system_clock::now().time_since_epoch().count();
+	std::random_device rd;
+	std::mt19937 gen(rd());
+	std::uniform_int_distribution<> dis(0, 100000);
+	return prefix + "_" + std::to_string(now) + "_" + std::to_string(dis(gen));
+}
+
 } // namespace duckdb

src/storage/sqlite_catalog.cpp

@@ -5,17 +5,27 @@
 #include "storage/sqlite_schema_entry.hpp"
 #include "storage/sqlite_transaction.hpp"
 #include "duckdb/common/exception/transaction_exception.hpp"
+#include "duckdb/common/local_file_system.hpp"
 
 namespace duckdb {
 
-SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options_p)
-    : Catalog(db_p), path(path), options(std::move(options_p)), in_memory(path == ":memory:"), active_in_memory(false) {
+SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, SQLiteDBLocation location, SQLiteOpenOptions options_p)
+	: Catalog(db_p), path(std::move(location.db_file_path)), options(std::move(options_p)), in_memory(path == ":memory:"), active_in_memory(false),
+		temporary_directory(std::move(location.temp_directory)) {
 	if (InMemory()) {
 		in_memory_db = SQLiteDB::Open(path, options, true);
 	}
 }
 
 SQLiteCatalog::~SQLiteCatalog() {
+	try {
+		if (!temporary_directory.empty()) {
+			LocalFileSystem fs;
+			fs.RemoveDirectory(temporary_directory);
+		} 
+	} catch (...) {
+		// suppress
+	}
 }
 
 void SQLiteCatalog::Initialize(bool load_builtin) {

test/sql/storage/attach_remote.test

@@ -0,0 +1,26 @@
+# name: test/sql/storage/attach_remote.test
+# description: test attaching remote DB (it is downloaded automatically)
+# group: [storage]
+
+require-env SQLITE_TEST_HTTPFS_AVAILABLE
+
+require sqlite_scanner
+
+require httpfs
+
+statement ok
+ATTACH 'https://github.com/duckdb/sqlite_scanner/raw/main/data/db/sakila.db'
+
+statement ok
+USE sakila
+
+query I
+SELECT COUNT(*) FROM actor
+----
+200
+
+statement ok
+USE memory
+
+statement ok
+DETACH sakila