From 7f31c10e586ba8c40a995ea6e0b4f7d8393484b3 Mon Sep 17 00:00:00 2001 From: "Joshua (D) Drake" <136637981+ChronicallyJD@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:47:22 -0600 Subject: [PATCH] security: derive a per-file salt for temporary-file encryption (H3) Temporary-file encryption derived the AES-XTS tweak (and the CTR IV for a sub-block tail) from a cluster-global base IV and the buffer's block position. The block position resets to zero in every new BufFile, so two different temporary files encrypted identical plaintext blocks under the same (key, tweak): an equality oracle across files for the XTS body and a two-time pad for any CTR tail. Core now assigns each BufFile a 16-byte salt and passes it to the temporary-file encryption hooks, which mix it into the tweak/IV ahead of the block position. A private temp file gets a random salt; a fileset based BufFile (shared by parallel workers) derives its salt deterministically from the fileset identity and name so the writer and every reader agree without communicating it. Temporary keys are per-boot and never persisted, so there is no on-disk compatibility concern. Applies to the PostgreSQL 16/17/18 core patch (0002) and the extension. - patches: add crypt_salt to struct BufFile, fill it at create/open, and extend the TempFileCryptHook signature with the salt. - open_pg_tde_tempfile.c: mix the salt into the XTS tweak / CTR IV. - test: two identical spilling sorts must now produce distinct ciphertext, with an encryption-off control and a parallel shared-fileset round trip. - docs: note per-file uniqueness in the encryption architecture page. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../architecture/encryption-architecture.md | 2 +- ...002-open_pg_tde-temp-file-encryption.patch | 159 ++++++++++++++++-- ...002-open_pg_tde-temp-file-encryption.patch | 159 ++++++++++++++++-- ...002-open_pg_tde-temp-file-encryption.patch | 159 ++++++++++++++++-- src/open_pg_tde_tempfile.c | 59 +++++-- t/temp_file_encryption.pl | 99 ++++++++++- 6 files changed, 587 insertions(+), 50 deletions(-) diff --git a/documentation/docs/architecture/encryption-architecture.md b/documentation/docs/architecture/encryption-architecture.md index 7c567f6..0d0e0d3 100644 --- a/documentation/docs/architecture/encryption-architecture.md +++ b/documentation/docs/architecture/encryption-architecture.md @@ -27,7 +27,7 @@ A table with 4 indexes will have at least 5 internal keys, one for the table and * `AES-128-CTR`, `AES-256-CTR` for WAL encryption; encrypted with internal keys * `AES-128-GCM`, `AES-256-GCM` for encrypting internal keys; encrypted with the principal key -Temporary files produced by query execution, such as external sorts and hash joins that exceed `work_mem` and spill to disk, are encrypted with `AES-128-XTS` when the core [`encrypt_temp_files`](../variables.md#encrypt_temp_files) GUC is on. The key is generated per boot, held only in memory, and never written to disk, so temporary files cannot be recovered once the server stops. +Temporary files produced by query execution, such as external sorts and hash joins that exceed `work_mem` and spill to disk, are encrypted with `AES-128-XTS` when the core [`encrypt_temp_files`](../variables.md#encrypt_temp_files) GUC is on. The key is generated per boot, held only in memory, and never written to disk, so temporary files cannot be recovered once the server stops. Each temporary file mixes a value unique to that file into the cipher tweak, so two files that hold the same data do not produce the same ciphertext. ## Pluggable cipher providers diff --git a/patches/postgresql/16/0002-open_pg_tde-temp-file-encryption.patch b/patches/postgresql/16/0002-open_pg_tde-temp-file-encryption.patch index 2436f51..af02aa2 100644 --- a/patches/postgresql/16/0002-open_pg_tde-temp-file-encryption.patch +++ b/patches/postgresql/16/0002-open_pg_tde-temp-file-encryption.patch @@ -1,16 +1,145 @@ --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c -@@ -52,6 +52,9 @@ +@@ -52,6 +52,10 @@ #include "storage/buf_internals.h" #include "storage/buffile.h" #include "storage/fd.h" +#ifdef USE_TDE_HOOKS ++#include "common/hashfn.h" +#include "storage/tempfile_crypt.h" +#endif #include "utils/resowner.h" /* -@@ -479,6 +482,17 @@ +@@ -95,6 +99,16 @@ + off_t curOffset; /* offset part of current pos */ + int pos; /* next read/write position in buffer */ + int nbytes; /* total # of valid bytes in buffer */ ++#ifdef USE_TDE_HOOKS ++ ++ /* ++ * Per-BufFile salt mixed into the temporary-file encryption IV so that ++ * two different BufFiles never encrypt under the same (key, IV/tweak) ++ * pair. Filled once at create/open time when encrypt_temp_files is on. ++ * See BufFileInitCryptSalt. ++ */ ++ unsigned char crypt_salt[TEMPFILE_CRYPT_SALT_LEN]; ++#endif + + /* + * XXX Should ideally us PGIOAlignedBlock, but might need a way to avoid +@@ -110,6 +124,9 @@ + static void BufFileDumpBuffer(BufFile *file); + static void BufFileFlush(BufFile *file); + static File MakeNewFileSetSegment(BufFile *buffile, int segment); ++#ifdef USE_TDE_HOOKS ++static void BufFileInitCryptSalt(BufFile *file); ++#endif + + /* + * Create BufFile and perform the common initialization. +@@ -131,6 +148,70 @@ + return file; + } + ++#ifdef USE_TDE_HOOKS ++/* ++ * Assign this BufFile's temporary-file encryption salt. Two BufFiles must ++ * never share a salt: with a cluster-global key, a repeated salt would repeat ++ * the (key, IV/tweak) pair, which is an equality oracle across files for the ++ * XTS body and a two-time pad for any sub-block CTR tail. Only meaningful when ++ * encrypt_temp_files is on. ++ * ++ * A private temp file is read back only by the backend that created it, so a ++ * random salt is unique and never has to be shared. A fileset based BufFile ++ * can be written by one backend and read by others (for example parallel ++ * workers sharing a hash-join spill), so its salt is derived deterministically ++ * from the fileset identity and the BufFile name -- values every participant ++ * already knows -- so the writer and every reader arrive at the same salt ++ * without communicating it. ++ */ ++static void ++BufFileInitCryptSalt(BufFile *file) ++{ ++ if (!encrypt_temp_files) ++ return; ++ ++ if (file->fileset == NULL) ++ { ++ /* Private temp file: a random salt is unique and never shared. */ ++ if (!pg_strong_random(file->crypt_salt, TEMPFILE_CRYPT_SALT_LEN)) ++ elog(ERROR, "could not generate temporary file encryption salt"); ++ } ++ else ++ { ++ FileSet *fileset = file->fileset; ++ char idbuf[MAXPGPATH + sizeof(fileset->creator_pid) + ++ sizeof(fileset->number)]; ++ size_t idlen = 0; ++ size_t namelen = strlen(file->name); ++ uint64 h1; ++ uint64 h2; ++ ++ /* ++ * (creator_pid, number) identifies the fileset across the cluster and ++ * name identifies the BufFile within it. Two independent 64-bit ++ * hashes of that identity give a 128-bit salt. ++ */ ++ if (namelen > MAXPGPATH) ++ namelen = MAXPGPATH; ++ memcpy(idbuf + idlen, &fileset->creator_pid, ++ sizeof(fileset->creator_pid)); ++ idlen += sizeof(fileset->creator_pid); ++ memcpy(idbuf + idlen, &fileset->number, sizeof(fileset->number)); ++ idlen += sizeof(fileset->number); ++ memcpy(idbuf + idlen, file->name, namelen); ++ idlen += namelen; ++ ++ StaticAssertStmt(TEMPFILE_CRYPT_SALT_LEN == sizeof(h1) + sizeof(h2), ++ "temp-file salt must hold two 64-bit hashes"); ++ h1 = hash_bytes_extended((const unsigned char *) idbuf, idlen, 0); ++ h2 = hash_bytes_extended((const unsigned char *) idbuf, idlen, ++ UINT64CONST(0x9e3779b97f4a7c15)); ++ memcpy(file->crypt_salt, &h1, sizeof(h1)); ++ memcpy(file->crypt_salt + sizeof(h1), &h2, sizeof(h2)); ++ } ++} ++#endif ++ + /* + * Create a BufFile given the first underlying physical file. + * NOTE: caller must set isInterXact if appropriate. +@@ -145,6 +226,9 @@ + file->readOnly = false; + file->fileset = NULL; + file->name = NULL; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -274,6 +358,9 @@ + file->files = (File *) palloc(sizeof(File)); + file->files[0] = MakeNewFileSetSegment(file, 0); + file->readOnly = false; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -344,6 +431,9 @@ + file->readOnly = (mode == O_RDONLY); + file->fileset = fileset; + file->name = pstrdup(name); ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -479,6 +569,18 @@ /* we choose not to advance curOffset here */ @@ -21,14 +150,15 @@ + int64 blocknum = (int64) file->curFile * BUFFILE_SEG_SIZE + + file->curOffset / BLCKSZ; + -+ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum); ++ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum, ++ file->crypt_salt); + } +#endif + if (file->nbytes > 0) pgBufferUsage.temp_blks_read++; } -@@ -496,6 +510,26 @@ +@@ -496,6 +598,27 @@ int wpos = 0; int bytestowrite; File thisfile; @@ -48,14 +178,15 @@ + file->curOffset / BLCKSZ; + + memcpy(cryptbuf.data, file->buffer.data, file->nbytes); -+ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum); ++ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum, ++ file->crypt_salt); + srcdata = cryptbuf.data; + } +#endif /* * Unlike BufFileLoadBuffer, we must dump the whole buffer even if it -@@ -535,7 +569,11 @@ +@@ -535,7 +658,11 @@ INSTR_TIME_SET_ZERO(io_start); bytestowrite = FileWrite(thisfile, @@ -148,7 +279,7 @@ +#endif /* USE_TDE_HOOKS */ --- a/src/include/storage/tempfile_crypt.h +++ b/src/include/storage/tempfile_crypt.h -@@ -0,0 +1,40 @@ +@@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * tempfile_crypt.h @@ -176,12 +307,20 @@ +/* GUC: master switch for temporary file encryption (default off). */ +extern PGDLLIMPORT bool encrypt_temp_files; + ++/* Length of the per-BufFile salt the hooks mix into the IV (bytes). */ ++#define TEMPFILE_CRYPT_SALT_LEN 16 ++ +/* + * Encrypt or decrypt one temporary-file buffer in place. blocknum is the -+ * logical block position of the buffer within the BufFile; an implementation -+ * uses it to derive a per-block IV. The transformation must preserve length. ++ * logical block position of the buffer within the BufFile; file_iv_salt is a ++ * per-BufFile value (TEMPFILE_CRYPT_SALT_LEN bytes) that is unique to each ++ * BufFile. An implementation must mix both into the IV: blocknum distinguishes ++ * blocks within one file, and file_iv_salt keeps two different BufFiles from ++ * ever repeating an (IV/tweak), which would otherwise leak equality across ++ * files. The transformation must preserve length. + */ -+typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum); ++typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum, ++ const unsigned char *file_iv_salt); + +extern PGDLLIMPORT TempFileCryptHook temp_file_encrypt_hook; +extern PGDLLIMPORT TempFileCryptHook temp_file_decrypt_hook; diff --git a/patches/postgresql/17/0002-open_pg_tde-temp-file-encryption.patch b/patches/postgresql/17/0002-open_pg_tde-temp-file-encryption.patch index 57a6948..15f251e 100644 --- a/patches/postgresql/17/0002-open_pg_tde-temp-file-encryption.patch +++ b/patches/postgresql/17/0002-open_pg_tde-temp-file-encryption.patch @@ -1,16 +1,145 @@ --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c -@@ -52,6 +52,9 @@ +@@ -52,6 +52,10 @@ #include "storage/buffile.h" #include "storage/bufmgr.h" #include "storage/fd.h" +#ifdef USE_TDE_HOOKS ++#include "common/hashfn.h" +#include "storage/tempfile_crypt.h" +#endif #include "utils/resowner.h" /* -@@ -479,6 +482,17 @@ +@@ -95,6 +99,16 @@ + off_t curOffset; /* offset part of current pos */ + int pos; /* next read/write position in buffer */ + int nbytes; /* total # of valid bytes in buffer */ ++#ifdef USE_TDE_HOOKS ++ ++ /* ++ * Per-BufFile salt mixed into the temporary-file encryption IV so that ++ * two different BufFiles never encrypt under the same (key, IV/tweak) ++ * pair. Filled once at create/open time when encrypt_temp_files is on. ++ * See BufFileInitCryptSalt. ++ */ ++ unsigned char crypt_salt[TEMPFILE_CRYPT_SALT_LEN]; ++#endif + + /* + * XXX Should ideally us PGIOAlignedBlock, but might need a way to avoid +@@ -110,6 +124,9 @@ + static void BufFileDumpBuffer(BufFile *file); + static void BufFileFlush(BufFile *file); + static File MakeNewFileSetSegment(BufFile *buffile, int segment); ++#ifdef USE_TDE_HOOKS ++static void BufFileInitCryptSalt(BufFile *file); ++#endif + + /* + * Create BufFile and perform the common initialization. +@@ -131,6 +148,70 @@ + return file; + } + ++#ifdef USE_TDE_HOOKS ++/* ++ * Assign this BufFile's temporary-file encryption salt. Two BufFiles must ++ * never share a salt: with a cluster-global key, a repeated salt would repeat ++ * the (key, IV/tweak) pair, which is an equality oracle across files for the ++ * XTS body and a two-time pad for any sub-block CTR tail. Only meaningful when ++ * encrypt_temp_files is on. ++ * ++ * A private temp file is read back only by the backend that created it, so a ++ * random salt is unique and never has to be shared. A fileset based BufFile ++ * can be written by one backend and read by others (for example parallel ++ * workers sharing a hash-join spill), so its salt is derived deterministically ++ * from the fileset identity and the BufFile name -- values every participant ++ * already knows -- so the writer and every reader arrive at the same salt ++ * without communicating it. ++ */ ++static void ++BufFileInitCryptSalt(BufFile *file) ++{ ++ if (!encrypt_temp_files) ++ return; ++ ++ if (file->fileset == NULL) ++ { ++ /* Private temp file: a random salt is unique and never shared. */ ++ if (!pg_strong_random(file->crypt_salt, TEMPFILE_CRYPT_SALT_LEN)) ++ elog(ERROR, "could not generate temporary file encryption salt"); ++ } ++ else ++ { ++ FileSet *fileset = file->fileset; ++ char idbuf[MAXPGPATH + sizeof(fileset->creator_pid) + ++ sizeof(fileset->number)]; ++ size_t idlen = 0; ++ size_t namelen = strlen(file->name); ++ uint64 h1; ++ uint64 h2; ++ ++ /* ++ * (creator_pid, number) identifies the fileset across the cluster and ++ * name identifies the BufFile within it. Two independent 64-bit ++ * hashes of that identity give a 128-bit salt. ++ */ ++ if (namelen > MAXPGPATH) ++ namelen = MAXPGPATH; ++ memcpy(idbuf + idlen, &fileset->creator_pid, ++ sizeof(fileset->creator_pid)); ++ idlen += sizeof(fileset->creator_pid); ++ memcpy(idbuf + idlen, &fileset->number, sizeof(fileset->number)); ++ idlen += sizeof(fileset->number); ++ memcpy(idbuf + idlen, file->name, namelen); ++ idlen += namelen; ++ ++ StaticAssertStmt(TEMPFILE_CRYPT_SALT_LEN == sizeof(h1) + sizeof(h2), ++ "temp-file salt must hold two 64-bit hashes"); ++ h1 = hash_bytes_extended((const unsigned char *) idbuf, idlen, 0); ++ h2 = hash_bytes_extended((const unsigned char *) idbuf, idlen, ++ UINT64CONST(0x9e3779b97f4a7c15)); ++ memcpy(file->crypt_salt, &h1, sizeof(h1)); ++ memcpy(file->crypt_salt + sizeof(h1), &h2, sizeof(h2)); ++ } ++} ++#endif ++ + /* + * Create a BufFile given the first underlying physical file. + * NOTE: caller must set isInterXact if appropriate. +@@ -145,6 +226,9 @@ + file->readOnly = false; + file->fileset = NULL; + file->name = NULL; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -274,6 +358,9 @@ + file->files = (File *) palloc(sizeof(File)); + file->files[0] = MakeNewFileSetSegment(file, 0); + file->readOnly = false; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -344,6 +431,9 @@ + file->readOnly = (mode == O_RDONLY); + file->fileset = fileset; + file->name = pstrdup(name); ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -479,6 +569,18 @@ /* we choose not to advance curOffset here */ @@ -21,14 +150,15 @@ + int64 blocknum = (int64) file->curFile * BUFFILE_SEG_SIZE + + file->curOffset / BLCKSZ; + -+ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum); ++ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum, ++ file->crypt_salt); + } +#endif + if (file->nbytes > 0) pgBufferUsage.temp_blks_read++; } -@@ -496,6 +510,26 @@ +@@ -496,6 +598,27 @@ int wpos = 0; int bytestowrite; File thisfile; @@ -48,14 +178,15 @@ + file->curOffset / BLCKSZ; + + memcpy(cryptbuf.data, file->buffer.data, file->nbytes); -+ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum); ++ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum, ++ file->crypt_salt); + srcdata = cryptbuf.data; + } +#endif /* * Unlike BufFileLoadBuffer, we must dump the whole buffer even if it -@@ -535,7 +569,11 @@ +@@ -535,7 +658,11 @@ INSTR_TIME_SET_ZERO(io_start); bytestowrite = FileWrite(thisfile, @@ -148,7 +279,7 @@ +#endif /* USE_TDE_HOOKS */ --- a/src/include/storage/tempfile_crypt.h +++ b/src/include/storage/tempfile_crypt.h -@@ -0,0 +1,40 @@ +@@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * tempfile_crypt.h @@ -176,12 +307,20 @@ +/* GUC: master switch for temporary file encryption (default off). */ +extern PGDLLIMPORT bool encrypt_temp_files; + ++/* Length of the per-BufFile salt the hooks mix into the IV (bytes). */ ++#define TEMPFILE_CRYPT_SALT_LEN 16 ++ +/* + * Encrypt or decrypt one temporary-file buffer in place. blocknum is the -+ * logical block position of the buffer within the BufFile; an implementation -+ * uses it to derive a per-block IV. The transformation must preserve length. ++ * logical block position of the buffer within the BufFile; file_iv_salt is a ++ * per-BufFile value (TEMPFILE_CRYPT_SALT_LEN bytes) that is unique to each ++ * BufFile. An implementation must mix both into the IV: blocknum distinguishes ++ * blocks within one file, and file_iv_salt keeps two different BufFiles from ++ * ever repeating an (IV/tweak), which would otherwise leak equality across ++ * files. The transformation must preserve length. + */ -+typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum); ++typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum, ++ const unsigned char *file_iv_salt); + +extern PGDLLIMPORT TempFileCryptHook temp_file_encrypt_hook; +extern PGDLLIMPORT TempFileCryptHook temp_file_decrypt_hook; diff --git a/patches/postgresql/18/0002-open_pg_tde-temp-file-encryption.patch b/patches/postgresql/18/0002-open_pg_tde-temp-file-encryption.patch index 5114bae..0bcc494 100644 --- a/patches/postgresql/18/0002-open_pg_tde-temp-file-encryption.patch +++ b/patches/postgresql/18/0002-open_pg_tde-temp-file-encryption.patch @@ -1,16 +1,145 @@ --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c -@@ -52,6 +52,9 @@ +@@ -52,6 +52,10 @@ #include "storage/buffile.h" #include "storage/bufmgr.h" #include "storage/fd.h" +#ifdef USE_TDE_HOOKS ++#include "common/hashfn.h" +#include "storage/tempfile_crypt.h" +#endif #include "utils/resowner.h" /* -@@ -479,6 +482,17 @@ +@@ -95,6 +99,16 @@ + off_t curOffset; /* offset part of current pos */ + int pos; /* next read/write position in buffer */ + int nbytes; /* total # of valid bytes in buffer */ ++#ifdef USE_TDE_HOOKS ++ ++ /* ++ * Per-BufFile salt mixed into the temporary-file encryption IV so that ++ * two different BufFiles never encrypt under the same (key, IV/tweak) ++ * pair. Filled once at create/open time when encrypt_temp_files is on. ++ * See BufFileInitCryptSalt. ++ */ ++ unsigned char crypt_salt[TEMPFILE_CRYPT_SALT_LEN]; ++#endif + + /* + * XXX Should ideally use PGIOAlignedBlock, but might need a way to avoid +@@ -110,6 +124,9 @@ + static void BufFileDumpBuffer(BufFile *file); + static void BufFileFlush(BufFile *file); + static File MakeNewFileSetSegment(BufFile *buffile, int segment); ++#ifdef USE_TDE_HOOKS ++static void BufFileInitCryptSalt(BufFile *file); ++#endif + + /* + * Create BufFile and perform the common initialization. +@@ -131,6 +148,70 @@ + return file; + } + ++#ifdef USE_TDE_HOOKS ++/* ++ * Assign this BufFile's temporary-file encryption salt. Two BufFiles must ++ * never share a salt: with a cluster-global key, a repeated salt would repeat ++ * the (key, IV/tweak) pair, which is an equality oracle across files for the ++ * XTS body and a two-time pad for any sub-block CTR tail. Only meaningful when ++ * encrypt_temp_files is on. ++ * ++ * A private temp file is read back only by the backend that created it, so a ++ * random salt is unique and never has to be shared. A fileset based BufFile ++ * can be written by one backend and read by others (for example parallel ++ * workers sharing a hash-join spill), so its salt is derived deterministically ++ * from the fileset identity and the BufFile name -- values every participant ++ * already knows -- so the writer and every reader arrive at the same salt ++ * without communicating it. ++ */ ++static void ++BufFileInitCryptSalt(BufFile *file) ++{ ++ if (!encrypt_temp_files) ++ return; ++ ++ if (file->fileset == NULL) ++ { ++ /* Private temp file: a random salt is unique and never shared. */ ++ if (!pg_strong_random(file->crypt_salt, TEMPFILE_CRYPT_SALT_LEN)) ++ elog(ERROR, "could not generate temporary file encryption salt"); ++ } ++ else ++ { ++ FileSet *fileset = file->fileset; ++ char idbuf[MAXPGPATH + sizeof(fileset->creator_pid) + ++ sizeof(fileset->number)]; ++ size_t idlen = 0; ++ size_t namelen = strlen(file->name); ++ uint64 h1; ++ uint64 h2; ++ ++ /* ++ * (creator_pid, number) identifies the fileset across the cluster and ++ * name identifies the BufFile within it. Two independent 64-bit ++ * hashes of that identity give a 128-bit salt. ++ */ ++ if (namelen > MAXPGPATH) ++ namelen = MAXPGPATH; ++ memcpy(idbuf + idlen, &fileset->creator_pid, ++ sizeof(fileset->creator_pid)); ++ idlen += sizeof(fileset->creator_pid); ++ memcpy(idbuf + idlen, &fileset->number, sizeof(fileset->number)); ++ idlen += sizeof(fileset->number); ++ memcpy(idbuf + idlen, file->name, namelen); ++ idlen += namelen; ++ ++ StaticAssertStmt(TEMPFILE_CRYPT_SALT_LEN == sizeof(h1) + sizeof(h2), ++ "temp-file salt must hold two 64-bit hashes"); ++ h1 = hash_bytes_extended((const unsigned char *) idbuf, idlen, 0); ++ h2 = hash_bytes_extended((const unsigned char *) idbuf, idlen, ++ UINT64CONST(0x9e3779b97f4a7c15)); ++ memcpy(file->crypt_salt, &h1, sizeof(h1)); ++ memcpy(file->crypt_salt + sizeof(h1), &h2, sizeof(h2)); ++ } ++} ++#endif ++ + /* + * Create a BufFile given the first underlying physical file. + * NOTE: caller must set isInterXact if appropriate. +@@ -145,6 +226,9 @@ + file->readOnly = false; + file->fileset = NULL; + file->name = NULL; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -274,6 +358,9 @@ + file->files = (File *) palloc(sizeof(File)); + file->files[0] = MakeNewFileSetSegment(file, 0); + file->readOnly = false; ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -344,6 +431,9 @@ + file->readOnly = (mode == O_RDONLY); + file->fileset = fileset; + file->name = pstrdup(name); ++#ifdef USE_TDE_HOOKS ++ BufFileInitCryptSalt(file); ++#endif + + return file; + } +@@ -479,6 +569,18 @@ /* we choose not to advance curOffset here */ @@ -21,14 +150,15 @@ + int64 blocknum = (int64) file->curFile * BUFFILE_SEG_SIZE + + file->curOffset / BLCKSZ; + -+ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum); ++ temp_file_decrypt_hook(file->buffer.data, file->nbytes, blocknum, ++ file->crypt_salt); + } +#endif + if (file->nbytes > 0) pgBufferUsage.temp_blks_read++; } -@@ -496,6 +510,26 @@ +@@ -496,6 +598,27 @@ int wpos = 0; int bytestowrite; File thisfile; @@ -48,14 +178,15 @@ + file->curOffset / BLCKSZ; + + memcpy(cryptbuf.data, file->buffer.data, file->nbytes); -+ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum); ++ temp_file_encrypt_hook(cryptbuf.data, file->nbytes, blocknum, ++ file->crypt_salt); + srcdata = cryptbuf.data; + } +#endif /* * Unlike BufFileLoadBuffer, we must dump the whole buffer even if it -@@ -535,7 +569,11 @@ +@@ -535,7 +658,11 @@ INSTR_TIME_SET_ZERO(io_start); bytestowrite = FileWrite(thisfile, @@ -148,7 +279,7 @@ +#endif /* USE_TDE_HOOKS */ --- a/src/include/storage/tempfile_crypt.h +++ b/src/include/storage/tempfile_crypt.h -@@ -0,0 +1,40 @@ +@@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * tempfile_crypt.h @@ -176,12 +307,20 @@ +/* GUC: master switch for temporary file encryption (default off). */ +extern PGDLLIMPORT bool encrypt_temp_files; + ++/* Length of the per-BufFile salt the hooks mix into the IV (bytes). */ ++#define TEMPFILE_CRYPT_SALT_LEN 16 ++ +/* + * Encrypt or decrypt one temporary-file buffer in place. blocknum is the -+ * logical block position of the buffer within the BufFile; an implementation -+ * uses it to derive a per-block IV. The transformation must preserve length. ++ * logical block position of the buffer within the BufFile; file_iv_salt is a ++ * per-BufFile value (TEMPFILE_CRYPT_SALT_LEN bytes) that is unique to each ++ * BufFile. An implementation must mix both into the IV: blocknum distinguishes ++ * blocks within one file, and file_iv_salt keeps two different BufFiles from ++ * ever repeating an (IV/tweak), which would otherwise leak equality across ++ * files. The transformation must preserve length. + */ -+typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum); ++typedef void (*TempFileCryptHook) (char *data, int nbytes, int64 blocknum, ++ const unsigned char *file_iv_salt); + +extern PGDLLIMPORT TempFileCryptHook temp_file_encrypt_hook; +extern PGDLLIMPORT TempFileCryptHook temp_file_decrypt_hook; diff --git a/src/open_pg_tde_tempfile.c b/src/open_pg_tde_tempfile.c index b1587de..949cd4c 100644 --- a/src/open_pg_tde_tempfile.c +++ b/src/open_pg_tde_tempfile.c @@ -17,14 +17,19 @@ * the cluster stops, even with access to the storage media. * * The buffer transformation is AES-128-XTS, keyed by the block's logical - * position through the XTS tweak. XTS is length-preserving for any buffer of at - * least one AES block (it uses ciphertext stealing for a non-block-multiple - * length) and is safe when a block position is rewritten in place, which - * temporary-file space reuse does. A buffer shorter than one AES block cannot - * use XTS; such a tail is masked with an AES-128-CTR keystream instead. Both - * XTS (NIST SP 800-38E) and CTR (SP 800-38A) are FIPS-approved modes, so with a - * FIPS build of OpenSSL every temporary-file operation uses approved - * cryptography. See documentation/docs/index/fips.md. + * position through the XTS tweak. The tweak combines a per-BufFile salt that + * core supplies (unique to each temporary file) with the block position, so + * two different BufFiles never reuse an (IV/tweak) even though every file + * shares the cluster-global key; without that salt the block position alone + * resets to zero in each new file and would repeat the keystream across + * files. XTS is length-preserving for any buffer of at least one AES block + * (it uses ciphertext stealing for a non-block-multiple length) and is safe + * when a block position is rewritten in place, which temporary-file space + * reuse does. A buffer shorter than one AES block cannot use XTS; such a tail + * is masked with an AES-128-CTR keystream instead. Both XTS (NIST SP + * 800-38E) and CTR (SP 800-38A) are FIPS-approved modes, so with a FIPS build + * of OpenSSL every temporary-file operation uses approved cryptography. See + * documentation/docs/index/fips.md. * *------------------------------------------------------------------------- */ @@ -45,13 +50,19 @@ #define TEMPFILE_IV_LEN 16 #define AES_BLOCK_LEN 16 +/* Core hands us a salt of this width; it must fill the whole IV/tweak. */ +StaticAssertDecl(TEMPFILE_CRYPT_SALT_LEN == TEMPFILE_IV_LEN, + "temp-file salt length must match the IV length"); + static bool tempfile_key_ready = false; static unsigned char tempfile_xts_key[TEMPFILE_XTS_KEY_LEN]; static unsigned char tempfile_ctr_key[TEMPFILE_CTR_KEY_LEN]; static unsigned char tempfile_base_iv[TEMPFILE_IV_LEN]; -static void tempfile_encrypt(char *data, int nbytes, int64 blocknum); -static void tempfile_decrypt(char *data, int nbytes, int64 blocknum); +static void tempfile_encrypt(char *data, int nbytes, int64 blocknum, + const unsigned char *file_iv_salt); +static void tempfile_decrypt(char *data, int nbytes, int64 blocknum, + const unsigned char *file_iv_salt); static void tempfile_ssl_error(const char *what) @@ -78,11 +89,20 @@ tempfile_init_key(void) tempfile_key_ready = true; } -/* Derive a per-block IV (XTS tweak) by mixing the block position into the base. */ +/* + * Derive a per-block IV (XTS tweak) from the base IV, the per-file salt, and + * the block position. The salt is mixed in first so that two different + * BufFiles never derive the same IV/tweak (which would repeat the keystream + * under our cluster-global key); the block position, XORed in next, only has + * to be unique within a single file. + */ static void -tempfile_block_iv(int64 blocknum, unsigned char *iv) +tempfile_block_iv(const unsigned char *file_iv_salt, int64 blocknum, + unsigned char *iv) { memcpy(iv, tempfile_base_iv, TEMPFILE_IV_LEN); + for (int i = 0; i < TEMPFILE_IV_LEN; i++) + iv[i] ^= file_iv_salt[i]; for (int i = 0; i < 8; i++) iv[TEMPFILE_IV_LEN - 1 - i] ^= (unsigned char) (blocknum >> (8 * i)); } @@ -148,7 +168,8 @@ tempfile_run_ctr(const unsigned char *iv, unsigned char *buf, int len) * at the length it was written. */ static void -tempfile_crypt_buffer(int enc, char *data, int nbytes, int64 blocknum) +tempfile_crypt_buffer(int enc, char *data, int nbytes, int64 blocknum, + const unsigned char *file_iv_salt) { unsigned char iv[TEMPFILE_IV_LEN]; @@ -157,7 +178,7 @@ tempfile_crypt_buffer(int enc, char *data, int nbytes, int64 blocknum) if (!tempfile_key_ready) tempfile_init_key(); - tempfile_block_iv(blocknum, iv); + tempfile_block_iv(file_iv_salt, blocknum, iv); if (nbytes >= AES_BLOCK_LEN) tempfile_run_xts(enc, iv, (unsigned char *) data, nbytes); @@ -166,15 +187,17 @@ tempfile_crypt_buffer(int enc, char *data, int nbytes, int64 blocknum) } static void -tempfile_encrypt(char *data, int nbytes, int64 blocknum) +tempfile_encrypt(char *data, int nbytes, int64 blocknum, + const unsigned char *file_iv_salt) { - tempfile_crypt_buffer(1, data, nbytes, blocknum); + tempfile_crypt_buffer(1, data, nbytes, blocknum, file_iv_salt); } static void -tempfile_decrypt(char *data, int nbytes, int64 blocknum) +tempfile_decrypt(char *data, int nbytes, int64 blocknum, + const unsigned char *file_iv_salt) { - tempfile_crypt_buffer(0, data, nbytes, blocknum); + tempfile_crypt_buffer(0, data, nbytes, blocknum, file_iv_salt); } /* diff --git a/t/temp_file_encryption.pl b/t/temp_file_encryption.pl index bc8aa25..0966a57 100644 --- a/t/temp_file_encryption.pl +++ b/t/temp_file_encryption.pl @@ -66,6 +66,65 @@ sub check_node $bp->quit; } +# The contents of each temp file under base/pgsql_tmp, one entry per file. +sub temp_file_contents +{ + my ($node) = @_; + my $dir = $node->data_dir . '/base/pgsql_tmp'; + my @out; + + opendir(my $dh, $dir) or return @out; + for my $f (sort readdir $dh) + { + next if $f eq '.' || $f eq '..'; + next unless -f "$dir/$f"; + push @out, slurp_file("$dir/$f"); + } + closedir $dh; + return @out; +} + +# True if any two arguments are byte-identical. +sub has_dup +{ + my %seen; + + for my $x (@_) + { + return 1 if $seen{$x}++; + } + return 0; +} + +# Run two identical spilling sorts behind held cursors so both of their +# temporary files are on disk at once, then return the per-file contents. The +# two sorts consume the same input in the same order, so their BufFiles hold +# byte-identical plaintext. +sub two_identical_temp_files +{ + my ($node) = @_; + my @bp; + + for my $i (0, 1) + { + my $h = $node->background_psql('postgres'); + $h->query_safe('BEGIN'); + $h->query_safe("DECLARE c CURSOR FOR SELECT '$canary' || g AS p " + . "FROM generate_series(1, 200000) g ORDER BY p"); + $h->query_safe('FETCH 1 FROM c'); + push @bp, $h; + } + + my @contents = temp_file_contents($node); + + for my $h (@bp) + { + $h->query_safe('COMMIT'); + $h->quit; + } + return @contents; +} + sub configure { my ($node, $mode) = @_; @@ -103,7 +162,6 @@ sub configure configure($off, 'off'); $off->start; check_node($off, 1, 'encryption off'); -$off->stop; # Encrypted node: canary must be absent. my $on = PostgreSQL::Test::Cluster->new('temp_enc'); @@ -111,6 +169,45 @@ sub configure configure($on, 'on'); $on->start; check_node($on, 0, 'encryption on'); + +# Temp-file IV reuse (H3): two BufFiles that hold identical plaintext must not +# encrypt to identical ciphertext. Each spilling sort is its own BufFile, and +# the block position that feeds the IV resets to zero in every BufFile, so with +# a cluster-global key and base IV the only thing keeping the two ciphertexts +# apart is the per-BufFile salt (open_pg_tde_tempfile.c mixes it into the XTS +# tweak / CTR IV). The control run with encryption off proves the two sorts +# really do produce byte-identical files, so the encrypted run's difference is +# the salt and not incidental. +my @plain = two_identical_temp_files($off); +ok(scalar(@plain) >= 2 && has_dup(@plain), + 'off: two identical sorts leave byte-identical temp files (control)'); + +my @cipher = two_identical_temp_files($on); +ok(scalar(@cipher) >= 2 && !has_dup(@cipher), + 'on: identical plaintext yields distinct ciphertext (per-BufFile salt)'); + +# Shared filesets (parallel workers): a BufFile written by one process and read +# by another derives its salt deterministically from the fileset identity, so +# the writer and reader must agree or the reader would decrypt to garbage. A +# parallel hash join that spills to shared temp files exercises that path; a +# correct result proves the salts agree. +$on->safe_psql('postgres', + 'CREATE TABLE t AS SELECT g FROM generate_series(1, 200000) g'); +my $pcount = $on->safe_psql( + 'postgres', q{ + SET max_parallel_workers_per_gather = 2; + SET enable_parallel_hash = on; + SET work_mem = '64kB'; + SET parallel_setup_cost = 0; + SET parallel_tuple_cost = 0; + SET min_parallel_table_scan_size = '0'; + SELECT count(*) FROM t a JOIN t b USING (g); +}); +is($pcount, '200000', + 'on: parallel hash join over encrypted shared temp files returns correct rows' +); + +$off->stop; $on->stop; done_testing();