From 2fbdc3d4d47e6b7afbd6c6a92c2d37ce82ec31b4 Mon Sep 17 00:00:00 2001 From: Nick Anderson Date: Mon, 20 Jul 2026 04:53:51 -0500 Subject: [PATCH 1/2] ENT-6511: Increase cf-monitord measurement slots from 100 to 300 ~80 of the 100 slots are consumed by default measurements, leaving little room for custom ones. Raise the limit to 300. Kept in sync: CF_OBSERVABLES and the two cf-check copies of the on-disk struct, plus the OBSERVABLES name table (GetObservable() indexes it for unregistered slots, so a short table crashes cf-monitord). cf-check dump now tolerates the shorter records and ts_key a pre-upgrade agent leaves behind. Ticket: ENT-6511 Changelog: Increased the maximum number of cf-monitord measurement slots (CF_OBSERVABLES) from 100 to 300 --- cf-check/db_structs.h | 2 +- cf-check/dump.c | 13 +- cf-check/observables.c | 21 +++- cf-check/observables.h | 2 +- libpromises/cf3.defs.h | 2 +- libpromises/constants.c | 204 ++++++++++++++++++++++++++++++++ tests/unit/mon_cpu_test.c | 2 +- tests/unit/mon_load_test.c | 2 +- tests/unit/mon_processes_test.c | 2 +- 9 files changed, 236 insertions(+), 14 deletions(-) diff --git a/cf-check/db_structs.h b/cf-check/db_structs.h index 17ba14e73b..8a92eed4c7 100644 --- a/cf-check/db_structs.h +++ b/cf-check/db_structs.h @@ -125,7 +125,7 @@ static const char *const observable_strings[] = }; // Not the actual count, just the room we set aside in struct (and LMDB): -#define CF_OBSERVABLES 100 +#define CF_OBSERVABLES 300 typedef struct Averages { diff --git a/cf-check/dump.c b/cf-check/dump.c index 6ded3c8bbd..e8af8eb8a3 100644 --- a/cf-check/dump.c +++ b/cf-check/dump.c @@ -162,10 +162,14 @@ static void print_struct_lock_data( static void print_struct_averages( const MDB_val value, const bool strip_strings, const char *tskey_filename) { - assert(sizeof(Averages) == value.mv_size); - if (sizeof(Averages) != value.mv_size) + // A record written before an increase of CF_OBSERVABLES is shorter than the + // current struct (e.g. after a package upgrade, until cf-monitord rewrites + // it). Accept any size up to the full struct and copy into a zeroed struct + // so the un-stored trailing slots read back as zero. + assert(value.mv_size <= sizeof(Averages)); + if (value.mv_size > sizeof(Averages)) { - // Fall back to simple printing in release builds: + // Unexpected: larger than the struct. Fall back to simple printing. print_json_string(value.mv_data, value.mv_size, strip_strings); } else @@ -173,7 +177,8 @@ static void print_struct_averages( // TODO: clean up Averages char **obnames = NULL; Averages averages; - memcpy(&averages, value.mv_data, sizeof(averages)); + memset(&averages, 0, sizeof(averages)); + memcpy(&averages, value.mv_data, value.mv_size); const time_t last_seen = averages.last_seen; obnames = GetObservableNames(tskey_filename); diff --git a/cf-check/observables.c b/cf-check/observables.c index 857c6a192d..2619ad1dce 100644 --- a/cf-check/observables.c +++ b/cf-check/observables.c @@ -67,10 +67,23 @@ char **GetObservableNames(const char *ts_key_path) if (fgets(line, CF_MAXVARSIZE, f) == NULL) { - Log(LOG_LEVEL_ERR, - "Error trying to read ts_key from file '%s'. (fgets: %s)", - filename, - GetErrorStr()); + if (ferror(f)) + { + Log(LOG_LEVEL_ERR, + "Error trying to read ts_key from file '%s'. (fgets: %s)", + filename, + GetErrorStr()); + } + /* The ts_key has fewer entries than CF_OBSERVABLES -- e.g. it + * was written by an older agent with a smaller CF_OBSERVABLES. + * Fill the remaining names so the returned array keeps its + * documented non-NULL guarantee (callers index all + * CF_OBSERVABLES entries). */ + for (int j = i; j < CF_OBSERVABLES; ++j) + { + snprintf(buf, CF_MAXVARSIZE, "spare[%d]", j); + temp[j] = xstrdup(buf); + } break; } diff --git a/cf-check/observables.h b/cf-check/observables.h index 1c8d932b96..dd33be3d56 100644 --- a/cf-check/observables.h +++ b/cf-check/observables.h @@ -4,7 +4,7 @@ #include // copy of libpromises/cf3.defs.h, TODO refactor -#define CF_OBSERVABLES 100 +#define CF_OBSERVABLES 300 char **GetObservableNames(const char *ts_key_path); diff --git a/libpromises/cf3.defs.h b/libpromises/cf3.defs.h index fd53bf8a68..c2e060c055 100644 --- a/libpromises/cf3.defs.h +++ b/libpromises/cf3.defs.h @@ -148,7 +148,7 @@ typedef enum #define CF_MEASURE_INTERVAL (5.0*60.0) #define CF_SHIFT_INTERVAL (6*3600) -#define CF_OBSERVABLES 100 +#define CF_OBSERVABLES 300 /* Special exit codes */ #define EC_EVAL_ABORTED 6 /* like SIGABRT, but signal exit codes are 120+SIG */ diff --git a/libpromises/constants.c b/libpromises/constants.c index 7562b5aa64..8ade1ced25 100644 --- a/libpromises/constants.c +++ b/libpromises/constants.c @@ -179,4 +179,208 @@ const char *const OBSERVABLES[CF_OBSERVABLES][2] = {"spare", "unused"}, {"spare", "unused"}, {"spare", "unused"}, + /* ENT-6511: slots 100-299, added when CF_OBSERVABLES was raised + 100->300. Every slot must have a name entry: cf-monitord's + GetObservable() falls back to OBSERVABLES[i] for unregistered + slots, so a table shorter than CF_OBSERVABLES => NULL deref. */ + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, + {"spare", "unused"}, }; diff --git a/tests/unit/mon_cpu_test.c b/tests/unit/mon_cpu_test.c index d8eafadd74..3399fb08c8 100644 --- a/tests/unit/mon_cpu_test.c +++ b/tests/unit/mon_cpu_test.c @@ -52,7 +52,7 @@ static double GetCpuStat() void test_cpu_monitor(void) { - double cf_this[100]; + double cf_this[300]; double dq1 = GetCpuStat(); if (dq1 == -1.0) { diff --git a/tests/unit/mon_load_test.c b/tests/unit/mon_load_test.c index c25789f9ac..de143e0690 100644 --- a/tests/unit/mon_load_test.c +++ b/tests/unit/mon_load_test.c @@ -6,7 +6,7 @@ void test_load_monitor(void) { - double cf_this[100]; + double cf_this[300]; MonLoadGatherData(cf_this); double load1[2] = {0,0}; double load2[2] = {0,0}; diff --git a/tests/unit/mon_processes_test.c b/tests/unit/mon_processes_test.c index 50b59fc620..fff48dd224 100644 --- a/tests/unit/mon_processes_test.c +++ b/tests/unit/mon_processes_test.c @@ -100,7 +100,7 @@ static bool GetSysUsers( int *userListSz, int *numRootProcs, int *numOtherProcs) void test_processes_monitor(void) { - double cf_this[100] = { 0.0 }; + double cf_this[300] = { 0.0 }; MonProcessesGatherData(cf_this); MonProcessesGatherData(cf_this); MonProcessesGatherData(cf_this); From 5f4e18791bf495c55e4e4e2b8bd3f7f6e4f66c16 Mon Sep 17 00:00:00 2001 From: Nick Anderson Date: Tue, 21 Jul 2026 08:19:58 -0500 Subject: [PATCH 2/2] Added a DB migration for the enlarged Averages struct Raising CF_OBSERVABLES enlarges the fixed-size Averages record, so records written by an older agent are shorter than the current struct. Added a migration (modelled on dbm_migration_lastseen.c) that rewrites old records to the current size, zero-filling the new slots, registered for the observations and history DBs. This replaces the cf-check dump leniency with a one-time on-disk migration; dump only needs to know the new "version" key. Ticket: ENT-6511 Changelog: None --- cf-check/dump.c | 25 ++- libpromises/Makefile.am | 1 + libpromises/dbm_migration.c | 7 +- libpromises/dbm_migration_observations.c | 113 +++++++++++ tests/unit/Makefile.am | 5 + tests/unit/observations_migration_test.c | 238 +++++++++++++++++++++++ 6 files changed, 378 insertions(+), 11 deletions(-) create mode 100644 libpromises/dbm_migration_observations.c create mode 100644 tests/unit/observations_migration_test.c diff --git a/cf-check/dump.c b/cf-check/dump.c index e8af8eb8a3..465173e891 100644 --- a/cf-check/dump.c +++ b/cf-check/dump.c @@ -162,14 +162,12 @@ static void print_struct_lock_data( static void print_struct_averages( const MDB_val value, const bool strip_strings, const char *tskey_filename) { - // A record written before an increase of CF_OBSERVABLES is shorter than the - // current struct (e.g. after a package upgrade, until cf-monitord rewrites - // it). Accept any size up to the full struct and copy into a zeroed struct - // so the un-stored trailing slots read back as zero. - assert(value.mv_size <= sizeof(Averages)); - if (value.mv_size > sizeof(Averages)) + assert(sizeof(Averages) == value.mv_size); + if (sizeof(Averages) != value.mv_size) { - // Unexpected: larger than the struct. Fall back to simple printing. + // Fall back to simple printing in release builds. Records written + // before CF_OBSERVABLES was raised are shorter, but a running agent + // migrates them (see dbm_migration_observations.c) before they are read. print_json_string(value.mv_data, value.mv_size, strip_strings); } else @@ -177,8 +175,7 @@ static void print_struct_averages( // TODO: clean up Averages char **obnames = NULL; Averages averages; - memset(&averages, 0, sizeof(averages)); - memcpy(&averages, value.mv_data, value.mv_size); + memcpy(&averages, value.mv_data, sizeof(averages)); const time_t last_seen = averages.last_seen; obnames = GetObservableNames(tskey_filename); @@ -293,7 +290,15 @@ static void print_struct_or_string( { if (structs) { - if (StringContains(file, "cf_lastseen.lmdb") + if ((StringContains(file, "cf_observations.lmdb") + || StringContains(file, "history.lmdb")) + && StringEqual(key.mv_data, "version")) + { + // Migration version scalar (dbm_migration_observations.c), a short + // string rather than an Averages struct. + print_json_string(value.mv_data, value.mv_size, strip_strings); + } + else if (StringContains(file, "cf_lastseen.lmdb") && StringStartsWith(key.mv_data, "q")) { print_struct_lastseen_quality(value, strip_strings); diff --git a/libpromises/Makefile.am b/libpromises/Makefile.am index 8f60e0f501..4cef498d15 100644 --- a/libpromises/Makefile.am +++ b/libpromises/Makefile.am @@ -84,6 +84,7 @@ libpromises_la_SOURCES = \ dbm_api.c dbm_api.h dbm_api_types.h dbm_priv.h \ dbm_migration.c dbm_migration.h \ dbm_migration_lastseen.c \ + dbm_migration_observations.c \ dbm_lmdb.c \ dbm_quick.c \ dbm_tokyocab.c \ diff --git a/libpromises/dbm_migration.c b/libpromises/dbm_migration.c index 6066483533..a833d1cf89 100644 --- a/libpromises/dbm_migration.c +++ b/libpromises/dbm_migration.c @@ -29,6 +29,7 @@ #include extern const DBMigrationFunction dbm_migration_plan_lastseen[]; +extern const DBMigrationFunction dbm_migration_plan_observations[]; #ifndef LMDB @@ -51,7 +52,11 @@ static size_t DBVersion(DBHandle *db) } static const DBMigrationFunction *const dbm_migration_plans[dbid_max] = { - [dbid_lastseen] = dbm_migration_plan_lastseen + [dbid_lastseen] = dbm_migration_plan_lastseen, + /* Both DBs store the same fixed-size Averages records, so they share the + * plan that expands them when CF_OBSERVABLES grows. */ + [dbid_observations] = dbm_migration_plan_observations, + [dbid_history] = dbm_migration_plan_observations }; bool DBMigrate(DBHandle *db, dbid id) diff --git a/libpromises/dbm_migration_observations.c b/libpromises/dbm_migration_observations.c new file mode 100644 index 0000000000..c505609006 --- /dev/null +++ b/libpromises/dbm_migration_observations.c @@ -0,0 +1,113 @@ +/* + Copyright 2024 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#include + +#include +#include + +/* Number of measurement slots (CF_OBSERVABLES) before ENT-6511 raised it from + * 100 to 300. A record written by an agent from before that change is this many + * slots long. */ +#define CF_OBSERVABLES_BEFORE_ENT_6511 100 + +typedef struct +{ + time_t last_seen; + QPoint Q[CF_OBSERVABLES_BEFORE_ENT_6511]; +} AveragesBeforeEnt6511; + +/* + * The observations (cf_observations.lmdb) and history (history.lmdb) databases + * store fixed-size Averages records keyed by time. Raising CF_OBSERVABLES grows + * that struct, so a record written by an older agent is shorter than the current + * struct. cf-monitord's own read path zero-extends short records, and it + * overwrites the observations records on its next cycle, but the history records + * are never rewritten. Migrate every old-size record to the current size, + * zero-filling the added slots, so all records on disk share one layout. + */ +static bool MeasurementsMigrationVersion0(DBHandle *db) +{ + DBCursor *cursor; + if (!NewDBCursor(db, &cursor)) + { + Log(LOG_LEVEL_ERR, + "Unable to create database cursor during measurement DB migration"); + return false; + } + + char *key; + void *value; + int key_size, value_size; + + while (NextDB(cursor, &key, &key_size, &value, &value_size)) + { + /* Only fixed-size Averages records written before CF_OBSERVABLES was + * raised need expanding. Scalar bookkeeping keys (e.g. "DATABASE_AGE" + * and "version") are a different size and are left untouched. */ + if (value_size != (int) sizeof(AveragesBeforeEnt6511)) + { + continue; + } + + /* Copy the old, shorter record into a full-size, zeroed struct so the + * slots added by the larger CF_OBSERVABLES read back as zero. */ + Averages expanded; + memset(&expanded, 0, sizeof(expanded)); + memcpy(&expanded, value, sizeof(AveragesBeforeEnt6511)); + + // This will overwrite the entry + if (!DBCursorWriteEntry(cursor, &expanded, sizeof(expanded))) + { + Log(LOG_LEVEL_ERR, + "Unable to expand measurement record for key '%s' during migration", + key); + DeleteDBCursor(cursor); + return false; + } + } + + if (!DeleteDBCursor(cursor)) + { + Log(LOG_LEVEL_ERR, + "Unable to close cursor during measurement DB migration"); + return false; + } + + if (!WriteDB(db, "version", "1", sizeof("1"))) + { + Log(LOG_LEVEL_ERR, + "Failed to update version number during measurement DB migration"); + return false; + } + + Log(LOG_LEVEL_INFO, "Migrated measurement database to version 1"); + return true; +} + +const DBMigrationFunction dbm_migration_plan_observations[] = +{ + MeasurementsMigrationVersion0, + NULL +}; diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index b59d378b28..3864b67876 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -68,6 +68,7 @@ libdb_la_SOURCES = db_stubs.c \ ../../libpromises/dbm_lmdb.c \ ../../libpromises/dbm_migration.c \ ../../libpromises/dbm_migration_lastseen.c \ + ../../libpromises/dbm_migration_observations.c \ ../../libpromises/global_mutex.c \ ../../cf-check/backup.c \ ../../cf-check/diagnose.c \ @@ -103,6 +104,7 @@ check_PROGRAMS = \ lastseen_test \ lastseen_migration_test \ changes_migration_test \ + observations_migration_test \ db_test \ db_concurrent_test \ item_lib_test \ @@ -261,6 +263,9 @@ db_concurrent_test_SOURCES = db_concurrent_test.c #db_concurrent_test_CPPFLAGS = $(libdb_la_CPPFLAGS) db_concurrent_test_LDADD = libtest.la libdb.la +observations_migration_test_SOURCES = observations_migration_test.c +observations_migration_test_LDADD = libtest.la libdb.la + lastseen_test_SOURCES = lastseen_test.c \ ../../libntech/libutils/statistics.c #lastseen_test_CPPFLAGS = $(libdb_la_CPPFLAGS) diff --git a/tests/unit/observations_migration_test.c b/tests/unit/observations_migration_test.c new file mode 100644 index 0000000000..72adc9e1a5 --- /dev/null +++ b/tests/unit/observations_migration_test.c @@ -0,0 +1,238 @@ +#include +#include +#include +#include + +#include +#include +#include /* StringEqual */ +#include /* xsnprintf */ + +#ifndef LMDB + +/* The measurement DB migration only exists for the LMDB backend (see + * dbm_migration.c); with other backends DBMigrate() is a no-op. */ +int main(void) +{ + return 0; +} + +#else + +/* The Averages layout from before ENT-6511 raised CF_OBSERVABLES from 100 to + * 300. A record written by such an agent is this size on disk. */ +#define CF_OBSERVABLES_BEFORE_ENT_6511 100 + +typedef struct +{ + time_t last_seen; + QPoint Q[CF_OBSERVABLES_BEFORE_ENT_6511]; +} AveragesBeforeEnt6511; + +char CFWORKDIR[CF_BUFSIZE]; + +void tests_setup(void) +{ + static char env[] = /* Needs to be static for putenv() */ + "CFENGINE_TEST_OVERRIDE_WORKDIR=/tmp/observations_migration_test.XXXXXX"; + + char *workdir = strchr(env, '=') + 1; /* start of the path */ + assert(workdir - 1 && workdir[0] == '/'); + + mkdtemp(workdir); + strlcpy(CFWORKDIR, workdir, CF_BUFSIZE); + putenv(env); + mkdir(GetStateDir(), (S_IRWXU | S_IRWXG | S_IRWXO)); +} + +static void tests_teardown(void) +{ + char cmd[CF_BUFSIZE]; + xsnprintf(cmd, CF_BUFSIZE, "rm -rf '%s'", CFWORKDIR); + system(cmd); +} + +/* + * Provides an observations DB in a pre-migration (version 0) state: OpenDB() + * runs the migration and writes a "version" marker, so wipe every entry to get + * back to how an old, never-migrated database looks on disk. + */ +static DBHandle *setup_unversioned(void) +{ + char cmd[CF_BUFSIZE]; + xsnprintf(cmd, CF_BUFSIZE, "rm -rf '%s'/*", GetStateDir()); + system(cmd); + + DBHandle *db; + assert_int_equal(OpenDB(&db, dbid_observations), true); + + DBCursor *cursor; + assert_int_equal(NewDBCursor(db, &cursor), true); + + char *key; + void *value; + int ksize, vsize; + while (NextDB(cursor, &key, &ksize, &value, &vsize)) + { + DBCursorDeleteEntry(cursor); + } + assert_int_equal(DeleteDBCursor(cursor), true); + + return db; +} + +/* Returns the on-disk size of a record, or -1 if the key is absent. */ +static int record_size(DBHandle *db, const char *want_key) +{ + DBCursor *cursor; + assert_int_equal(NewDBCursor(db, &cursor), true); + + char *key; + void *value; + int ksize, vsize; + int found = -1; + while (NextDB(cursor, &key, &ksize, &value, &vsize)) + { + if (StringEqual(key, want_key)) + { + found = vsize; + } + } + assert_int_equal(DeleteDBCursor(cursor), true); + return found; +} + +static void test_migrate_expands_old_record(void) +{ + DBHandle *db = setup_unversioned(); + + /* A measurement record in the old, shorter (100-slot) layout ... */ + AveragesBeforeEnt6511 old; + memset(&old, 0, sizeof(old)); + old.last_seen = 1234567; + old.Q[0].q = 1.0; + old.Q[0].expect = 2.0; + old.Q[0].var = 3.0; + old.Q[0].dq = 4.0; + old.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].q = 5.0; + old.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].expect = 6.0; + old.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].var = 7.0; + old.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].dq = 8.0; + assert_int_equal(WriteDB(db, "Mon_Hr12_Q1", &old, sizeof(old)), true); + + /* ... and a scalar bookkeeping record that must NOT be touched. */ + double age = 42.0; + assert_int_equal(WriteDB(db, "DATABASE_AGE", &age, sizeof(age)), true); + + CloseDB(db); + + /* Reopening runs the migration (no version marker present yet). */ + assert_int_equal(OpenDB(&db, dbid_observations), true); + + /* The measurement record is now the full, current size ... */ + assert_int_equal(record_size(db, "Mon_Hr12_Q1"), (int) sizeof(Averages)); + /* ... the scalar record is left at its original size ... */ + assert_int_equal(record_size(db, "DATABASE_AGE"), (int) sizeof(double)); + /* ... and the version marker was written. */ + assert_int_equal(HasKeyDB(db, "version", strlen("version") + 1), true); + + /* The original slots are preserved and the new slots read back as zero. */ + Averages migrated; + memset(&migrated, 0, sizeof(migrated)); + assert_int_equal(ReadDB(db, "Mon_Hr12_Q1", &migrated, sizeof(migrated)), true); + assert_int_equal(migrated.last_seen, 1234567); + assert_double_close(migrated.Q[0].q, 1.0); + assert_double_close(migrated.Q[0].dq, 4.0); + assert_double_close(migrated.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].q, 5.0); + assert_double_close(migrated.Q[CF_OBSERVABLES_BEFORE_ENT_6511 - 1].dq, 8.0); + assert_double_close(migrated.Q[CF_OBSERVABLES_BEFORE_ENT_6511].q, 0.0); + assert_double_close(migrated.Q[CF_OBSERVABLES - 1].var, 0.0); + + double read_age = 0.0; + assert_int_equal(ReadDB(db, "DATABASE_AGE", &read_age, sizeof(read_age)), true); + assert_double_close(read_age, 42.0); + + CloseDB(db); +} + +static void test_current_record_untouched(void) +{ + /* A record already at the current size must not be altered by the + * migration (the size-exact check must ignore it). */ + DBHandle *db = setup_unversioned(); + + Averages current; + memset(¤t, 0, sizeof(current)); + current.last_seen = 7654321; + current.Q[CF_OBSERVABLES - 1].q = 99.0; + assert_int_equal(WriteDB(db, "Tue_Hr06_Q2", ¤t, sizeof(current)), true); + CloseDB(db); + + assert_int_equal(OpenDB(&db, dbid_observations), true); + + assert_int_equal(record_size(db, "Tue_Hr06_Q2"), (int) sizeof(Averages)); + + Averages read; + memset(&read, 0, sizeof(read)); + assert_int_equal(ReadDB(db, "Tue_Hr06_Q2", &read, sizeof(read)), true); + assert_int_equal(read.last_seen, 7654321); + assert_double_close(read.Q[CF_OBSERVABLES - 1].q, 99.0); + + CloseDB(db); +} + +static void test_migration_is_idempotent(void) +{ + /* Once migrated, reopening must not run the migration again nor disturb + * the data. */ + DBHandle *db = setup_unversioned(); + + AveragesBeforeEnt6511 old; + memset(&old, 0, sizeof(old)); + old.last_seen = 111; + assert_int_equal(WriteDB(db, "Wed_Hr18_Q3", &old, sizeof(old)), true); + CloseDB(db); + + /* First open migrates. */ + assert_int_equal(OpenDB(&db, dbid_observations), true); + assert_int_equal(record_size(db, "Wed_Hr18_Q3"), (int) sizeof(Averages)); + CloseDB(db); + + /* Second open is a no-op. */ + assert_int_equal(OpenDB(&db, dbid_observations), true); + assert_int_equal(record_size(db, "Wed_Hr18_Q3"), (int) sizeof(Averages)); + + char version[8]; + assert_int_equal(ReadDB(db, "version", version, sizeof(version)), true); + assert_string_equal(version, "1"); + + CloseDB(db); +} + +int main(void) +{ + tests_setup(); + + const UnitTest tests[] = + { + unit_test(test_migrate_expands_old_record), + unit_test(test_current_record_untouched), + unit_test(test_migration_is_idempotent), + }; + + PRINT_TEST_BANNER(); + int ret = run_tests(tests); + + tests_teardown(); + return ret; +} + +/* STUBS */ + +void FatalError(ARG_UNUSED char *s, ...) +{ + fail(); + exit(42); +} + +#endif // LMDB