diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index 578cb1e543..eef39e28fc 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -353,6 +353,7 @@ option: allow-list: - none - text + - json version: default: text allow-list: diff --git a/src/build/help/help.xml b/src/build/help/help.xml index 6641fe1520..16dfda37c2 100644 --- a/src/build/help/help.xml +++ b/src/build/help/help.xml @@ -2734,6 +2734,7 @@ none - No verify output. text - Output verify information to stdout. + json - Output verify information to stdout in json format. diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index 6f5ebd8e0e..9b3fef78a2 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -21,6 +21,7 @@ Verify contents of the repository. #include "common/io/io.h" #include "common/log.h" #include "common/regExp.h" +#include "common/type/json.h" #include "config/config.h" #include "info/infoArchive.h" #include "info/infoBackup.h" @@ -37,6 +38,28 @@ Constants #define VERIFY_STATUS_OK "ok" #define VERIFY_STATUS_ERROR "error" +// Naming convention: _KEY__VAR. If the key exists in multiple sections, then _ is omitted. +VARIANT_STRDEF_STATIC(KEY_ARCHIVES_VAR, "archives"); +VARIANT_STRDEF_STATIC(ARCHIVE_KEY_ARCHIVEID_VAR, "archiveId"); +VARIANT_STRDEF_STATIC(ARCHIVE_KEY_CHECKED_VAR, "checked"); +VARIANT_STRDEF_STATIC(KEY_VALID_VAR, "valid"); +VARIANT_STRDEF_STATIC(KEY_MISSING_VAR, "missing"); +VARIANT_STRDEF_STATIC(KEY_FILEERRORS_VAR, "fileErrors"); +VARIANT_STRDEF_STATIC(KEY_CHECKSUMINVALID_VAR, "checksumInvalid"); +VARIANT_STRDEF_STATIC(KEY_SIZEINVALID_VAR, "sizeInvalid"); +VARIANT_STRDEF_STATIC(KEY_OTHER_VAR, "other"); +VARIANT_STRDEF_STATIC(KEY_BACKUPS_VAR, "backups"); +VARIANT_STRDEF_STATIC(BACKUPS_KEY_LABEL_VAR, "label"); +VARIANT_STRDEF_STATIC(KEY_STATUS_VAR, "status"); +VARIANT_STRDEF_STATIC(BACKUPS_KEY_CHECKED_VAR, "checked"); +VARIANT_STRDEF_STATIC(KEY_MESSAGES_VAR, "messages"); +VARIANT_STRDEF_STATIC(VERIFY_KEY_STANZA_VAR, "stanza"); +VARIANT_STRDEF_STATIC(VERIFY_KEY_STATUS_ERROR, "error"); +VARIANT_STRDEF_STATIC(VERIFY_KEY_STATUS_OK, "ok"); +VARIANT_STRDEF_STATIC(VERIFY_MSG_KEY_LEVEL, "level"); +VARIANT_STRDEF_STATIC(VERIFY_MSG_KEY_MESSAGE, "message"); +VARIANT_STRDEF_STATIC(VERIFY_MSG_KEY_PID, "pid"); + /*********************************************************************************************************************************** Data Types and Structures ***********************************************************************************************************************************/ @@ -132,8 +155,104 @@ typedef struct VerifyJobData bool enableArchiveFilter; // Only check archives in the specified range const String *archiveStart; // Start of the WAL range to be verified const String *archiveStop; // End of the WAL range to be verified + VariantList *messageList; // List of errors that occurred during the job } VerifyJobData; +/*********************************************************************************************************************************** +Helper function to add a string with log level to an error list +***********************************************************************************************************************************/ +static void +verifyErrorNew(VariantList *messageList, const LogLevel logLevel, const String *const message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(VARIANT_LIST, messageList); // Error list to add to + FUNCTION_LOG_PARAM(ENUM, logLevel); // Log level + FUNCTION_TEST_PARAM(STRING, message); // Error message + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(message != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext((List *)messageList)) + { + KeyValue *errorMsg = kvNew(); + + kvPut(errorMsg, VERIFY_MSG_KEY_LEVEL, VARUINT(logLevel)); + kvPut(errorMsg, VERIFY_MSG_KEY_MESSAGE, VARSTR(strDup(message))); + + varLstAdd(messageList, varNewKv(errorMsg)); + + // Duplicate the message to the log as soon as in appears. + // Perhaps user may want to see it immediately. + LOG(logLevel, 0, strZ(message)); + } + + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/********************************************************************************************************************************** +Helper function to add a string with log level and process id to an error list +***********************************************************************************************************************************/ +static void +verifyErrorPidNew(VariantList *messageList, const LogLevel logLevel, const unsigned int pid, const String *const message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(VARIANT_LIST, messageList); // Error list to add to + FUNCTION_LOG_PARAM(ENUM, logLevel); // Log level + FUNCTION_TEST_PARAM(UINT, pid); // Process id + FUNCTION_TEST_PARAM(STRING, message); // Error message + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(message != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext((List *)messageList)) + { + KeyValue *const errorMsg = kvNew(); + + kvPut(errorMsg, VERIFY_MSG_KEY_LEVEL, VARUINT(logLevel)); + kvPut(errorMsg, VERIFY_MSG_KEY_MESSAGE, VARSTR(strDup(message))); + kvPut(errorMsg, VERIFY_MSG_KEY_PID, VARUINT(pid)); + + varLstAdd(messageList, varNewKv(errorMsg)); + } + LOG_PID(logLevel, pid, 0, strZ(message)); + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Helper function to add a message going to stdout to the message list. +***********************************************************************************************************************************/ +static void +verifyMessageNew(VariantList *messageList, const String *const message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(VARIANT_LIST, messageList); // Error list to add to + FUNCTION_TEST_PARAM(STRING, message); // Error message + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(message != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext((List *)messageList)) + { + KeyValue *const errorMsg = kvNew(); + kvPut(errorMsg, VERIFY_MSG_KEY_MESSAGE, VARSTR(strDup(message))); + varLstAdd(messageList, varNewKv(errorMsg)); + } + + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + /*********************************************************************************************************************************** Helper function to add a file to an invalid file list ***********************************************************************************************************************************/ @@ -198,12 +317,13 @@ verifyFileLoad(const String *const pathFileName, const String *const cipherPass) Get status of info files in the repository ***********************************************************************************************************************************/ static VerifyInfoFile -verifyInfoFile(const String *const pathFileName, const bool keepFile, const String *const cipherPass) +verifyInfoFile(const String *const pathFileName, const bool keepFile, const String *const cipherPass, VariantList *const messageList) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, pathFileName); // Fully qualified path/file name FUNCTION_LOG_PARAM(BOOL, keepFile); // Should the file be kept in memory? FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to open file if encrypted + FUNCTION_TEST_PARAM(LIST, messageList); // List of errors FUNCTION_LOG_END(); FUNCTION_AUDIT_STRUCT(); @@ -248,7 +368,7 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Stri if (result.errorCode == errorTypeCode(&ChecksumError)) strCat(errorMsg, strNewFmt(" %s", strZ(pathFileName))); - LOG_DETAIL(strZ(errorMsg)); + verifyErrorNew(messageList, logLevelDetail, errorMsg); } TRY_END(); } @@ -261,9 +381,11 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Stri Get the archive.info file ***********************************************************************************************************************************/ static InfoArchive * -verifyArchiveInfoFile(void) +verifyArchiveInfoFile(VariantList *const messageList) { - FUNCTION_LOG_VOID(logLevelDebug); + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_TEST_PARAM(LIST, messageList); // List of errors + FUNCTION_LOG_END(); InfoArchive *result = NULL; @@ -271,7 +393,8 @@ verifyArchiveInfoFile(void) { // Get the main info file const VerifyInfoFile verifyArchiveInfo = verifyInfoFile( - INFO_ARCHIVE_PATH_FILE_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_ARCHIVE_PATH_FILE_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass), + messageList); // If the main file did not error, then report on the copy's status and check checksums if (verifyArchiveInfo.errorCode == 0) @@ -281,7 +404,7 @@ verifyArchiveInfoFile(void) // Attempt to load the copy and report on it's status but don't keep it in memory const VerifyInfoFile verifyArchiveInfoCopy = verifyInfoFile( - INFO_ARCHIVE_PATH_FILE_COPY_STR, false, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_ARCHIVE_PATH_FILE_COPY_STR, false, cfgOptionStrNull(cfgOptRepoCipherPass), messageList); // If the copy loaded successfully, then check the checksums if (verifyArchiveInfoCopy.errorCode == 0) @@ -289,14 +412,14 @@ verifyArchiveInfoFile(void) // If the info and info.copy checksums don't match each other than one (or both) of the files could be corrupt so // log a warning but must trust main if (!strEq(verifyArchiveInfo.checksum, verifyArchiveInfoCopy.checksum)) - LOG_DETAIL("archive.info.copy does not match archive.info"); + verifyErrorNew(messageList, logLevelDetail, strNewZ("archive.info.copy does not match archive.info")); } } else { // Attempt to load the copy const VerifyInfoFile verifyArchiveInfoCopy = verifyInfoFile( - INFO_ARCHIVE_PATH_FILE_COPY_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_ARCHIVE_PATH_FILE_COPY_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass), messageList); // If loaded successfully, then return the copy as usable if (verifyArchiveInfoCopy.errorCode == 0) @@ -315,9 +438,11 @@ verifyArchiveInfoFile(void) Get the backup.info file ***********************************************************************************************************************************/ static InfoBackup * -verifyBackupInfoFile(void) +verifyBackupInfoFile(VariantList *const messageList) { - FUNCTION_LOG_VOID(logLevelDebug); + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_TEST_PARAM(LIST, messageList); // List of errors + FUNCTION_LOG_END(); InfoBackup *result = NULL; @@ -325,7 +450,7 @@ verifyBackupInfoFile(void) { // Get the main info file const VerifyInfoFile verifyBackupInfo = verifyInfoFile( - INFO_BACKUP_PATH_FILE_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_BACKUP_PATH_FILE_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass), messageList); // If the main file did not error, then report on the copy's status and check checksums if (verifyBackupInfo.errorCode == 0) @@ -335,7 +460,7 @@ verifyBackupInfoFile(void) // Attempt to load the copy and report on it's status but don't keep it in memory const VerifyInfoFile verifyBackupInfoCopy = verifyInfoFile( - INFO_BACKUP_PATH_FILE_COPY_STR, false, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_BACKUP_PATH_FILE_COPY_STR, false, cfgOptionStrNull(cfgOptRepoCipherPass), messageList); // If the copy loaded successfully, then check the checksums if (verifyBackupInfoCopy.errorCode == 0) @@ -343,14 +468,14 @@ verifyBackupInfoFile(void) // If the info and info.copy checksums don't match each other than one (or both) of the files could be corrupt so // log a warning but must trust main if (!strEq(verifyBackupInfo.checksum, verifyBackupInfoCopy.checksum)) - LOG_DETAIL("backup.info.copy does not match backup.info"); + verifyErrorNew(messageList, logLevelDetail, strNewZ("backup.info.copy does not match backup.info")); } } else { // Attempt to load the copy const VerifyInfoFile verifyBackupInfoCopy = verifyInfoFile( - INFO_BACKUP_PATH_FILE_COPY_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass)); + INFO_BACKUP_PATH_FILE_COPY_STR, true, cfgOptionStrNull(cfgOptRepoCipherPass), messageList); // If loaded successfully, then return the copy as usable if (verifyBackupInfoCopy.errorCode == 0) @@ -371,7 +496,7 @@ Get the manifest file static Manifest * verifyManifestFile( VerifyBackupResult *const backupResult, const String *const cipherPass, bool currentBackup, const InfoPg *const pgHistory, - unsigned int *const jobErrorTotal) + unsigned int *const jobErrorTotal, VariantList *const messageList) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM_P(VERIFY_BACKUP_RESULT, backupResult); // The result set for the backup being processed @@ -379,6 +504,7 @@ verifyManifestFile( FUNCTION_LOG_PARAM(BOOL, currentBackup); // Is this possibly a backup currently in progress? FUNCTION_LOG_PARAM(INFO_PG, pgHistory); // Database history FUNCTION_LOG_PARAM_P(UINT, jobErrorTotal); // Pointer to the overall job error total + FUNCTION_TEST_PARAM(LIST, messageList); // List of errors FUNCTION_LOG_END(); Manifest *result = NULL; @@ -388,7 +514,7 @@ verifyManifestFile( const String *const fileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupResult->backupLabel)); // Get the main manifest file - const VerifyInfoFile verifyManifestInfo = verifyInfoFile(fileName, true, cipherPass); + const VerifyInfoFile verifyManifestInfo = verifyInfoFile(fileName, true, cipherPass, messageList); // If the main file did not error, then report on the copy's status and check checksums if (verifyManifestInfo.errorCode == 0) @@ -402,7 +528,7 @@ verifyManifestFile( // Attempt to load the copy and report on it's status but don't keep it in memory const VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile( - strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), false, cipherPass); + strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), false, cipherPass, messageList); // If the copy loaded successfully, then check the checksums if (verifyManifestInfoCopy.errorCode == 0) @@ -410,7 +536,10 @@ verifyManifestFile( // If the manifest and manifest.copy checksums don't match each other than one (or both) of the files could be // corrupt so log a warning but trust main if (!strEq(verifyManifestInfo.checksum, verifyManifestInfoCopy.checksum)) - LOG_DETAIL_FMT("backup '%s' manifest.copy does not match manifest", strZ(backupResult->backupLabel)); + { + String *errorMsg = strNewFmt("backup '%s' manifest.copy does not match manifest", strZ(backupResult->backupLabel)); + verifyErrorNew(messageList, logLevelDetail, errorMsg); + } } } else @@ -423,12 +552,13 @@ verifyManifestFile( currentBackup = false; const VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile( - strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), true, cipherPass); + strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), true, cipherPass, messageList); // If loaded successfully, then return the copy as usable if (verifyManifestInfoCopy.errorCode == 0) { - LOG_DETAIL_FMT("%s/backup.manifest is missing or unusable, using copy", strZ(backupResult->backupLabel)); + String *errorMsg = strNewFmt("%s/backup.manifest is missing or unusable, using copy", strZ(backupResult->backupLabel)); + verifyErrorNew(messageList, logLevelDetail, errorMsg); result = verifyManifestInfoCopy.manifest; } @@ -437,14 +567,16 @@ verifyManifestFile( { backupResult->status = backupMissingManifest; - LOG_DETAIL_FMT("manifest missing for '%s' - backup may have expired", strZ(backupResult->backupLabel)); + String *errorMsg = strNewFmt("manifest missing for '%s' - backup may have expired", strZ(backupResult->backupLabel)); + verifyErrorNew(messageList, logLevelDetail, errorMsg); } } else { backupResult->status = backupInProgress; - LOG_INFO_FMT("backup '%s' appears to be in progress, skipping", strZ(backupResult->backupLabel)); + String *errorMsg = strNewFmt("backup '%s' appears to be in progress, skipping", strZ(backupResult->backupLabel)); + verifyErrorNew(messageList, logLevelInfo, errorMsg); } } @@ -470,11 +602,13 @@ verifyManifestFile( // If the PG data is not found in the backup.info history, then error and reset the result if (!found) { - LOG_INFO_FMT( + String *errorMsg = strNewFmt( "'%s' may not be recoverable - PG data (id %u, version %s, system-id %" PRIu64 ") is not in the backup.info" " history, skipping", strZ(backupResult->backupLabel), manData->pgId, strZ(pgVersionToStr(manData->pgVersion)), manData->pgSystemId); + verifyErrorNew(messageList, logLevelInfo, errorMsg); + manifestFree(result); result = NULL; } @@ -545,12 +679,13 @@ Populate the WAL ranges from the provided, sorted, WAL files list for a given ar ***********************************************************************************************************************************/ static void verifyCreateArchiveIdRange( - const VerifyArchiveResult *const archiveIdResult, StringList *const walFileList, unsigned int *const jobErrorTotal) + const VerifyArchiveResult *const archiveIdResult, StringList *const walFileList, unsigned int *const jobErrorTotal, VariantList *const messageList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM_P(VERIFY_ARCHIVE_RESULT, archiveIdResult); // The result set for the archive Id being processed FUNCTION_TEST_PARAM(STRING_LIST, walFileList); // Sorted (ascending) list of WAL files in a timeline FUNCTION_TEST_PARAM_P(UINT, jobErrorTotal); // Pointer to the overall job error total + FUNCTION_TEST_PARAM(VARIANT_LIST, messageList); // List of errors that occurred during the job FUNCTION_TEST_END(); FUNCTION_AUDIT_HELPER(); @@ -582,7 +717,8 @@ verifyCreateArchiveIdRange( { if (strEq(walSegment, strSubN(strLstGet(walFileList, walFileIdx + 1), 0, WAL_SEGMENT_NAME_SIZE))) { - LOG_INFO_FMT("duplicate WAL '%s' for '%s' exists, skipping", strZ(walSegment), strZ(archiveIdResult->archiveId)); + String *errorMsg = strNewFmt("duplicate WAL '%s' for '%s' exists, skipping", strZ(walSegment), strZ(archiveIdResult->archiveId)); + verifyErrorNew(messageList, logLevelInfo, errorMsg); (*jobErrorTotal)++; @@ -850,7 +986,7 @@ verifyArchive(VerifyJobData *const jobData) // log archiveResult->totalWalFile += strLstSize(jobData->walFileList); - verifyCreateArchiveIdRange(archiveResult, jobData->walFileList, &jobData->jobErrorTotal); + verifyCreateArchiveIdRange(archiveResult, jobData->walFileList, &jobData->jobErrorTotal, jobData->messageList); } } @@ -893,9 +1029,10 @@ verifyArchive(VerifyJobData *const jobData) else { // No valid WAL to process (may be only duplicates or nothing in WAL path) - remove WAL path from the list - LOG_DETAIL_FMT( + String *errorMsg = strNewFmt( "path '%s/%s' does not contain any valid WAL to be processed", strZ(archiveResult->archiveId), strZ(walPath)); + verifyErrorNew(jobData->messageList, logLevelDetail, errorMsg); strLstRemoveIdx(jobData->walPathList, 0); } @@ -916,7 +1053,8 @@ verifyArchive(VerifyJobData *const jobData) else { // Log that no WAL paths exist in the archive Id dir - remove the archive Id from the list (nothing to process) - LOG_DETAIL_FMT("archive path '%s' is empty", strZ(strLstGet(jobData->archiveIdList, 0))); + String *errorMsg = strNewFmt("archive path '%s' is empty", strZ(strLstGet(jobData->archiveIdList, 0))); + verifyErrorNew(jobData->messageList, logLevelDetail, errorMsg); strLstRemoveIdx(jobData->archiveIdList, 0); } } @@ -973,7 +1111,7 @@ verifyBackup(VerifyJobData *const jobData) // Get a usable backup manifest file Manifest *const manifest = verifyManifestFile( - backupResult, jobData->manifestCipherPass, inProgressBackup, jobData->pgHistory, &jobData->jobErrorTotal); + backupResult, jobData->manifestCipherPass, inProgressBackup, jobData->pgHistory, &jobData->jobErrorTotal, jobData->messageList); // If a usable backup.manifest file is not found if (manifest == NULL) @@ -1162,7 +1300,8 @@ verifyBackup(VerifyJobData *const jobData) { // Nothing to process so report an error, free the manifest, set the status, and remove the backup from processing // list - LOG_INFO_FMT("backup '%s' manifest does not contain any target files to verify", strZ(backupResult->backupLabel)); + String *errorMsg = strNewFmt("backup '%s' manifest does not contain any target files to verify", strZ(backupResult->backupLabel)); + verifyErrorNew(jobData->messageList, logLevelInfo, errorMsg); jobData->jobErrorTotal++; @@ -1253,13 +1392,14 @@ Helper function to output a log message based on job result that is not verifyOk ***********************************************************************************************************************************/ static unsigned int verifyLogInvalidResult( - const String *const fileType, const VerifyResult verifyResult, const unsigned int processId, const String *const filePathName) + const String *const fileType, const VerifyResult verifyResult, const unsigned int processId, const String *const filePathName, VariantList *const messageList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING, fileType); // Indicates archive or backup file FUNCTION_TEST_PARAM(ENUM, verifyResult); // Result code from the verifyFile() function FUNCTION_TEST_PARAM(UINT, processId); // Process Id reporting the result FUNCTION_TEST_PARAM(STRING, filePathName); // File for which results are being reported + FUNCTION_TEST_PARAM(LIST, messageList); // List of error messages FUNCTION_TEST_END(); ASSERT(fileType != NULL); @@ -1269,11 +1409,22 @@ verifyLogInvalidResult( // legitimately so it is not necessarily an error so the jobErrorTotal should not be incremented if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR) && verifyResult == verifyFileMissing) { - LOG_WARN_PID_FMT(processId, "%s '%s'", verifyErrorMsg(verifyResult), strZ(filePathName)); + MEM_CONTEXT_TEMP_BEGIN(); + { + String *errorMsg = strNewFmt("%s '%s'", verifyErrorMsg(verifyResult), strZ(filePathName)); + verifyErrorPidNew(messageList, logLevelWarn, processId, errorMsg); + } + MEM_CONTEXT_TEMP_END(); FUNCTION_TEST_RETURN(UINT, 0); } - LOG_INFO_PID_FMT(processId, "%s '%s'", verifyErrorMsg(verifyResult), strZ(filePathName)); + MEM_CONTEXT_TEMP_BEGIN(); + { + String *errorMsg = strNewFmt("%s '%s'", verifyErrorMsg(verifyResult), strZ(filePathName)); + verifyErrorPidNew(messageList, logLevelInfo, processId, errorMsg); + } + MEM_CONTEXT_TEMP_END(); + FUNCTION_TEST_RETURN(UINT, 1); } @@ -1283,7 +1434,7 @@ Helper function to set the currently processing backup label, if any, and check static String * verifySetBackupCheckArchive( const StringList *const backupList, const InfoBackup *const backupInfo, const StringList *const archiveIdList, - const InfoPg *const pgHistory, unsigned int *const jobErrorTotal) + const InfoPg *const pgHistory, unsigned int *const jobErrorTotal, VariantList *const messageList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_LIST, backupList); // List of backup labels in the backup directory @@ -1291,6 +1442,7 @@ verifySetBackupCheckArchive( FUNCTION_TEST_PARAM(STRING_LIST, archiveIdList); // List of archiveIds in the archive directory FUNCTION_TEST_PARAM(INFO_PG, pgHistory); // Pointer to InfoPg of archive.info for accessing PG history FUNCTION_TEST_PARAM_P(UINT, jobErrorTotal); // Pointer to overall job error total + FUNCTION_TEST_PARAM(VARIANT_LIST, messageList); // List of errors that occurred during the job FUNCTION_TEST_END(); String *result = NULL; @@ -1339,7 +1491,8 @@ verifySetBackupCheckArchive( if (!strEmpty(missingFromHistory)) { - LOG_INFO_FMT("archiveIds '%s' are not in the archive.info history list", strZ(missingFromHistory)); + String *errorMsg = strNewFmt("archiveIds '%s' are not in the archive.info history list", strZ(missingFromHistory)); + verifyErrorNew(messageList, logLevelInfo, errorMsg); (*jobErrorTotal)++; } @@ -1426,16 +1579,40 @@ verifyCreateFileErrorsStr( FUNCTION_TEST_RETURN(STRING, result); } +/*********************************************************************************************************************************** +Create file errors KV +***********************************************************************************************************************************/ +static KeyValue * +verifyCreateFileErrorsKv( + const unsigned int errMissing, const unsigned int errChecksum, const unsigned int errSize, const unsigned int errOther) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, errMissing); // Number of files missing + FUNCTION_TEST_PARAM(UINT, errChecksum); // Number of files with checksum errors + FUNCTION_TEST_PARAM(UINT, errSize); // Number of files with invalid size + FUNCTION_TEST_PARAM(UINT, errOther); // Number of files with other errors + FUNCTION_TEST_END(); + + KeyValue *const result = kvNew(); + + kvPut(result, KEY_MISSING_VAR, VARUINT(errMissing)); + kvPut(result, KEY_CHECKSUMINVALID_VAR, VARUINT(errChecksum)); + kvPut(result, KEY_SIZEINVALID_VAR, VARUINT(errSize)); + kvPut(result, KEY_OTHER_VAR, VARUINT(errOther)); + + FUNCTION_TEST_RETURN(KEY_VALUE, result); +} + /*********************************************************************************************************************************** Render the results of the verify command ***********************************************************************************************************************************/ -static String * -verifyRender(const List *const archiveIdResultList, const List *const backupResultList, const bool verboseText) +static void +verifyPrepareResult(const List *const archiveIdResultList, const List *const backupResultList, KeyValue *resultKv) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(LIST, archiveIdResultList); // Result list for all archive Ids in the repo FUNCTION_TEST_PARAM(LIST, backupResultList); // Result list for all backups in the repo - FUNCTION_TEST_PARAM(BOOL, verboseText); // Is verbose output requested? + FUNCTION_TEST_PARAM(KEY_VALUE, resultKv); // Result key-value pair FUNCTION_TEST_END(); FUNCTION_AUDIT_HELPER(); @@ -1443,38 +1620,37 @@ verifyRender(const List *const archiveIdResultList, const List *const backupResu ASSERT(archiveIdResultList != NULL); ASSERT(backupResultList != NULL); - String *const result = strNew(); + // Prepare archive results + VariantList *archivesList = varLstNew(); + VariantList *backupsList = varLstNew(); + VariantList *messageList = varLstNew(); - // Render archive results - if (verboseText && lstEmpty(archiveIdResultList)) - strCatZ(result, "\n archiveId: none found"); - else + if (!lstEmpty(archiveIdResultList)) { for (unsigned int archiveIdx = 0; archiveIdx < lstSize(archiveIdResultList); archiveIdx++) { const VerifyArchiveResult *const archiveIdResult = lstGet(archiveIdResultList, archiveIdx); + KeyValue *const archiveKv = kvNew(); - if (verboseText || archiveIdResult->totalWalFile - archiveIdResult->totalValidWal != 0) - { - strCatFmt( - result, "\n archiveId: %s, total WAL checked: %u, total valid WAL: %u", strZ(archiveIdResult->archiveId), - archiveIdResult->totalWalFile, archiveIdResult->totalValidWal); - } + kvPut(archiveKv, ARCHIVE_KEY_ARCHIVEID_VAR, VARSTR(archiveIdResult->archiveId)); + kvPut(archiveKv, ARCHIVE_KEY_CHECKED_VAR, VARUINT(archiveIdResult->totalWalFile)); + kvPut(archiveKv, KEY_VALID_VAR, VARUINT(archiveIdResult->totalValidWal)); + + unsigned int errMissing = 0; + unsigned int errChecksum = 0; + unsigned int errSize = 0; + unsigned int errOther = 0; if (archiveIdResult->totalWalFile > 0) { - unsigned int errMissing = 0; - unsigned int errChecksum = 0; - unsigned int errSize = 0; - unsigned int errOther = 0; - for (unsigned int walIdx = 0; walIdx < lstSize(archiveIdResult->walRangeList); walIdx++) { const VerifyWalRange *const walRange = lstGet(archiveIdResult->walRangeList, walIdx); - LOG_DETAIL_FMT( + String *errorMsg = strNewFmt( "archiveId: %s, wal start: %s, wal stop: %s", strZ(archiveIdResult->archiveId), strZ(walRange->start), strZ(walRange->stop)); + verifyErrorNew(messageList, logLevelDetail, errorMsg); unsigned int invalidIdx = 0; @@ -1495,22 +1671,26 @@ verifyRender(const List *const archiveIdResultList, const List *const backupResu } } - // Create/append file errors string - if (verboseText || errMissing + errChecksum + errSize + errOther > 0) - strCat(result, verifyCreateFileErrorsStr(errMissing, errChecksum, errSize, errOther, verboseText)); + KeyValue *const fileErrorsKv = verifyCreateFileErrorsKv(errMissing, errChecksum, errSize, errOther); + kvPut(archiveKv, KEY_FILEERRORS_VAR, varNewKv(fileErrorsKv)); } + + kvPut(archiveKv, KEY_MISSING_VAR, VARUINT(errMissing)); + kvPut(archiveKv, KEY_CHECKSUMINVALID_VAR, VARUINT(errChecksum)); + kvPut(archiveKv, KEY_SIZEINVALID_VAR, VARUINT(errSize)); + kvPut(archiveKv, KEY_OTHER_VAR, VARUINT(errOther)); + + varLstAdd(archivesList, varNewKv(archiveKv)); } } - - // Render backup results - if (verboseText && lstEmpty(backupResultList)) - strCatZ(result, "\n backup: none found"); - else + // Prepare backup results + if (!lstEmpty(backupResultList)) { for (unsigned int backupIdx = 0; backupIdx < lstSize(backupResultList); backupIdx++) { const VerifyBackupResult *const backupResult = lstGet(backupResultList, backupIdx); const char *status; + KeyValue *const backupKv = kvNew(); switch (backupResult->status) { @@ -1535,20 +1715,18 @@ verifyRender(const List *const archiveIdResultList, const List *const backupResu } } - if (verboseText || (strcmp(status, "valid") != 0 && strcmp(status, "in-progress") != 0)) - { - strCatFmt( - result, "\n backup: %s, status: %s, total files checked: %u, total valid files: %u", - strZ(backupResult->backupLabel), status, backupResult->totalFileVerify, backupResult->totalFileValid); - } + kvPut(backupKv, BACKUPS_KEY_LABEL_VAR, VARSTR(backupResult->backupLabel)); + kvPut(backupKv, KEY_STATUS_VAR, VARSTRZ(status)); + kvPut(backupKv, BACKUPS_KEY_CHECKED_VAR, VARUINT(backupResult->totalFileVerify)); + kvPut(backupKv, KEY_VALID_VAR, VARUINT(backupResult->totalFileValid)); + + unsigned int errMissing = 0; + unsigned int errChecksum = 0; + unsigned int errSize = 0; + unsigned int errOther = 0; if (backupResult->totalFileVerify > 0) { - unsigned int errMissing = 0; - unsigned int errChecksum = 0; - unsigned int errSize = 0; - unsigned int errOther = 0; - for (unsigned int invalidIdx = 0; invalidIdx < lstSize(backupResult->invalidFileList); invalidIdx++) { const VerifyInvalidFile *const invalidFile = lstGet(backupResult->invalidFileList, invalidIdx); @@ -1563,14 +1741,161 @@ verifyRender(const List *const archiveIdResultList, const List *const backupResu errOther++; } - // Create/append file errors string - if (verboseText || errMissing + errChecksum + errSize + errOther > 0) - strCat(result, verifyCreateFileErrorsStr(errMissing, errChecksum, errSize, errOther, verboseText)); + KeyValue *const fileErrorsKv = verifyCreateFileErrorsKv(errMissing, errChecksum, errSize, errOther); + kvPut(backupKv, KEY_FILEERRORS_VAR, varNewKv(fileErrorsKv)); } + + kvPut(backupKv, KEY_MISSING_VAR, VARUINT(errMissing)); + kvPut(backupKv, KEY_CHECKSUMINVALID_VAR, VARUINT(errChecksum)); + kvPut(backupKv, KEY_SIZEINVALID_VAR, VARUINT(errSize)); + kvPut(backupKv, KEY_OTHER_VAR, VARUINT(errOther)); + varLstAdd(backupsList, varNewKv(backupKv)); } } - FUNCTION_TEST_RETURN(STRING, result); + kvPut(resultKv, KEY_ARCHIVES_VAR, varNewVarLst(archivesList)); + kvPut(resultKv, KEY_BACKUPS_VAR, varNewVarLst(backupsList)); + kvPut(resultKv, KEY_MESSAGES_VAR, varNewVarLst(messageList)); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** + * Render the results of the verify command into text + ***********************************************************************************************************************************/ +static String * +verifyRenderText(const KeyValue *const resultKv, const bool verboseText) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(KEY_VALUE, resultKv); // Result key-value pair + FUNCTION_LOG_PARAM(BOOL, verboseText); // Is verbose output requested? + FUNCTION_LOG_END(); + + ASSERT(resultKv != NULL); + + String *const result = strNew(); + MEM_CONTEXT_TEMP_BEGIN() + { + const VariantList *const messageList = kvGetList(resultKv, KEY_MESSAGES_VAR); + String *resultStr = strNew(); + + const Variant *const varStatus = kvGet(resultKv, KEY_STATUS_VAR); + + // Show stanza and status if we completed all verifyProcess + if (varStatus != NULL) + strCatFmt( + resultStr, "stanza: %s\nstatus: %s", strZ(cfgOptionStr(cfgOptStanza)), + strZ(varStr(varStatus))); + + const VariantList *const archivesList = kvGetList(resultKv, KEY_ARCHIVES_VAR); + + if (verboseText && varLstEmpty(archivesList)) + strCatZ(resultStr, "\n archiveId: none found"); + else + { + for (unsigned int archiveIdx = 0; archiveIdx < varLstSize(archivesList); archiveIdx++) + { + Variant *const archiveKv = varLstGet(archivesList, archiveIdx); + if (archiveKv == NULL) + continue; + + const Variant *const archiveId = kvGet(varKv(archiveKv), ARCHIVE_KEY_ARCHIVEID_VAR); + const Variant *const totalWalFile = kvGet(varKv(archiveKv), ARCHIVE_KEY_CHECKED_VAR); + const Variant *const totalValidWAL = kvGet(varKv(archiveKv), KEY_VALID_VAR); + + if (verboseText || varUInt(totalWalFile) - varUInt(totalValidWAL) != 0) + { + strCatFmt(resultStr,"\n archiveId: %s, total WAL checked: %u, total valid WAL: %u", + strZ(varStr(archiveId)), varUInt(totalWalFile), varUInt(totalValidWAL)); + } + + unsigned int errMissing = varUInt(kvGet(varKv(archiveKv), KEY_MISSING_VAR)); + unsigned int errChecksum = varUInt(kvGet(varKv(archiveKv), KEY_CHECKSUMINVALID_VAR)); + unsigned int errSize = varUInt(kvGet(varKv(archiveKv), KEY_SIZEINVALID_VAR)); + unsigned int errOther = varUInt(kvGet(varKv(archiveKv), KEY_OTHER_VAR)); + + if (varUInt(totalWalFile) > 0 && (verboseText || errMissing + errChecksum + errSize + errOther > 0)) + { + strCat(resultStr, + verifyCreateFileErrorsStr(errMissing, errChecksum, errSize, errOther, verboseText)); + } + } + } + + const VariantList *const backupsList = kvGetList(resultKv, KEY_BACKUPS_VAR); + + if (verboseText && varLstEmpty(backupsList)) + strCatZ(resultStr, "\n backup: none found"); + else + { + for (unsigned int backupIdx = 0; backupIdx < varLstSize(backupsList); backupIdx++) + { + Variant *const backupKv = varLstGet(backupsList, backupIdx); + if (backupKv == NULL) + continue; + + const Variant *const varLabel = kvGet(varKv(backupKv), BACKUPS_KEY_LABEL_VAR); + const Variant *const varStatus = kvGet(varKv(backupKv), KEY_STATUS_VAR); + const Variant *const varTotalFileVerify = kvGet(varKv(backupKv), BACKUPS_KEY_CHECKED_VAR); + const Variant *const varTotalFileValid = kvGet(varKv(backupKv), KEY_VALID_VAR); + + const char *const status = strZ(varStr(varStatus)); + + if (verboseText || (strcmp(status, "valid") != 0 && strcmp(status, "in-progress") != 0)) + { + strCatFmt(resultStr, + "\n backup: %s, status: %s, total files checked: %u, total valid files: %u", + strZ(varStr(varLabel)), status, varUInt(varTotalFileVerify), varUInt(varTotalFileValid)); + } + + unsigned int errMissing = varUInt(kvGet(varKv(backupKv), KEY_MISSING_VAR)); + unsigned int errChecksum = varUInt(kvGet(varKv(backupKv), KEY_CHECKSUMINVALID_VAR)); + unsigned int errSize = varUInt(kvGet(varKv(backupKv), KEY_SIZEINVALID_VAR)); + unsigned int errOther = varUInt(kvGet(varKv(backupKv), KEY_OTHER_VAR)); + + if (varUInt(varTotalFileVerify) > 0 && (verboseText || errMissing + errChecksum + errSize + errOther > 0)) + { + strCat(resultStr, + verifyCreateFileErrorsStr(errMissing, errChecksum, errSize, errOther, verboseText)); + } + } + } + + // Render messages which does not have a log level + for (unsigned int errIdx = 0; errIdx < varLstSize(messageList); errIdx++) + { + const KeyValue *const msg = varKv(varLstGet(messageList, errIdx)); + const String *const message = varStr(kvGet(msg, VERIFY_MSG_KEY_MESSAGE)); + const Variant *varLevel = kvGet(msg, VERIFY_MSG_KEY_LEVEL); + + if (varLevel != NULL) + continue; + + strCatFmt(resultStr, "\n %s", strZ(message)); + } + strCat(result, resultStr); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(STRING, result); +} + +static String * +verifyRenderJson(KeyValue *const resultKv) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(KEY_VALUE, resultKv); // Result key-value pair + FUNCTION_LOG_END(); + + String *const result = strNew(); + + MEM_CONTEXT_TEMP_BEGIN() + { + strCat(result, jsonFromVar(varNewKv(resultKv))); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(STRING, result); } /*********************************************************************************************************************************** @@ -1583,33 +1908,35 @@ verifyProcess(const bool verboseText) FUNCTION_LOG_PARAM(BOOL, verboseText); // Is verbose output requested? FUNCTION_LOG_END(); - String *const result = strNew(); + String *result = strNew(); + bool json = cfgOptionStrId(cfgOptOutput) == CFGOPTVAL_OUTPUT_JSON; MEM_CONTEXT_TEMP_BEGIN() { unsigned int errorTotal = 0; - String *resultStr = strNew(); + KeyValue *resultKv = kvNew(); + VariantList *messageList = varLstNew(); // Get the repo storage in case it is remote and encryption settings need to be pulled down const Storage *const storage = storageRepo(); // Get a usable backup info file - const InfoBackup *const backupInfo = verifyBackupInfoFile(); + const InfoBackup *const backupInfo = verifyBackupInfoFile(messageList); // If a usable backup.info file is not found, then report an error in the log if (backupInfo == NULL) { - strCatZ(resultStr, "\n No usable backup.info file"); + verifyMessageNew(messageList, strNewZ("No usable backup.info file")); errorTotal++; } // Get a usable archive info file - const InfoArchive *const archiveInfo = verifyArchiveInfoFile(); + const InfoArchive *const archiveInfo = verifyArchiveInfoFile(messageList); // If a usable archive.info file is not found, then report an error in the log if (archiveInfo == NULL) { - strCatZ(resultStr, "\n No usable archive.info file"); + verifyMessageNew(messageList, strNewZ("No usable archive.info file")); errorTotal++; } @@ -1623,7 +1950,7 @@ verifyProcess(const bool verboseText) } CATCH_ANY() { - strCatFmt(resultStr, "\n%s", errorMessage()); + verifyMessageNew(messageList, strNewZ(errorMessage())); errorTotal++; } TRY_END(); @@ -1643,6 +1970,7 @@ verifyProcess(const bool verboseText) .walCipherPass = infoPgCipherPass(infoArchivePg(archiveInfo)), .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator), .backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr), + .messageList = messageList, }; // Use backup label if specified via --set @@ -1654,7 +1982,7 @@ verifyProcess(const bool verboseText) { if (!regExpMatchOne(backupRegExpStr, backupLabel)) { - strCatFmt(resultStr, "\n '%s' is not a valid backup label format", strZ(backupLabel)); + verifyMessageNew(messageList, strNewFmt("'%s' is not a valid backup label format", strZ(backupLabel))); backupLabelInvalid = true; errorTotal++; @@ -1672,7 +2000,7 @@ verifyProcess(const bool verboseText) if (!backupLabelInvalid && backupLabel != NULL && strLstEmpty(jobData.backupList)) { - strCatFmt(resultStr, "\n backup set %s is not valid", strZ(backupLabel)); + verifyMessageNew(messageList, strNewFmt("backup set %s is not valid", strZ(backupLabel))); backupLabelInvalid = true; errorTotal++; @@ -1697,8 +2025,10 @@ verifyProcess(const bool verboseText) { // Warn if there are no archives or there are no backups in the repo so that the callback need not try to // distinguish between having processed all of the list or if the list was missing in the first place - if (strLstEmpty(jobData.archiveIdList) || strLstEmpty(jobData.backupList)) - LOG_DETAIL_FMT("no %s exist in the repo", strLstEmpty(jobData.archiveIdList) ? "archives" : "backups"); + if (strLstEmpty(jobData.archiveIdList) || strLstEmpty(jobData.backupList)){ + String *errorMsg = strNewFmt("no %s exist in the repo", strLstEmpty(jobData.archiveIdList) ? "archives" : "backups"); + verifyErrorNew(messageList, logLevelDetail, errorMsg); + } // If there are no archives to process, then set the processing flag to skip to processing the backups if (strLstEmpty(jobData.archiveIdList)) @@ -1706,7 +2036,7 @@ verifyProcess(const bool verboseText) // Set current backup if there is one and verify the archive history on disk is in the database history jobData.currentBackup = verifySetBackupCheckArchive( - jobData.backupList, backupInfo, jobData.archiveIdList, jobData.pgHistory, &jobData.jobErrorTotal); + jobData.backupList, backupInfo, jobData.archiveIdList, jobData.pgHistory, &jobData.jobErrorTotal, messageList); // Create the parallel executor ProtocolParallel *const parallelExec = protocolParallelNew( @@ -1772,7 +2102,7 @@ verifyProcess(const bool verboseText) else { jobData.jobErrorTotal += verifyLogInvalidResult( - fileType, verifyResult, processId, filePathName); + fileType, verifyResult, processId, filePathName, messageList); // Add invalid file to the WAL range verifyAddInvalidWalFile( @@ -1788,7 +2118,7 @@ verifyProcess(const bool verboseText) else { jobData.jobErrorTotal += verifyLogInvalidResult( - fileType, verifyResult, processId, filePathName); + fileType, verifyResult, processId, filePathName, messageList); backupResult->status = backupInvalid; verifyInvalidFileAdd(backupResult->invalidFileList, verifyResult, filePathName); } @@ -1798,10 +2128,10 @@ verifyProcess(const bool verboseText) else { // Log a protocol error and increment the jobErrorTotal - LOG_INFO_PID_FMT( - processId, + String *errorMsg = strNewFmt( "%s %s: [%d] %s", verifyErrorMsg(verifyOtherError), strZ(filePathName), protocolParallelJobErrorCode(job), strZ(protocolParallelJobErrorMessage(job))); + verifyErrorPidNew(messageList, logLevelInfo, processId, errorMsg); jobData.jobErrorTotal++; @@ -1840,21 +2170,42 @@ verifyProcess(const bool verboseText) // ??? Need to do the final reconciliation - checking backup required WAL against, valid WAL - // Report results - resultStr = verifyRender(jobData.archiveIdResultList, jobData.backupResultList, verboseText); + // Prepare results KV + verifyPrepareResult(jobData.archiveIdResultList, jobData.backupResultList, resultKv); } else if (!backupLabelInvalid) - strCatZ(resultStr, "\n no archives or backups exist in the repo"); + { + verifyMessageNew(messageList, strNewZ("no archives or backups exist in the repo")); + } errorTotal += jobData.jobErrorTotal; } - // If verbose output or errors then output results - if (verboseText || errorTotal > 0) + kvPut(resultKv, VERIFY_KEY_STANZA_VAR, VARSTR(cfgOptionStr(cfgOptStanza))); + kvPut(resultKv, KEY_STATUS_VAR, errorTotal > 0 ? VERIFY_KEY_STATUS_ERROR : VERIFY_KEY_STATUS_OK); + + const Variant *resultMessages = kvGet(resultKv, KEY_MESSAGES_VAR); + if (resultMessages != NULL) { - strCatFmt( - result, "stanza: %s\nstatus: %s%s", strZ(cfgOptionStr(cfgOptStanza)), - errorTotal > 0 ? VERIFY_STATUS_ERROR : VERIFY_STATUS_OK, strZ(resultStr)); + VariantList *const resultMessagesList = varVarLst(resultMessages); + for (unsigned int errIdx = 0; errIdx < varLstSize(resultMessagesList); errIdx++) + { + Variant *item = varLstGet(resultMessagesList, errIdx); + varLstAdd(messageList, varDup(item)); + } + } + + kvPut(resultKv, KEY_MESSAGES_VAR, varNewVarLst(messageList)); + + if (json) + { + // Render the results as JSON + strCat(result, verifyRenderJson(resultKv)); + } + else if (verboseText || errorTotal > 0) + { + // Render the results as text + strCat(result, verifyRenderText(resultKv, verboseText)); } } MEM_CONTEXT_TEMP_END(); @@ -1879,7 +2230,8 @@ cmdVerify(void) LOG_INFO_FMT("%s", strZ(result)); // Output to console when requested - if (cfgOptionStrId(cfgOptOutput) == CFGOPTVAL_OUTPUT_TEXT) + if (cfgOptionStrId(cfgOptOutput) == CFGOPTVAL_OUTPUT_TEXT + || cfgOptionStrId(cfgOptOutput) == CFGOPTVAL_OUTPUT_JSON) { ioFdWriteOneStr(STDOUT_FILENO, result); ioFdWriteOneStr(STDOUT_FILENO, LF_STR); diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index c3e5f7b6a4..755c78e449 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -3666,6 +3666,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ( // opt/output PARSE_RULE_VAL_STRID(None), // opt/output PARSE_RULE_VAL_STRID(Text), // opt/output + PARSE_RULE_VAL_STRID(Json), // opt/output ), // opt/output // opt/output PARSE_RULE_OPTIONAL_DEFAULT // opt/output diff --git a/test/define.yaml b/test/define.yaml index 2789e2055e..ccc75b6414 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -986,7 +986,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: verify - total: 14 + total: 17 coverage: - command/verify/file diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index bf7979b673..2a5100974b 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -222,6 +222,7 @@ testRun(void) Manifest *manifest = NULL; unsigned int jobErrorTotal = 0; + VariantList *errorList = varLstNew(); VerifyBackupResult backupResult = {.backupLabel = strNewZ(TEST_BACKUP_LABEL_FULL)}; InfoPg *infoPg = NULL; @@ -256,9 +257,28 @@ testRun(void) harnessLogLevelSet(logLevelDetail); backupResult.status = backupValid; - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg version mismatch"); TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + TEST_RESULT_UINT(varLstSize(errorList), 2, "error list size"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest.copy' for read\"" + "}," + "{" + "\"level\":4," + "\"message\":\"'20181119-152138F' may not be recoverable - PG data (id 1, version 9.6, system-id " HRN_PG_SYSTEMID_95_Z ") is not in the backup.info history, skipping\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG( "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest.copy'" " for read\n" @@ -291,9 +311,30 @@ testRun(void) .comment = "manifest copy - invalid system-id"); backupResult.status = backupValid; - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + + lstClear((List *)errorList); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg system-id mismatch"); TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\"" + "}," + "{\"level\":5,\"message\":\"20181119-152138F/backup.manifest is missing or unusable, using copy\"}," + "{" + "\"level\":4," + "\"message\":\"'20181119-152138F' may not be recoverable - PG data (id 1, version 9.5, system-id 0) is not in the backup.info history, skipping\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG( "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\n" "P00 DETAIL: 20181119-152138F/backup.manifest is missing or unusable, using copy\n" @@ -325,9 +366,32 @@ testRun(void) .comment = "manifest copy - invalid db-id"); backupResult.status = backupValid; - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + lstClear((List *)errorList); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg db-id mismatch"); TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\"" + "}," + "{" + "\"level\":5," + "\"message\":\"20181119-152138F/backup.manifest is missing or unusable, using copy\"" + "}," + "{" + "\"level\":4," + "\"message\":\"'20181119-152138F' may not be recoverable - PG data (id 0, version 9.5, system-id " HRN_PG_SYSTEMID_95_Z ") is not in the backup.info history, skipping\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG( "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\n" "P00 DETAIL: 20181119-152138F/backup.manifest is missing or unusable, using copy\n" @@ -338,13 +402,31 @@ testRun(void) TEST_TITLE("missing main manifest, errored copy"); backupResult.status = backupValid; - + lstClear((List *)errorList); HRN_STORAGE_PUT_Z( storageRepoWrite(), TEST_PATH "/repo/" STORAGE_PATH_BACKUP "/db/" TEST_BACKUP_LABEL_FULL "/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, TEST_INVALID_BACKREST_INFO, .comment = "invalid manifest copy"); - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\"" + "}," + "{" + "\"level\":5," + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /20181119-152138F/backup.manifest.copy\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG( "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\n" "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" @@ -353,13 +435,32 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("current backup true"); + lstClear((List *)errorList); HRN_STORAGE_PUT_Z( storageRepoWrite(), TEST_PATH "/repo/" STORAGE_PATH_BACKUP "/db/" TEST_BACKUP_LABEL_FULL "/" BACKUP_MANIFEST_FILE, TEST_INVALID_BACKREST_INFO, .comment = "invalid manifest"); - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest"); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_PTR(manifest, NULL, "manifest not set"); TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /20181119-152138F/backup.manifest\"" + "}," + "{" + "\"level\":5," + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /20181119-152138F/backup.manifest.copy\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG( "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" " /20181119-152138F/backup.manifest\n" @@ -383,9 +484,24 @@ testRun(void) .comment = "valid manifest"); backupResult.status = backupValid; - TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest"); + lstClear((List *)errorList); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal, errorList), "verify manifest"); TEST_RESULT_PTR_NE(manifest, NULL, "manifest set"); TEST_RESULT_UINT(backupResult.status, backupValid, "manifest usable"); + + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":5," + "\"message\":\"backup '20181119-152138F' manifest.copy does not match manifest\"" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG("P00 DETAIL: backup '20181119-152138F' manifest.copy does not match manifest"); harnessLogLevelReset(); @@ -412,10 +528,11 @@ testRun(void) archiveIdResult->pgWalInfo.size = HRN_PG_WAL_SEGMENT_SIZE_DEFAULT; archiveIdResult->pgWalInfo.version = PG_VERSION_95; + VariantList *errorList = varLstNew(); strLstAddZ(walFileList, "000000020000000200000000-daa497dba64008db824607940609ba1cd7c6c501.gz"); - TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal), "create archiveId WAL range"); + TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal, errorList), "create archiveId WAL range"); TEST_RESULT_UINT(errTotal, 0, "no errors"); TEST_RESULT_UINT(lstSize(((VerifyArchiveResult *)lstGet(archiveIdResultList, 0))->walRangeList), 1, "single range"); TEST_ASSIGN( @@ -423,19 +540,25 @@ testRun(void) "get range"); TEST_RESULT_STR_Z(walRangeResult->start, "000000020000000200000000", "start range"); TEST_RESULT_STR_Z(walRangeResult->stop, "000000020000000200000000", "stop range"); + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + "[]", + "errorList does not match" + ); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("Duplicate WAL only - no range, all removed from list"); lstClear(archiveIdResult->walRangeList); - + lstClear((List *)errorList); // Add a duplicate strLstAddZ(walFileList, "000000020000000200000000"); - TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal), "create archiveId WAL range"); + TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal, errorList), "create archiveId WAL range"); TEST_RESULT_UINT(errTotal, 1, "duplicate WAL error"); TEST_RESULT_UINT(strLstSize(walFileList), 0, "all WAL removed from WAL file list"); TEST_RESULT_UINT(lstSize(archiveIdResult->walRangeList), 0, "no range"); + TEST_RESULT_UINT(varLstSize(errorList), 1, "one error"); TEST_RESULT_LOG("P00 INFO: duplicate WAL '000000020000000200000000' for '9.5-1' exists, skipping"); // ------------------------------------------------------------------------------------------------------------------------- @@ -452,7 +575,8 @@ testRun(void) strLstAddZ(walFileList, "000000020000000200000001"); strLstAddZ(walFileList, "000000020000000200000001"); - TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal), "create archiveId WAL range"); + lstClear((List *)errorList); + TEST_RESULT_VOID(verifyCreateArchiveIdRange(archiveIdResult, walFileList, &errTotal, errorList), "create archiveId WAL range"); TEST_RESULT_UINT(errTotal, 2, "triplicate WAL error at beginning, duplicate WAL at end"); TEST_RESULT_UINT(strLstSize(walFileList), 4, "only duplicate WAL removed from WAL list"); TEST_RESULT_UINT(lstSize(archiveIdResultList), 1, "single archiveId result"); @@ -590,21 +714,23 @@ testRun(void) strLstAddZ(archiveIdList, "11-2"); unsigned int errTotal = 0; + VariantList *errorList = varLstNew(); // Add backup to end of list strLstAddZ(backupList, "20181119-153000F"); strLstAddZ(archiveIdList, "12-3"); TEST_RESULT_STR_Z( - verifySetBackupCheckArchive(backupList, backupInfo, archiveIdList, pgHistory, &errTotal), + verifySetBackupCheckArchive(backupList, backupInfo, archiveIdList, pgHistory, &errTotal, errorList), "20181119-153000F", "current backup, missing archive"); TEST_RESULT_UINT(errTotal, 1, "error logged"); TEST_RESULT_LOG("P00 INFO: archiveIds '12-3' are not in the archive.info history list"); errTotal = 0; + lstClear((List *)errorList); strLstAddZ(archiveIdList, "13-4"); TEST_RESULT_STR_Z( - verifySetBackupCheckArchive(backupList, backupInfo, archiveIdList, pgHistory, &errTotal), + verifySetBackupCheckArchive(backupList, backupInfo, archiveIdList, pgHistory, &errTotal, errorList), "20181119-153000F", "test multiple archiveIds on disk not in archive.info"); TEST_RESULT_UINT(errTotal, 1, "error logged"); TEST_RESULT_LOG("P00 INFO: archiveIds '12-3, 13-4' are not in the archive.info history list"); @@ -613,12 +739,30 @@ testRun(void) TEST_TITLE("verifyLogInvalidResult() - missing file"); TEST_RESULT_UINT( - verifyLogInvalidResult(STORAGE_REPO_ARCHIVE_STR, verifyFileMissing, 0, STRDEF("missingfilename")), + verifyLogInvalidResult(STORAGE_REPO_ARCHIVE_STR, verifyFileMissing, 0, STRDEF("missingfilename"), errorList), 0, "file missing message"); + TEST_RESULT_STR_Z( + jsonFromVar(varNewVarLst(errorList)), + // {uncrustify_off - indentation} + "[" + "{" + "\"level\":4," + "\"message\":\"archiveIds '12-3, 13-4' are not in the archive.info history list\"" + "}," + "{" + "\"level\":3," + "\"message\":\"file missing 'missingfilename'\"," + "\"pid\":0" + "}" + "]", + // {uncrustify_on} + "errorList does not match" + ); + TEST_RESULT_LOG("P00 WARN: file missing 'missingfilename'"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("verifyRender() - missing file, empty invalidList"); + TEST_TITLE("verifyRender - missing file, empty invalidList"); List *archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator); List *backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr); @@ -639,10 +783,36 @@ testRun(void) lstAdd(archiveIdResult.walRangeList, &walRange); lstAdd(archiveIdResultList, &archiveIdResult); + KeyValue *resultKv = kvNew(); + verifyPrepareResult(archiveIdResultList, backupResultList, resultKv); + TEST_RESULT_STR_Z( - verifyRender(archiveIdResultList, backupResultList, cfgOptionBool(cfgOptVerbose)), + verifyRenderText(resultKv, cfgOptionBool(cfgOptVerbose)), "\n" " archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0", "archive: no invalid file list"); + TEST_RESULT_STR_Z( + verifyRenderJson(resultKv), + // {uncrustify_off - indentation} + "{" + "\"archives\":[" + "{" + "\"archiveId\":\"9.6-1\"," + "\"checked\":1," + "\"checksumInvalid\":0," + "\"fileErrors\":{\"checksumInvalid\":0,\"missing\":0,\"other\":0,\"sizeInvalid\":0}," + "\"missing\":0," + "\"other\":0," + "\"sizeInvalid\":0," + "\"valid\":0" + "}" + "]," + "\"backups\":[]," + "\"messages\":[" + "{\"level\":5,\"message\":\"archiveId: 9.6-1, wal start: 0, wal stop: 2\"}" + "]" + "}", + // {uncrustify_on} + "archive: no invalid file list"); VerifyInvalidFile invalidFile = { @@ -661,14 +831,54 @@ testRun(void) lstAdd(backupResult.invalidFileList, &invalidFile); lstAdd(backupResultList, &backupResult); + resultKv = kvNew(); + verifyPrepareResult(archiveIdResultList, backupResultList, resultKv); + TEST_RESULT_STR_Z( - verifyRender(archiveIdResultList, backupResultList, cfgOptionBool(cfgOptVerbose)), + verifyRenderText(resultKv, cfgOptionBool(cfgOptVerbose)), "\n" " archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0\n" " missing: 1\n" " backup: test-backup-label, status: invalid, total files checked: 1, total valid files: 0\n" " missing: 1", "archive file missing, backup file missing, no text, no verbose"); + lstClear((List *)errorList); + TEST_RESULT_STR_Z( + verifyRenderJson(resultKv), + // {uncrustify_off - indentation} + "{" + "\"archives\":[" + "{" + "\"archiveId\":\"9.6-1\"," + "\"checked\":1," + "\"checksumInvalid\":0," + "\"fileErrors\":{\"checksumInvalid\":0,\"missing\":1,\"other\":0,\"sizeInvalid\":0}," + "\"missing\":1," + "\"other\":0," + "\"sizeInvalid\":0," + "\"valid\":0" + "}" + "]," + "\"backups\":[" + "{" + "\"checked\":1," + "\"checksumInvalid\":0," + "\"fileErrors\":{\"checksumInvalid\":0,\"missing\":1,\"other\":0,\"sizeInvalid\":0}," + "\"label\":\"test-backup-label\"," + "\"missing\":1," + "\"other\":0," + "\"sizeInvalid\":0," + "\"status\":\"invalid\"," + "\"valid\":0" + "}" + "]," + "\"messages\":[" + "{\"level\":5,\"message\":\"archiveId: 9.6-1, wal start: 0, wal stop: 2\"}" + "]" + "}", + // {uncrustify_on} + "archive file missing, backup file missing, no text, no verbose, json output"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("verifyAddInvalidWalFile() - file missing (coverage test)"); @@ -787,7 +997,7 @@ testRun(void) " /archive.info\n" "P00 INFO: stanza: db\n" " status: error\n" - " backup info file and archive info file do not match\n" + " backup info file and archive info file do not match\n" " archive: id = 1, version = 9.5, system-id = 10000000000000090500\n" " backup : id = 2, version = 11, system-id = 10000000000000110000\n" " HINT: this may be a symptom of repository corruption!"); @@ -849,6 +1059,323 @@ testRun(void) harnessLogLevelReset(); } + // ***************************************************************************************************************************** + if (testBegin("cmdVerify() - info files JSON output")){ + // Load Parameters + StringList *argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptOutput, "json"); + HRN_CFG_LOAD(cfgCmdVerify, argList); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info invalid checksum, neither backup copy nor archive infos exist"); + + HRN_STORAGE_PUT_Z(storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_INVALID_BACKREST_INFO, .comment = "invalid backup.info"); + + harnessLogLevelSet(logLevelDetail); + + // Redirect stdout to a file + int stdoutSave = dup(STDOUT_FILENO); + const String *stdoutFile = STRDEF(TEST_PATH "/stdout.info"); + + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":[" \ + "{"\ + "\"level\":5,"\ + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /backup.info\""\ + "},{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\""\ + "},{"\ + "\"message\":\"No usable backup.info file\""\ + "},{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info' for read\""\ + "},{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info.copy' for read\""\ + "},{"\ + "\"message\":\"No usable archive.info file\""\ + "}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"error\""\ + "}" + + // Check output of verify command stored in file + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + TEST_RESULT_LOG( + "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" + " /backup.info\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info' for read\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info.copy' for read\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); +#undef EXPECTED_OUTPUT_JSON + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info invalid checksum, backup.info.copy valid, archive.info not exist, archive copy checksum invalid"); + + HRN_STORAGE_PUT_Z( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_INVALID_BACKREST_INFO, + .comment = "invalid archive.info.copy"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + "[backup:current]\n" + TEST_BACKUP_DB1_CURRENT_FULL1 + "\n" + "[db]\n" + TEST_BACKUP_DB1_95 + "\n" + "[db:history]\n" + TEST_BACKUP_DB1_HISTORY, + .comment = "valid backup.info.copy"); + + // Redirect stdout to a file + stdoutSave = dup(STDOUT_FILENO); + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /backup.info\""\ + "},{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info' for read\""\ + "},{"\ + "\"level\":5,\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /archive.info.copy\""\ + "},{"\ + "\"message\":\"No usable archive.info file\""\ + "}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"error\""\ + "}" + + // Check output of verify command stored in file + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + /* Consume log */ + TEST_RESULT_LOG( + "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" + " /backup.info\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info' for read\n" + "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" + " /archive.info.copy\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); +#undef EXPECTED_OUTPUT_JSON + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info and copy valid but checksum mismatch, archive.info checksum invalid, archive.info copy valid"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_BACKUP_INFO_MULTI_HISTORY_BASE, .comment = "valid backup.info"); + HRN_STORAGE_PUT_Z( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_INVALID_BACKREST_INFO, .comment = "invalid archive.info"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_BASE, .comment = "valid archive.info.copy"); + + // Redirect stdout to a file + stdoutSave = dup(STDOUT_FILENO); + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"backup.info.copy does not match backup.info\""\ + "},{"\ + "\"level\":5,"\ + "\"message\":\"invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /archive.info\""\ + "},{"\ + "\"message\":\"backup info file and archive info file do not match\\narchive: id = 1, version = 9.5, system-id = 10000000000000090500\\nbackup : id = 2, version = 11, system-id = 10000000000000110000\\nHINT: this may be a symptom of repository corruption!\""\ + "}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"error\""\ + "}" + + // Check output of verify command stored in file + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + /* Consume log */ + TEST_RESULT_LOG( + "P00 DETAIL: backup.info.copy does not match backup.info\n" + "P00 DETAIL: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' /archive.info\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); +#undef EXPECTED_OUTPUT_JSON + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info and copy valid and checksums match, archive.info and copy valid, but checksum mismatch"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, TEST_BACKUP_INFO_MULTI_HISTORY_BASE, + .comment = "valid backup.info.copy"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .comment = "valid archive.info"); + + // Redirect stdout to a file + stdoutSave = dup(STDOUT_FILENO); + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"archive.info.copy does not match archive.info\""\ + "},"\ + "{\"message\":\"no archives or backups exist in the repo\"}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"ok\""\ + "}" + + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + /* Consume log */ + TEST_RESULT_LOG( + "P00 DETAIL: archive.info.copy does not match archive.info\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); + +#undef EXPECTED_OUTPUT_JSON + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info valid, copy invalid, archive.info valid, copy invalid"); + + HRN_STORAGE_REMOVE(storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, .comment = "remove backup.info.copy"); + HRN_STORAGE_REMOVE(storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, .comment = "remove archive.info.copy"); + + // Redirect stdout to a file + stdoutSave = dup(STDOUT_FILENO); + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\""\ + "},"\ + "{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info.copy' for read\""\ + "},"\ + "{\"message\":\"no archives or backups exist in the repo\"}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"ok\""\ + "}" + + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + /* Consume log */ + TEST_RESULT_LOG( + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/archive/db/archive.info.copy' for read\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); + +#undef EXPECTED_OUTPUT_JSON + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("backup.info and copy missing, archive.info and copy valid"); + + hrnCfgArgRawZ(argList, cfgOptVerbose, "y"); + HRN_CFG_LOAD(cfgCmdVerify, argList); + + HRN_STORAGE_REMOVE(storageRepoWrite(), INFO_BACKUP_PATH_FILE); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + .comment = "valid and matching archive.info.copy"); + + // Redirect stdout to a file + stdoutSave = dup(STDOUT_FILENO); + THROW_ON_SYS_ERROR(freopen(strZ(stdoutFile), "w", stdout) == NULL, FileWriteError, "unable to reopen stdout"); + + // Not in a test wrapper to compare stdout + cmdVerify(); + + // Restore normal stdout + dup2(stdoutSave, STDOUT_FILENO); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info' for read\""\ + "},"\ + "{"\ + "\"level\":5,"\ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\""\ + "},"\ + "{"\ + "\"message\":\"No usable backup.info file\""\ + "}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"error\""\ + "}" + + TEST_STORAGE_GET(storageTest, strZ(stdoutFile), + EXPECTED_OUTPUT_JSON "\n", + .remove = true); + + /* Consume log */ + TEST_RESULT_LOG( + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info' for read\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\n" + "P00 INFO: " EXPECTED_OUTPUT_JSON + ); +#undef EXPECTED_OUTPUT_JSON + harnessLogLevelReset(); + } + // ***************************************************************************************************************************** if (testBegin("verifyFile()")) { @@ -1018,6 +1545,81 @@ testRun(void) harnessLogLevelReset(); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("JSON output"); + + StringList *argListJSON = strLstDup(argListBase); + hrnCfgArgRawZ(argListJSON, cfgOptOutput, "json"); + HRN_CFG_LOAD(cfgCmdVerify, argListJSON); + + #define EXPECTED_OUTPUT_JSON "{"\ + "\"archives\":["\ + "{"\ + "\"archiveId\":\"9.5-1\","\ + "\"checked\":0,"\ + "\"checksumInvalid\":0,"\ + "\"missing\":0,"\ + "\"other\":0,"\ + "\"sizeInvalid\":0,"\ + "\"valid\":0"\ + "},"\ + "{"\ + "\"archiveId\":\"11-2\","\ + "\"checked\":4,"\ + "\"checksumInvalid\":1,"\ + "\"fileErrors\":{\"checksumInvalid\":1,\"missing\":0,\"other\":0,\"sizeInvalid\":1},"\ + "\"missing\":0,"\ + "\"other\":0,"\ + "\"sizeInvalid\":1,"\ + "\"valid\":2"\ + "}"\ + "],"\ + "\"backups\":[],"\ + "\"messages\":["\ + "{"\ + "\"level\":5,"\ + "\"message\":\"no backups exist in the repo\""\ + "},"\ + "{"\ + "\"level\":5,"\ + "\"message\":\"archive path '9.5-1' is empty\""\ + "},"\ + "{"\ + "\"level\":5,"\ + "\"message\":\"path '11-2/0000000100000000' does not contain any valid WAL to be processed\""\ + "},"\ + "{"\ + "\"level\":4,"\ + "\"message\":\"invalid checksum '11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\","\ + "\"pid\":1"\ + "},"\ + "{"\ + "\"level\":4,"\ + "\"message\":\"invalid size '11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'\","\ + "\"pid\":1"\ + "},"\ + "{"\ + "\"level\":5,"\ + "\"message\":\"archiveId: 11-2, wal start: 000000020000000700000FFD, wal stop: 000000020000000800000000\""\ + "}"\ + "],"\ + "\"stanza\":\"db\","\ + "\"status\":\"error\""\ + "}" + + TEST_RESULT_STR_Z( + verifyProcess(cfgOptionBool(cfgOptVerbose)), + EXPECTED_OUTPUT_JSON, + "verifyProcess() json, no verbose"); + +#undef EXPECTED_OUTPUT_JSON + + TEST_RESULT_LOG( + "P01 INFO: invalid checksum" + " '11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n" + "P01 INFO: invalid size" + " '11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("no text output, verbose, with verify failures"); @@ -1040,6 +1642,78 @@ testRun(void) "P01 INFO: invalid size" " '11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("JSON output, verbose, with verify failures"); + + hrnCfgArgRawZ(argListJSON, cfgOptVerbose, "y"); + HRN_CFG_LOAD(cfgCmdVerify, argListJSON); + + // Verify text output, verbose, with verify failures + TEST_RESULT_STR_Z( + verifyProcess(cfgOptionBool(cfgOptVerbose)), + // {uncrustify_off - indentation} + "{" \ + "\"archives\":[" \ + "{" \ + "\"archiveId\":\"9.5-1\"," \ + "\"checked\":0," \ + "\"checksumInvalid\":0," \ + "\"missing\":0," \ + "\"other\":0," \ + "\"sizeInvalid\":0," \ + "\"valid\":0" \ + "}," \ + "{" \ + "\"archiveId\":\"11-2\"," \ + "\"checked\":4," \ + "\"checksumInvalid\":1," \ + "\"fileErrors\":{\"checksumInvalid\":1,\"missing\":0,\"other\":0,\"sizeInvalid\":1}," + "\"missing\":0," \ + "\"other\":0," \ + "\"sizeInvalid\":1," \ + "\"valid\":2" \ + "}" \ + "]," \ + "\"backups\":[]," \ + "\"messages\":[" \ + "{" \ + "\"level\":5," \ + "\"message\":\"no backups exist in the repo\"" \ + "}," \ + "{" \ + "\"level\":5," \ + "\"message\":\"archive path '9.5-1' is empty\"" \ + "}," \ + "{" \ + "\"level\":5," \ + "\"message\":\"path '11-2/0000000100000000' does not contain any valid WAL to be processed\"" \ + "}," \ + "{" \ + "\"level\":4," \ + "\"message\":\"invalid checksum '11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\"," \ + "\"pid\":1" \ + "}," \ + "{" \ + "\"level\":4," \ + "\"message\":\"invalid size '11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'\"," \ + "\"pid\":1" \ + "}," \ + "{" \ + "\"level\":5," \ + "\"message\":\"archiveId: 11-2, wal start: 000000020000000700000FFD, wal stop: 000000020000000800000000\"" \ + "}" \ + "]," \ + "\"stanza\":\"db\"," \ + "\"status\":\"error\"" \ + "}", + // {uncrustify_on} + "JSON verbose, with failures"); + TEST_RESULT_LOG( + "P01 INFO: invalid checksum" + " '11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n" + "P01 INFO: invalid size" + " '11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("text output, verbose, with verify failures"); @@ -1349,6 +2023,90 @@ testRun(void) " Permission denied"); } + // ***************************************************************************************************************************** + if (testBegin("cmdVerify(), verifyProcess() - errors JSON")) + { + StringList *argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptOutput, "json"); + HRN_CFG_LOAD(cfgCmdVerify, argList); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("valid info files, WAL files present, no backups"); + + // Store valid archive/backup info files + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .comment = "valid archive.info"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + .comment = "valid archive.info.copy"); + + #define TEST_NO_CURRENT_BACKUP \ + "[db]\n" \ + TEST_BACKUP_DB2_11 \ + "\n" \ + "[db:history]\n" \ + TEST_BACKUP_DB1_HISTORY \ + "\n" \ + TEST_BACKUP_DB2_HISTORY + + HRN_INFO_PUT(storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .comment = "no current backups"); + + HRN_STORAGE_PATH_CREATE( + storageRepoIdxWrite(0), STORAGE_REPO_BACKUP "/20181119-152800F", .comment = "prior backup path missing manifests"); + + harnessLogLevelSet(logLevelDetail); + + TEST_RESULT_STR_Z( + verifyProcess(cfgOptionBool(cfgOptVerbose)), + // {uncrustify_off - indentation} + "{" \ + "\"archives\":[]," \ + "\"backups\":[" \ + "{" \ + "\"checked\":0," \ + "\"checksumInvalid\":0," \ + "\"label\":\"20181119-152800F\"," \ + "\"missing\":0," \ + "\"other\":0," \ + "\"sizeInvalid\":0," \ + "\"status\":\"in-progress\"," \ + "\"valid\":0" \ + "}" \ + "]," \ + "\"messages\":[" \ + "{" \ + "\"level\":5," \ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\"" \ + "}," \ + "{" \ + "\"level\":5," \ + "\"message\":\"no archives exist in the repo\"" \ + "}," \ + "{" \ + "\"level\":5," \ + "\"message\":\"unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152800F/backup.manifest' for read\"" \ + "}," \ + "{" \ + "\"level\":4," \ + "\"message\":\"backup '20181119-152800F' appears to be in progress, skipping\"" \ + "}" \ + "]," \ + "\"stanza\":\"db\"," \ + "\"status\":\"ok\"" \ + "}", + // {uncrustify_on} + "verifyProcess() JSON missing no total file verify"); + + TEST_RESULT_LOG( + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/backup.info.copy' for read\n" + "P00 DETAIL: no archives exist in the repo\n" + "P00 DETAIL: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152800F/backup.manifest' for read\n" + "P00 INFO: backup '20181119-152800F' appears to be in progress, skipping" + ); + + harnessLogLevelReset(); + } + // ***************************************************************************************************************************** if (testBegin("cmdVerify()")) { @@ -1758,6 +2516,173 @@ testRun(void) " missing: 0, checksum invalid: 1, size invalid: 0, other: 0"); } + // ***************************************************************************************************************************** + if (testBegin("cmdVerify() verbose JSON")) + { + // Load Parameters + StringList *argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptOutput, "json"); + hrnCfgArgRawZ(argList, cfgOptVerbose, "y"); + HRN_CFG_LOAD(cfgCmdVerify, argList); + + #define TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1 \ + "20181119-152900F_20181119-152909D={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-archive-start\":\"000000010000000000000006\",\"backup-archive-stop\":\"000000010000000000000007\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + #define TEST_BACKUP_DB2_CURRENT_FULL1 \ + "20201119-163000F={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-archive-start\":\"000000020000000000000001\",\"backup-archive-stop\":\"000000020000000000000001\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("prior backup verification incomplete - referenced file checked verbose, text output"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .comment = "valid archive.info"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + .comment = "valid archive.info.copy"); + + #define TEST_BACKUP_INFO \ + "[backup:current]\n" \ + TEST_BACKUP_DB1_CURRENT_FULL3 \ + TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1 \ + TEST_BACKUP_DB2_CURRENT_FULL1 \ + "\n" \ + "[db]\n" \ + TEST_BACKUP_DB2_11 \ + "\n" \ + "[db:history]\n" \ + TEST_BACKUP_DB1_HISTORY \ + "\n" \ + TEST_BACKUP_DB2_HISTORY + + HRN_INFO_PUT(storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_BACKUP_INFO); + HRN_INFO_PUT(storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, TEST_BACKUP_INFO); + + // Create valid full backup + #define TEST_MANIFEST_FULL_DB2 \ + TEST_MANIFEST_HEADER \ + TEST_MANIFEST_DB_94 \ + TEST_MANIFEST_OPTION_ALL \ + TEST_MANIFEST_TARGET \ + TEST_MANIFEST_DB \ + TEST_MANIFEST_FILE \ + "pg_data/biind={\"bi\":1,\"bim\":3,\"checksum\":\"ffffffffffffffffffffffffffffffffffffffff\",\"size\":4" \ + ",\"timestamp\":1565282114}\n" \ + TEST_MANIFEST_FILE_DEFAULT \ + TEST_MANIFEST_LINK \ + TEST_MANIFEST_LINK_DEFAULT \ + TEST_MANIFEST_PATH \ + TEST_MANIFEST_PATH_DEFAULT + + // Write manifests for full backup + HRN_INFO_PUT( + storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F/" BACKUP_MANIFEST_FILE, TEST_MANIFEST_FULL_DB2, + .comment = "valid manifest - full"); + HRN_INFO_PUT( + storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, TEST_MANIFEST_FULL_DB2, + .comment = "valid manifest copy - full"); + HRN_STORAGE_PUT_Z( + storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F/pg_data/biind.pgbi", "ZVZV", .comment = "pgbi file"); + + // Create valid diff backup + #define TEST_MANIFEST_DIFF_DB2 \ + TEST_MANIFEST_HEADER \ + TEST_MANIFEST_DB_94 \ + TEST_MANIFEST_OPTION_ALL \ + TEST_MANIFEST_TARGET \ + TEST_MANIFEST_DB \ + "\n" \ + "[target:file]\n" \ + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20181119-152900F\"" \ + ",\"size\":4,\"timestamp\":1565282114}\n" \ + "pg_data/biind={\"bi\":1,\"bim\":3,\"checksum\":\"ffffffffffffffffffffffffffffffffffffffff\"," \ + "\"reference\":\"20181119-152900F\",\"size\":4,\"timestamp\":1565282114}\n" \ + TEST_MANIFEST_FILE_DEFAULT \ + TEST_MANIFEST_LINK \ + TEST_MANIFEST_LINK_DEFAULT \ + TEST_MANIFEST_PATH \ + TEST_MANIFEST_PATH_DEFAULT + + // Write manifests for diff backup + HRN_INFO_PUT( + storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F_20181119-152909D/" BACKUP_MANIFEST_FILE, + TEST_MANIFEST_DIFF_DB2, .comment = "valid manifest - diff"); + HRN_INFO_PUT( + storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F_20181119-152909D/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, + TEST_MANIFEST_DIFF_DB2, .comment = "valid manifest copy - diff"); + + // Put the file referenced by both backups into the full backup + HRN_STORAGE_PUT_Z(storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F/pg_data/PG_VERSION", fileContents); + + TEST_RESULT_STR_Z( + verifyProcess(cfgOptionBool(cfgOptVerbose)), + // {uncrustify_off - indentation} + "{" \ + "\"archives\":[]," \ + "\"backups\":[" \ + "{" \ + "\"checked\":2," \ + "\"checksumInvalid\":2," \ + "\"fileErrors\":{\"checksumInvalid\":2,\"missing\":0,\"other\":0,\"sizeInvalid\":0}," \ + "\"label\":\"20181119-152900F\"," \ + "\"missing\":0," \ + "\"other\":0," \ + "\"sizeInvalid\":0," \ + "\"status\":\"invalid\"," \ + "\"valid\":0" \ + "}," \ + "{" \ + "\"checked\":2," \ + "\"checksumInvalid\":1," \ + "\"fileErrors\":{\"checksumInvalid\":1,\"missing\":0,\"other\":0,\"sizeInvalid\":0}," \ + "\"label\":\"20181119-152900F_20181119-152909D\"," \ + "\"missing\":0," \ + "\"other\":0," \ + "\"sizeInvalid\":0," \ + "\"status\":\"invalid\"," \ + "\"valid\":1" \ + "}" \ + "]," \ + "\"messages\":[" \ + "{" \ + "\"level\":5," \ + "\"message\":\"no archives exist in the repo\"" \ + "}," \ + "{" \ + "\"level\":4," \ + "\"message\":\"invalid checksum '20181119-152900F/pg_data/PG_VERSION'\"," \ + "\"pid\":1" \ + "}," \ + "{" \ + "\"level\":4," \ + "\"message\":\"invalid checksum '20181119-152900F/pg_data/biind.pgbi'\"," \ + "\"pid\":1" \ + "}" \ + "]," \ + "\"stanza\":\"db\"," \ + "\"status\":\"error\"" \ + "}", + // {uncrustify_on} + "verifyProcess() verbose JSON"); + + TEST_RESULT_LOG( + "P01 INFO: invalid checksum '20181119-152900F/pg_data/PG_VERSION'\n" + "P01 INFO: invalid checksum '20181119-152900F/pg_data/biind.pgbi'"); + } + // ***************************************************************************************************************************** if (testBegin("verifyProcess(), none output, not verbose, no failures")) {