From b1117231861bc56f3bd450c45a1f2c3a0635fa47 Mon Sep 17 00:00:00 2001 From: AshokThangavel Date: Thu, 4 Dec 2025 18:51:59 +0530 Subject: [PATCH 01/18] feat: Add JSON, YAML, and Toon output options for zpm test results --- src/cls/IPM/Main.cls | 1 + src/cls/IPM/ResourceProcessor/Test.cls | 41 +++++++-- src/cls/IPM/Test/Abstract.cls | 68 ++++++++++++++ src/cls/IPM/Test/JsonOutput.cls | 89 +++++++++++++++++++ src/cls/IPM/Test/ToonOutput.cls | 75 ++++++++++++++++ src/cls/IPM/Test/YamlOutput.cls | 83 +++++++++++++++++ .../TestResultsOPFormatAndFileGenTest.cls | 60 +++++++++++++ 7 files changed, 408 insertions(+), 9 deletions(-) create mode 100644 src/cls/IPM/Test/Abstract.cls create mode 100644 src/cls/IPM/Test/JsonOutput.cls create mode 100644 src/cls/IPM/Test/ToonOutput.cls create mode 100644 src/cls/IPM/Test/YamlOutput.cls create mode 100644 tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index e2eb416c2..85079fb50 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -75,6 +75,7 @@ Can also specify desired version to update to. + diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index efb500c85..6366e826e 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -189,18 +189,41 @@ Method OnPhase( zkill ^UnitTestRoot $$$ThrowOnError(tSC) - if $data(pParams("UnitTest","JUnitOutput"),tJUnitFile) { - set tPostfix = "-"_$zconvert(pPhase,"L")_"-" - if (..Package '= "") { - set tPostfix = tPostfix_$replace(..Package,".","-")_"-PKG" - } elseif (..Class '= "") { - set tPostfix = tPostfix_$replace(..Class,".","-")_"-CLS" + if $data(pParams("UnitTest"))>1 { + set outputType="" + for { + set outputType = $order(pParams("UnitTest",outputType),1,fileName) + quit:outputType="" + set tPostfix = "-"_$$$lcase(pPhase)_"-" + if (..Package '= "") { + set tPostfix = tPostfix_$replace(..Package,".","-")_"-PKG" + } elseif (..Class '= "") { + set tPostfix = tPostfix_$replace(..Class,".","-")_"-CLS" + } + set outputClass = "%IPM.Test."_outputType + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputType_" output format does not exist.")) + } + set extension = $select(outputType="JsonOutput":".json",outputType="ToonOutput":".toon",outputType="YamlOutput":".yaml",1:".xml") + set fileName = $piece(fileName,".",1,*-1)_tPostfix_extension + set tSC = $classmethod(outputClass,"ToFile",fileName) + $$$ThrowOnError(tSC) + } + } + write ! + if $data(pParams("outputformat"),outputFormat)||('tVerbose) { + write !,"Test result summary",! + // TODO: Move this default format to ^IPM.Config.Test("outputFormat") rather than keeping it hardcoded. + set:$get(outputFormat)="" outputFormat="Toon" + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputType_" output format does not exist.")) } - set tJUnitFile = $piece(tJUnitFile,".",1,*-1)_tPostfix_".xml" - set tSC = ##class(%IPM.Test.JUnitOutput).ToFile(tJUnitFile) + set defaultTestStatus = "failed" + set tSC = $classmethod(outputClass,"OutputToDevice",,defaultTestStatus) $$$ThrowOnError(tSC) + write ! } - // By default, detect and report unit test failures as an error from this phase if $get(pParams("UnitTest","FailuresAreFatal"),1) { do ##class(%IPM.Test.Manager).OutputFailures() diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls new file mode 100644 index 000000000..e1a4b239e --- /dev/null +++ b/src/cls/IPM/Test/Abstract.cls @@ -0,0 +1,68 @@ +/// The class serves as the base class for all the unit test result formatting. +Class %IPM.Test.Abstract Extends %RegisteredObject +{ + +ClassMethod ToFile( + pFileName As %String, + pCaseStatus As %String = "", + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] +{ +} + +ClassMethod OutputToDevice( + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + pTestStatus As %String = "") [ Abstract ] +{ +} + +Query FilteredTestResults( + pInstance As %Integer, + pTestStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") +{ +SELECT +count(*) as TotalCounts, +tinstance.Namespace AS namespace, +tinstance.Duration AS duration, +tinstance.DateTime AS testDateTime, +tsuite.Name AS suiteName, +tcase.Name AS testcaseName, +tmethod.Name AS methodName, +tassert.TestMethod AS testMethod, +tassert.Action AS assertAction, +tassert.Counter AS assertCounter, +tassert.Description AS assertDescription, +tassert.Location AS assertLocation +FROM +%UnitTest_Result.TestInstance tinstance +JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID +JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID +JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID +JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID +WHERE tinstance.ID=:pInstance AND tassert.Status=:pTestStatus +} + +Query GetAllTestResults(pInstance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") +{ +SELECT +count(*) as TotalCounts, +tinstance.Namespace AS namespace, +tinstance.Duration AS duration, +tinstance.DateTime AS testDateTime, +tsuite.Name AS suiteName, +tcase.Name AS testcaseName, +tmethod.Name AS methodName, +tassert.TestMethod AS testMethod, +tassert.Action AS assertAction, +tassert.Counter AS assertCounter, +tassert.Description AS assertDescription, +tassert.Location AS assertLocation +FROM +%UnitTest_Result.TestInstance tinstance +JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID +JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID +JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID +JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID +WHERE tinstance.ID=:pInstance +} + +} diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls new file mode 100644 index 000000000..6c6f0e38a --- /dev/null +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -0,0 +1,89 @@ +Class %IPM.Test.JsonOutput Extends %IPM.Test.Abstract +{ + +ClassMethod ToFile( + pFileName As %String, + pTestStatus As %String = "", + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +{ + set tSC = $$$OK + try { + set fileStream = ##class(%Stream.FileCharacter).%New() + set fileStream.TranslateTable="UTF8" + do fileStream.LinkToFile(pFileName) + set responseJson = ..JSON(pTestIndex, pTestStatus) + do fileStream.Write(responseJson.%ToJSON()) + $$$ThrowOnError(fileStream.%Save()) + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +ClassMethod OutputToDevice( + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + pCaseStatus As %String = "") As %Status +{ + set tSC = $$$OK + try { + set responseJson= ..JSON(pTestIndex, pCaseStatus) + write ! + do responseJson.%ToJSON() + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +ClassMethod JSON( + pTestIndex, + pTestStatus) As %DynamicObject +{ + if pTestStatus'=""{ + set tResult = ..FilteredTestResultsFunc(pTestIndex,pTestStatus) + } else { + set tResult = ..GetAllTestResultsFunc(pTestIndex) + } + set unitTest = {} + set unitTest.results = [] + set (previousID,currentSuite,currentTestcase,suiteObj,testcaseObj) = "" + + while tResult.%Next() { + if previousID = "" { + set unitTest.id = pTestIndex + set unitTest.namespace = tResult.namespace + set unitTest.duration = tResult.duration + set unitTest.testDateTime = tResult.testDateTime + } + set previousID = pTestIndex + if tResult.suiteName '= currentSuite { + set currentSuite = tResult.suiteName + set suiteObj = { + "suiteName": (currentSuite), + "testcases": [] + } + do unitTest.results.%Push(suiteObj) + set currentTestcase = "" + } + if tResult.testcaseName '= currentTestcase { + set currentTestcase = tResult.testcaseName + set testcaseObj = { + "testcaseName": (currentTestcase), + "methods": [] + } + do suiteObj.testcases.%Push(testcaseObj) + } + set methodObj = { + "methodName": (tResult.methodName), + "testMethod": (tResult.testMethod), + "assertAction": (tResult.assertAction), + "assertCounter": (tResult.assertCounter), + "assertDescription": (tResult.assertDescription), + "assertLocation": (tResult.assertLocation) + } + do testcaseObj.methods.%Push(methodObj) + } + return unitTest +} + +} diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls new file mode 100644 index 000000000..5a689862a --- /dev/null +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -0,0 +1,75 @@ +Class %IPM.Test.ToonOutput Extends %IPM.Test.Abstract +{ + +ClassMethod ToFile( + pFileName As %String, + pTestStatus As %String = "", + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +{ + set tSC = $$$OK + try { + set fileStream = ##class(%Stream.FileCharacter).%New() + set fileStream.TranslateTable="UTF8" + do fileStream.LinkToFile(pFileName) + if pTestStatus'=""{ + set tResult = ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + } else { + set tResult = ..GetAllTestResultsFunc(pTestIndex) + } + set currentID="" + while tResult.%Next() { + if currentID = "" { + set currentID = pTestIndex + do fileStream.WriteLine("unitTest:") + do fileStream.WriteLine(" id: "_pTestIndex) + do fileStream.WriteLine(" namespace: "_tResult.namespace) + do fileStream.WriteLine(" duration: "_tResult.duration) + do fileStream.WriteLine(" testDateTime: "_tResult.testDateTime) + do fileStream.WriteLine() + do fileStream.WriteLine("results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + } + set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_pTestIndex_","_ + tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" + do fileStream.WriteLine(data) + } + $$$ThrowOnError(fileStream.%Save()) + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +ClassMethod OutputToDevice( + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + pTestStatus As %String = "") As %Status +{ + set tSC = $$$OK + try { + if pTestStatus'=""{ + set tResult = ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + } else { + set tResult = ..GetAllTestResultsFunc(pTestIndex) + } + set currentID="" + while tResult.%Next() { + if currentID = "" { + set currentID = pTestIndex + write !,"unitTest:" + write !," id: "_pTestIndex + write !," namespace: "_tResult.namespace + write !," duration: "_tResult.duration + write !," testDateTime: "_tResult.testDateTime + write ! + write !,"results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + } + set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_pTestIndex_","_ + tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" + write !,data + } + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +} diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls new file mode 100644 index 000000000..019c59755 --- /dev/null +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -0,0 +1,83 @@ +Class %IPM.Test.YamlOutput Extends %IPM.Test.Abstract +{ + +ClassMethod ToFile( + pFileName As %String, + pTestStatus As %String = "", + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +{ + set tSC = $$$OK + try { + set fileStream = ##class(%Stream.FileCharacter).%New() + set fileStream.TranslateTable="UTF8" + do fileStream.LinkToFile(pFileName) + do fileStream.CopyFrom(..YAML(pTestIndex, pTestStatus)) + $$$ThrowOnError(fileStream.%Save()) + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +ClassMethod OutputToDevice( + pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + pTestStatus As %String = "") As %Status +{ + set tSC = $$$OK + try { + set yamlStream = ..YAML(pTestIndex, pTestStatus) + write ! + while 'yamlStream.AtEnd { + write yamlStream.Read() + } + } catch e { + set tSC = e.AsStatus() + } + return tSC +} + +ClassMethod YAML( + pTestIndex = {$order(^UnitTest.Result(""),-1)}, + pTestStatus As %String = "") As %Stream.TmpCharacter +{ + if pTestStatus'=""{ + set tResult= ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + } else { + set tResult= ..GetAllTestResultsFunc(pTestIndex) + } + set yamlStream = ##class(%Stream.TmpCharacter).%New() + set (yaml,currentID,currentSuite,currentTestcase) = "" + while tResult.%Next() { + if currentID = "" { + set currentID = pTestIndex + do yamlStream.WriteLine("unitTest:") + do yamlStream.WriteLine(" id: "_pTestIndex) + do yamlStream.WriteLine(" namespace: """_tResult.namespace_"""") + do yamlStream.WriteLine(" duration: "_tResult.duration) + do yamlStream.WriteLine(" testDateTime: """_tResult.testDateTime_"""") + do yamlStream.WriteLine( "") + do yamlStream.WriteLine(" results:") + } + if tResult.suiteName '= currentSuite { + set currentSuite = tResult.suiteName + set currentTestcase = "" + do yamlStream.WriteLine(" - suiteName: """_tResult.suiteName_"""") + do yamlStream.WriteLine(" testcases:") + } + if tResult.testcaseName '= currentTestcase { + set currentTestcase = tResult.testcaseName + do yamlStream.WriteLine(" - testcaseName: """_tResult.testcaseName_"""") + do yamlStream.WriteLine(" methods:") + } + do yamlStream.WriteLine(" - methodName: """_tResult.methodName_"""") + //do yamlStream.WriteLine(" testMethod: """_tResult.testMethod_"""") + do yamlStream.WriteLine(" assertAction: """_tResult.assertAction_"""") + do yamlStream.WriteLine(" assertCounter: "_tResult.assertCounter) + //do yamlStream.WriteLine(" assertDescription: |") + //do yamlStream.WriteLine(" "_$replace(tResult.assertDescription, $c(10), $c(10)_" ")) + do yamlStream.WriteLine(" assertLocation: """_tResult.assertLocation_"""") + } + return yamlStream +} + +} diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls new file mode 100644 index 000000000..abe8abc9b --- /dev/null +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -0,0 +1,60 @@ +/// Unit Test Class to validate the ZPM 'test' command output configuration. +/// This class ensures two primary functions work correctly: +/// 1. Console Formatting: Verifies the `-f` / `-output` flags correctly +/// format test results for terminal display (e.g., YAML, JSON). +/// 2. File Generation: Verifies the `-DUnitTest.Output` definitions +/// successfully create and populate the structured results files (e.g., .json, .yaml, .toon). +Class Test.PM.Unit.TestResultsOPFormatAndFileGenTest Extends %UnitTest.TestCase +{ + +/// generate .yaml,.json,.toon files +Method TestResultFileGeneration() +{ + #define NormalizeFilename(%file) ##class(%File).NormalizeFilename(%file) + + do $$$LogMessage("This file generation picks the last or current unit test id and generate the reports") + set fileDir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") + if '##class(%File).DirectoryExists(fileDir){ + set status = ##class(%File).CreateDirectoryChain(fileDir) + do $$$AssertStatusOK(status,"Directory created: "_fileDir) + } + do $$$LogMessage("Start generating the reports") + set fileName = $$$NormalizeFilename(fileDir_"/test.yaml") + set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName) + do $$$AssertStatusOK(status,"yaml file generated successfully in "_fileDir) + + set fileName = $$$NormalizeFilename(fileDir_"/test.Json") + set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName) + do $$$AssertStatusOK(status,"Json file generated successfully in "_fileDir) + + set fileName = $$$NormalizeFilename(fileDir_"/test.toon") + set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName) + do $$$AssertStatusOK(status,"Toon file generated successfully in "_fileDir) + + do ..ShowGeneratedFilesAndCleaup(fileDir) + + //set status = ##class(%File).RemoveDirectory(fileDir) + //do $$$AssertStatusOK(status,"Deleted the directory "_fileDir) +} + +Method ShowGeneratedFilesAndCleaup(fileDir As %String) +{ + do $$$LogMessage("Display the generated unit test report files") + set fileSet = ##class(%File).FileSetFunc(fileDir) + while fileSet.%Next(){ + set file = fileSet.Name + set fileNames(file) = fileSet.ItemName + do $$$LogMessage("Generated file "_file) + } + do $$$LogMessage("Started Cleanup the generated unit test report files") + set file = "" + for { + set file = $order(fileNames(file),1,fileName) + quit:file="" + set status = ##class(%File).Delete(file) + do $$$AssertStatusOK(status," File '"_fileName_"' has been deleted successfully") + } + do $$$LogMessage("File cleanup completed") +} + +} From 7421a94347256167cf774a2d29a0b3b840901985 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Tue, 21 Apr 2026 10:16:09 -0400 Subject: [PATCH 02/18] docs: Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6470f007..55f66de4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #959: In ORAS repos, external name can now be used interchangeably with (default) name for `install` and `update`, i.e. a module published with its (default) name can be installed using its external name. - #951: The `unpublish` command will skip user confirmation prompt if the `-force` flag is provided. - #1018: Require module name for uninstall when not using the -all flag +- #971: Adds support for JSON, YAML, and Toon formats via the -f flag and new -DUnitTest.*Output directives. ### Changed - #316: All parameters, except developer mode, included with a `load`, `install` or `update` command will be propagated to dependencies From 6fec04e751e7118cf00dd01c26131cb059f6b233 Mon Sep 17 00:00:00 2001 From: AshokThangavel Date: Mon, 16 Mar 2026 22:30:51 +0530 Subject: [PATCH 03/18] refactor: updated the code based on comments --- preload/cls/IPM/Installer.cls | 1 + src/cls/IPM/Repo/UniversalSettings.cls | 17 ++++- src/cls/IPM/ResourceProcessor/Test.cls | 20 ++++-- src/cls/IPM/Test/Abstract.cls | 68 +++++++++---------- src/cls/IPM/Test/JsonOutput.cls | 54 ++++++++------- src/cls/IPM/Test/ToonOutput.cls | 56 +++++++-------- src/cls/IPM/Test/YamlOutput.cls | 56 +++++++-------- tests/unit_tests/Test/PM/Unit/CLI.cls | 12 ++++ .../TestResultsOPFormatAndFileGenTest.cls | 11 ++- 9 files changed, 168 insertions(+), 127 deletions(-) diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls index 26970544b..02916635b 100644 --- a/preload/cls/IPM/Installer.cls +++ b/preload/cls/IPM/Installer.cls @@ -113,6 +113,7 @@ ClassMethod ZPMInit( $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("UseStandalonePip", "", 0)) $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0, 0)) $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("DefaultLogEntryLimit",20, 0)) + $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("TestReportFormat","toon", 0)) quit $$$OK } diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls index e1ff1f3ed..f644c3dc7 100644 --- a/src/cls/IPM/Repo/UniversalSettings.cls +++ b/src/cls/IPM/Repo/UniversalSettings.cls @@ -45,7 +45,10 @@ Parameter SemVerPostRelease = "SemVerPostRelease"; /// to retain IPM history records before they are eligible for cleanup. Parameter HistoryRetain = "history_retain"; -Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip,SemVerPostRelease,DefaultLogEntryLimit,HistoryRetain"; +/// Specifies the serialization format (JSON, TOON, YAML) for unit and integration test results in the shell. +Parameter TestReportFormat = "TestReportFormat"; + +Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip,SemVerPostRelease,DefaultLogEntryLimit,HistoryRetain,TestReportFormat"; /// Returns configArray, that includes all configurable settings ClassMethod GetAll(Output configArray) As %Status @@ -190,4 +193,16 @@ ClassMethod GetHistoryRetain() As %Integer return ..GetValue(..#HistoryRetain) } +ClassMethod SetTestReportFormat( + val As %String, + overwrite As %Boolean = 1) As %Boolean +{ + return ..SetValue(..#TestReportFormat, val, overwrite) +} + +ClassMethod GetTestReportFormat() As %String +{ + return ..GetValue(..#TestReportFormat) +} + } diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 6366e826e..070aba8f4 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -213,22 +213,30 @@ Method OnPhase( write ! if $data(pParams("outputformat"),outputFormat)||('tVerbose) { write !,"Test result summary",! - // TODO: Move this default format to ^IPM.Config.Test("outputFormat") rather than keeping it hardcoded. - set:$get(outputFormat)="" outputFormat="Toon" + set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + if defaultOutputFormat'="" { + set outputFormat = defaultOutputFormat + } + else { + if $get(outputFormat)="" { + set outputFormat="Toon" + } + } + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" if '$$$defClassDefined(outputClass) { $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputType_" output format does not exist.")) } set defaultTestStatus = "failed" - set tSC = $classmethod(outputClass,"OutputToDevice",,defaultTestStatus) - $$$ThrowOnError(tSC) + set sc = $classmethod(outputClass,"OutputToDevice",,defaultTestStatus) + $$$ThrowOnError(sc) write ! } // By default, detect and report unit test failures as an error from this phase if $get(pParams("UnitTest","FailuresAreFatal"),1) { do ##class(%IPM.Test.Manager).OutputFailures() - set tSC = ##class(%IPM.Test.Manager).GetLastStatus() - $$$ThrowOnError(tSC) + set sc = ##class(%IPM.Test.Manager).GetLastStatus() + $$$ThrowOnError(sc) } write ! } diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index e1a4b239e..c64cc3817 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -3,66 +3,66 @@ Class %IPM.Test.Abstract Extends %RegisteredObject { ClassMethod ToFile( - pFileName As %String, - pCaseStatus As %String = "", - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + FileName As %String, + CaseStatus As %String = "", + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] { } ClassMethod OutputToDevice( - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - pTestStatus As %String = "") [ Abstract ] + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + TestStatus As %String = "") [ Abstract ] { } Query FilteredTestResults( - pInstance As %Integer, - pTestStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") + Instance As %Integer, + TestStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") { SELECT -count(*) as TotalCounts, -tinstance.Namespace AS namespace, -tinstance.Duration AS duration, -tinstance.DateTime AS testDateTime, -tsuite.Name AS suiteName, -tcase.Name AS testcaseName, -tmethod.Name AS methodName, -tassert.TestMethod AS testMethod, -tassert.Action AS assertAction, -tassert.Counter AS assertCounter, -tassert.Description AS assertDescription, -tassert.Location AS assertLocation + count(*) as TotalCounts, + tinstance.Namespace AS namespace, + tinstance.Duration AS duration, + tinstance.DateTime AS testDateTime, + tsuite.Name AS suiteName, + tcase.Name AS testcaseName, + tmethod.Name AS methodName, + tassert.TestMethod AS testMethod, + tassert.Action AS assertAction, + tassert.Counter AS assertCounter, + tassert.Description AS assertDescription, + tassert.Location AS assertLocation FROM %UnitTest_Result.TestInstance tinstance JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:pInstance AND tassert.Status=:pTestStatus +WHERE tinstance.ID=:Instance AND tassert.Status=:TestStatus } -Query GetAllTestResults(pInstance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") +Query GetAllTestResults(Instance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") { SELECT -count(*) as TotalCounts, -tinstance.Namespace AS namespace, -tinstance.Duration AS duration, -tinstance.DateTime AS testDateTime, -tsuite.Name AS suiteName, -tcase.Name AS testcaseName, -tmethod.Name AS methodName, -tassert.TestMethod AS testMethod, -tassert.Action AS assertAction, -tassert.Counter AS assertCounter, -tassert.Description AS assertDescription, -tassert.Location AS assertLocation + count(*) as TotalCounts, + tinstance.Namespace AS namespace, + tinstance.Duration AS duration, + tinstance.DateTime AS testDateTime, + tsuite.Name AS suiteName, + tcase.Name AS testcaseName, + tmethod.Name AS methodName, + tassert.TestMethod AS testMethod, + tassert.Action AS assertAction, + tassert.Counter AS assertCounter, + tassert.Description AS assertDescription, + tassert.Location AS assertLocation FROM %UnitTest_Result.TestInstance tinstance JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:pInstance +WHERE tinstance.ID=:Instance } } diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 6c6f0e38a..dac91487d 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -2,47 +2,51 @@ Class %IPM.Test.JsonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - pFileName As %String, - pTestStatus As %String = "", - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + FileName As %String, + TestStatus As %String = "", + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { - set tSC = $$$OK + set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable="UTF8" - do fileStream.LinkToFile(pFileName) - set responseJson = ..JSON(pTestIndex, pTestStatus) - do fileStream.Write(responseJson.%ToJSON()) + set fileStream.TranslateTable = "UTF8" + $$$ThrowOnError(fileStream.LinkToFile(FileName)) + set responseJson = ..JSON(TestIndex, TestStatus) + if $isobject(responseJson) { + do fileStream.Write(responseJson.%ToJSON()) + } $$$ThrowOnError(fileStream.%Save()) - } catch e { - set tSC = e.AsStatus() + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } ClassMethod OutputToDevice( - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, pCaseStatus As %String = "") As %Status { - set tSC = $$$OK + set sc = $$$OK try { - set responseJson= ..JSON(pTestIndex, pCaseStatus) + set responseJson = ..JSON(TestIndex, pCaseStatus) write ! - do responseJson.%ToJSON() - } catch e { - set tSC = e.AsStatus() + if $isobject(responseJson) { + do responseJson.%ToJSON() + } + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } ClassMethod JSON( - pTestIndex, - pTestStatus) As %DynamicObject + TestIndex As %Integer, + TestStatus As %String) As %DynamicObject { - if pTestStatus'=""{ - set tResult = ..FilteredTestResultsFunc(pTestIndex,pTestStatus) + if TestStatus'="" { + set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) } else { - set tResult = ..GetAllTestResultsFunc(pTestIndex) + set tResult = ..GetAllTestResultsFunc(TestIndex) } set unitTest = {} set unitTest.results = [] @@ -50,12 +54,12 @@ ClassMethod JSON( while tResult.%Next() { if previousID = "" { - set unitTest.id = pTestIndex + set unitTest.id = TestIndex set unitTest.namespace = tResult.namespace set unitTest.duration = tResult.duration set unitTest.testDateTime = tResult.testDateTime } - set previousID = pTestIndex + set previousID = TestIndex if tResult.suiteName '= currentSuite { set currentSuite = tResult.suiteName set suiteObj = { diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index 5a689862a..f0826a21f 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -2,74 +2,76 @@ Class %IPM.Test.ToonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - pFileName As %String, - pTestStatus As %String = "", - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + FileName As %String, + TestStatus As %String = "", + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { - set tSC = $$$OK + set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable="UTF8" - do fileStream.LinkToFile(pFileName) - if pTestStatus'=""{ - set tResult = ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + set fileStream.TranslateTable = "UTF8" + $$$ThrowOnError(fileStream.LinkToFile(FileName)) + + if TestStatus'="" { + set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) } else { - set tResult = ..GetAllTestResultsFunc(pTestIndex) + set tResult = ..GetAllTestResultsFunc(TestIndex) } + set currentID="" while tResult.%Next() { if currentID = "" { - set currentID = pTestIndex + set currentID = TestIndex do fileStream.WriteLine("unitTest:") - do fileStream.WriteLine(" id: "_pTestIndex) + do fileStream.WriteLine(" id: "_TestIndex) do fileStream.WriteLine(" namespace: "_tResult.namespace) do fileStream.WriteLine(" duration: "_tResult.duration) do fileStream.WriteLine(" testDateTime: "_tResult.testDateTime) do fileStream.WriteLine() do fileStream.WriteLine("results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") } - set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_pTestIndex_","_ + set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_TestIndex_","_ tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" do fileStream.WriteLine(data) } $$$ThrowOnError(fileStream.%Save()) - } catch e { - set tSC = e.AsStatus() + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } ClassMethod OutputToDevice( - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - pTestStatus As %String = "") As %Status + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + TestStatus As %String = "") As %Status { - set tSC = $$$OK + set sc = $$$OK try { - if pTestStatus'=""{ - set tResult = ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + if TestStatus'="" { + set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) } else { - set tResult = ..GetAllTestResultsFunc(pTestIndex) + set tResult = ..GetAllTestResultsFunc(TestIndex) } set currentID="" while tResult.%Next() { if currentID = "" { - set currentID = pTestIndex + set currentID = TestIndex write !,"unitTest:" - write !," id: "_pTestIndex + write !," id: "_TestIndex write !," namespace: "_tResult.namespace write !," duration: "_tResult.duration write !," testDateTime: "_tResult.testDateTime write ! write !,"results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" } - set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_pTestIndex_","_ + set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_TestIndex_","_ tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" write !,data } - } catch e { - set tSC = e.AsStatus() + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } } diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 019c59755..bc6c1d229 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -2,60 +2,60 @@ Class %IPM.Test.YamlOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - pFileName As %String, - pTestStatus As %String = "", - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + FileName As %String, + TestStatus As %String = "", + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { - set tSC = $$$OK + set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable="UTF8" - do fileStream.LinkToFile(pFileName) - do fileStream.CopyFrom(..YAML(pTestIndex, pTestStatus)) + set fileStream.TranslateTable = "UTF8" + $$$ThrowOnError(fileStream.LinkToFile(FileName)) + do fileStream.CopyFrom(..YAML(TestIndex, TestStatus)) $$$ThrowOnError(fileStream.%Save()) - } catch e { - set tSC = e.AsStatus() + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } ClassMethod OutputToDevice( - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - pTestStatus As %String = "") As %Status + TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + TestStatus As %String = "") As %Status { - set tSC = $$$OK + set sc = $$$OK try { - set yamlStream = ..YAML(pTestIndex, pTestStatus) + set yamlStream = ..YAML(TestIndex, TestStatus) write ! while 'yamlStream.AtEnd { write yamlStream.Read() } - } catch e { - set tSC = e.AsStatus() + } catch ex { + set sc = ex.AsStatus() } - return tSC + return sc } ClassMethod YAML( - pTestIndex = {$order(^UnitTest.Result(""),-1)}, - pTestStatus As %String = "") As %Stream.TmpCharacter + TestIndex = {$order(^UnitTest.Result(""),-1)}, + TestStatus As %String = "") As %Stream.TmpCharacter { - if pTestStatus'=""{ - set tResult= ..FilteredTestResultsFunc(pTestIndex, pTestStatus) + if TestStatus'="" { + set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) } else { - set tResult= ..GetAllTestResultsFunc(pTestIndex) + set tResult = ..GetAllTestResultsFunc(TestIndex) } set yamlStream = ##class(%Stream.TmpCharacter).%New() set (yaml,currentID,currentSuite,currentTestcase) = "" while tResult.%Next() { if currentID = "" { - set currentID = pTestIndex + set currentID = TestIndex do yamlStream.WriteLine("unitTest:") - do yamlStream.WriteLine(" id: "_pTestIndex) + do yamlStream.WriteLine(" id: "_TestIndex) do yamlStream.WriteLine(" namespace: """_tResult.namespace_"""") do yamlStream.WriteLine(" duration: "_tResult.duration) do yamlStream.WriteLine(" testDateTime: """_tResult.testDateTime_"""") - do yamlStream.WriteLine( "") + do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") } if tResult.suiteName '= currentSuite { @@ -70,11 +70,11 @@ ClassMethod YAML( do yamlStream.WriteLine(" methods:") } do yamlStream.WriteLine(" - methodName: """_tResult.methodName_"""") - //do yamlStream.WriteLine(" testMethod: """_tResult.testMethod_"""") + do yamlStream.WriteLine(" testMethod: """_tResult.testMethod_"""") do yamlStream.WriteLine(" assertAction: """_tResult.assertAction_"""") do yamlStream.WriteLine(" assertCounter: "_tResult.assertCounter) - //do yamlStream.WriteLine(" assertDescription: |") - //do yamlStream.WriteLine(" "_$replace(tResult.assertDescription, $c(10), $c(10)_" ")) + do yamlStream.WriteLine(" assertDescription: |") + do yamlStream.WriteLine(" "_$replace(tResult.assertDescription, $char(10), $char(10)_" ")) do yamlStream.WriteLine(" assertLocation: """_tResult.assertLocation_"""") } return yamlStream diff --git a/tests/unit_tests/Test/PM/Unit/CLI.cls b/tests/unit_tests/Test/PM/Unit/CLI.cls index 8e9f2329a..797ce4541 100644 --- a/tests/unit_tests/Test/PM/Unit/CLI.cls +++ b/tests/unit_tests/Test/PM/Unit/CLI.cls @@ -352,4 +352,16 @@ Method TestUninstallWithoutModuleName() do $$$AssertNotTrue(exists, "Module removed successfully.") } +/// Specifies the serialization format (json, toon, yaml) for unit and integration test results in the shell. +Method TestReportFormatConfiguration() +{ + do ..RunCommand("config set TestReportFormat json") + set format = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + do $$$AssertEquals(format, "json", "Verify TestReportFormat is set to JSON") + + do ..RunCommand("config set TestReportFormat yaml") + set format = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + do $$$AssertEquals(format, "yaml", "Verify TestReportFormat is set to YAML") +} + } diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index abe8abc9b..354eaa0d3 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -13,11 +13,13 @@ Method TestResultFileGeneration() #define NormalizeFilename(%file) ##class(%File).NormalizeFilename(%file) do $$$LogMessage("This file generation picks the last or current unit test id and generate the reports") + set fileDir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") if '##class(%File).DirectoryExists(fileDir){ set status = ##class(%File).CreateDirectoryChain(fileDir) do $$$AssertStatusOK(status,"Directory created: "_fileDir) } + do $$$LogMessage("Start generating the reports") set fileName = $$$NormalizeFilename(fileDir_"/test.yaml") set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName) @@ -32,16 +34,13 @@ Method TestResultFileGeneration() do $$$AssertStatusOK(status,"Toon file generated successfully in "_fileDir) do ..ShowGeneratedFilesAndCleaup(fileDir) - - //set status = ##class(%File).RemoveDirectory(fileDir) - //do $$$AssertStatusOK(status,"Deleted the directory "_fileDir) } -Method ShowGeneratedFilesAndCleaup(fileDir As %String) +Method ShowGeneratedFilesAndCleaup(FileDir As %String) { do $$$LogMessage("Display the generated unit test report files") - set fileSet = ##class(%File).FileSetFunc(fileDir) - while fileSet.%Next(){ + set fileSet = ##class(%File).FileSetFunc(FileDir) + while fileSet.%Next() { set file = fileSet.Name set fileNames(file) = fileSet.ItemName do $$$LogMessage("Generated file "_file) From 7fcb1b19d960d844d99139414fe57a847edb8c2b Mon Sep 17 00:00:00 2001 From: AshokThangavel Date: Mon, 16 Mar 2026 22:42:43 +0530 Subject: [PATCH 04/18] Refactor: add format alias and clarify output-format description --- src/cls/IPM/Main.cls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 85079fb50..2c7cc4154 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -75,7 +75,7 @@ Can also specify desired version to update to. - + From 2a7a9c26d340f0d285c4d190cd8af15a60fe5a2a Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Apr 2026 10:22:38 -0400 Subject: [PATCH 05/18] First pass refactor --- src/cls/IPM/ResourceProcessor/Test.cls | 15 ++--- src/cls/IPM/Test/Abstract.cls | 26 ++++---- src/cls/IPM/Test/JUnitOutput.cls | 41 +++++++++--- src/cls/IPM/Test/JsonOutput.cls | 59 ++++++++--------- src/cls/IPM/Test/ToonOutput.cls | 64 +++++++++--------- src/cls/IPM/Test/YamlOutput.cls | 65 ++++++++++--------- tests/unit_tests/Test/PM/Unit/CLI.cls | 9 +++ .../TestResultsOPFormatAndFileGenTest.cls | 21 ++++-- 8 files changed, 169 insertions(+), 131 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 070aba8f4..640de0922 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -213,22 +213,17 @@ Method OnPhase( write ! if $data(pParams("outputformat"),outputFormat)||('tVerbose) { write !,"Test result summary",! + // CLI flag takes precedence; fall back to global config, then default to Toon set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() - if defaultOutputFormat'="" { - set outputFormat = defaultOutputFormat - } - else { - if $get(outputFormat)="" { - set outputFormat="Toon" - } + if $get(outputFormat)="" { + set outputFormat = $select(defaultOutputFormat'="":defaultOutputFormat,1:"Toon") } set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputType_" output format does not exist.")) + $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) } - set defaultTestStatus = "failed" - set sc = $classmethod(outputClass,"OutputToDevice",,defaultTestStatus) + set sc = $classmethod(outputClass,"OutputToDevice",,"failed") $$$ThrowOnError(sc) write ! } diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index c64cc3817..f75d9799b 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -3,21 +3,21 @@ Class %IPM.Test.Abstract Extends %RegisteredObject { ClassMethod ToFile( - FileName As %String, - CaseStatus As %String = "", - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + fileName As %String, + caseStatus As %String = "", + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] { } ClassMethod OutputToDevice( - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - TestStatus As %String = "") [ Abstract ] + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + testStatus As %String = "") As %Status [ Abstract ] { } Query FilteredTestResults( - Instance As %Integer, - TestStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") + instance As %Integer, + testStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") { SELECT count(*) as TotalCounts, @@ -31,17 +31,18 @@ SELECT tassert.Action AS assertAction, tassert.Counter AS assertCounter, tassert.Description AS assertDescription, - tassert.Location AS assertLocation + tassert.Location AS assertLocation, + tassert.Status AS assertStatus FROM %UnitTest_Result.TestInstance tinstance JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:Instance AND tassert.Status=:TestStatus +WHERE tinstance.ID=:instance AND tassert.Status=:testStatus } -Query GetAllTestResults(Instance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String", SELECTMODE = "DISPLAY") +Query GetAllTestResults(instance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") { SELECT count(*) as TotalCounts, @@ -55,14 +56,15 @@ SELECT tassert.Action AS assertAction, tassert.Counter AS assertCounter, tassert.Description AS assertDescription, - tassert.Location AS assertLocation + tassert.Location AS assertLocation, + tassert.Status AS assertStatus FROM %UnitTest_Result.TestInstance tinstance JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:Instance +WHERE tinstance.ID=:instance } } diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index bcdd4ba1b..68de980dc 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -1,39 +1,40 @@ -Class %IPM.Test.JUnitOutput +Class %IPM.Test.JUnitOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - pFileName As %String, - pTestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + caseStatus As %String = "", + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set tSC = $$$OK try { set tFile = ##class(%Stream.FileCharacter).%New() set tFile.TranslateTable="UTF8" - do tFile.LinkToFile(pFileName) + do tFile.LinkToFile(fileName) kill ^||TMP // results global set tSuite="" for { - set tSuite=$order(^UnitTest.Result(pTestIndex,tSuite),1,tSuiteData) + set tSuite=$order(^UnitTest.Result(testIndex,tSuite),1,tSuiteData) quit:tSuite="" set ^||TMP("S",tSuite,"time")=$listget(tSuiteData,2) set tCase="" for { - set tCase=$order(^UnitTest.Result(pTestIndex,tSuite,tCase),1,tCaseData) + set tCase=$order(^UnitTest.Result(testIndex,tSuite,tCase),1,tCaseData) quit:tCase="" do $increment(^||TMP("S",tSuite,"tests")) set ^||TMP("S",tSuite,"C",tCase,"time")=$listget(tCaseData,2) set tMethod="" for { - set tMethod=$order(^UnitTest.Result(pTestIndex,tSuite,tCase,tMethod),1,tMethodData) + set tMethod=$order(^UnitTest.Result(testIndex,tSuite,tCase,tMethod),1,tMethodData) quit:tMethod="" set ^||TMP("S",tSuite,"C",tCase,"M",tMethod,"time")=$listget(tMethodData,2) set tAssert="" for { - set tAssert=$order(^UnitTest.Result(pTestIndex,tSuite,tCase,tMethod,tAssert),1,tAssertData) + set tAssert=$order(^UnitTest.Result(testIndex,tSuite,tCase,tMethod,tAssert),1,tAssertData) quit:tAssert="" do $increment(^||TMP("S",tSuite,"assertions")) @@ -131,7 +132,29 @@ ClassMethod ToFile( } catch e { set tSC = e.AsStatus() } - quit $$$OK + quit tSC +} + +ClassMethod OutputToDevice( + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + testStatus As %String = "") As %Status +{ + set sc = $$$OK + try { + set result = ..GetAllTestResultsFunc(testIndex) + set currentID = "" + while result.%Next() { + if currentID = "" { + set currentID = testIndex + write !,"JUnit Test Run #"_testIndex_" ("_result.namespace_") "_result.duration_"s "_result.testDateTime + write ! + } + write !,result.suiteName_"/"_result.testcaseName_"/"_result.methodName_" "_result.assertAction_" ["_result.assertStatus_"]" + } + } catch ex { + set sc = ex.AsStatus() + } + return sc } } diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index dac91487d..08f3ff5c4 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -2,16 +2,16 @@ Class %IPM.Test.JsonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - FileName As %String, - TestStatus As %String = "", - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + caseStatus As %String = "", + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(FileName)) - set responseJson = ..JSON(TestIndex, TestStatus) + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set responseJson = ..JSON(testIndex, caseStatus) if $isobject(responseJson) { do fileStream.Write(responseJson.%ToJSON()) } @@ -23,12 +23,12 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - pCaseStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + caseStatus As %String = "") As %Status { set sc = $$$OK try { - set responseJson = ..JSON(TestIndex, pCaseStatus) + set responseJson = ..JSON(testIndex, caseStatus) write ! if $isobject(responseJson) { do responseJson.%ToJSON() @@ -40,28 +40,28 @@ ClassMethod OutputToDevice( } ClassMethod JSON( - TestIndex As %Integer, - TestStatus As %String) As %DynamicObject + testIndex As %Integer, + testStatus As %String) As %DynamicObject { - if TestStatus'="" { - set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) + if testStatus'="" { + set result = ..FilteredTestResultsFunc(testIndex, testStatus) } else { - set tResult = ..GetAllTestResultsFunc(TestIndex) + set result = ..GetAllTestResultsFunc(testIndex) } set unitTest = {} set unitTest.results = [] set (previousID,currentSuite,currentTestcase,suiteObj,testcaseObj) = "" - while tResult.%Next() { + while result.%Next() { if previousID = "" { - set unitTest.id = TestIndex - set unitTest.namespace = tResult.namespace - set unitTest.duration = tResult.duration - set unitTest.testDateTime = tResult.testDateTime + set unitTest.id = testIndex + set unitTest.namespace = result.namespace + set unitTest.duration = result.duration + set unitTest.testDateTime = result.testDateTime } - set previousID = TestIndex - if tResult.suiteName '= currentSuite { - set currentSuite = tResult.suiteName + set previousID = testIndex + if result.suiteName '= currentSuite { + set currentSuite = result.suiteName set suiteObj = { "suiteName": (currentSuite), "testcases": [] @@ -69,8 +69,8 @@ ClassMethod JSON( do unitTest.results.%Push(suiteObj) set currentTestcase = "" } - if tResult.testcaseName '= currentTestcase { - set currentTestcase = tResult.testcaseName + if result.testcaseName '= currentTestcase { + set currentTestcase = result.testcaseName set testcaseObj = { "testcaseName": (currentTestcase), "methods": [] @@ -78,12 +78,13 @@ ClassMethod JSON( do suiteObj.testcases.%Push(testcaseObj) } set methodObj = { - "methodName": (tResult.methodName), - "testMethod": (tResult.testMethod), - "assertAction": (tResult.assertAction), - "assertCounter": (tResult.assertCounter), - "assertDescription": (tResult.assertDescription), - "assertLocation": (tResult.assertLocation) + "methodName": (result.methodName), + "testMethod": (result.testMethod), + "assertAction": (result.assertAction), + "assertCounter": (result.assertCounter), + "assertDescription": (result.assertDescription), + "assertLocation": (result.assertLocation), + "assertStatus": (result.assertStatus) } do testcaseObj.methods.%Push(methodObj) } diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index f0826a21f..638b6c56f 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -2,38 +2,38 @@ Class %IPM.Test.ToonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - FileName As %String, - TestStatus As %String = "", - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + caseStatus As %String = "", + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(FileName)) + $$$ThrowOnError(fileStream.LinkToFile(fileName)) - if TestStatus'="" { - set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) + if caseStatus'="" { + set result = ..FilteredTestResultsFunc(testIndex, caseStatus) } else { - set tResult = ..GetAllTestResultsFunc(TestIndex) + set result = ..GetAllTestResultsFunc(testIndex) } set currentID="" - while tResult.%Next() { + while result.%Next() { if currentID = "" { - set currentID = TestIndex + set currentID = testIndex do fileStream.WriteLine("unitTest:") - do fileStream.WriteLine(" id: "_TestIndex) - do fileStream.WriteLine(" namespace: "_tResult.namespace) - do fileStream.WriteLine(" duration: "_tResult.duration) - do fileStream.WriteLine(" testDateTime: "_tResult.testDateTime) + do fileStream.WriteLine(" id: "_testIndex) + do fileStream.WriteLine(" namespace: "_result.namespace) + do fileStream.WriteLine(" duration: "_result.duration) + do fileStream.WriteLine(" testDateTime: "_result.testDateTime) do fileStream.WriteLine() - do fileStream.WriteLine("results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + do fileStream.WriteLine("results["_result.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") } - set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_TestIndex_","_ - tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" + set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ + result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" do fileStream.WriteLine(data) - } + } $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -42,32 +42,32 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - TestStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + testStatus As %String = "") As %Status { set sc = $$$OK try { - if TestStatus'="" { - set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) + if testStatus'="" { + set result = ..FilteredTestResultsFunc(testIndex, testStatus) } else { - set tResult = ..GetAllTestResultsFunc(TestIndex) + set result = ..GetAllTestResultsFunc(testIndex) } set currentID="" - while tResult.%Next() { + while result.%Next() { if currentID = "" { - set currentID = TestIndex + set currentID = testIndex write !,"unitTest:" - write !," id: "_TestIndex - write !," namespace: "_tResult.namespace - write !," duration: "_tResult.duration - write !," testDateTime: "_tResult.testDateTime + write !," id: "_testIndex + write !," namespace: "_result.namespace + write !," duration: "_result.duration + write !," testDateTime: "_result.testDateTime write ! - write !,"results["_tResult.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + write !,"results["_result.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" } - set data = " "_tResult.suiteName_","_tResult.testcaseName_","_tResult.methodName_","_TestIndex_","_ - tResult.assertAction_","_tResult.assertCounter_","""_$translate(tResult.assertDescription,"""")_""","""_tResult.assertLocation_"""" + set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ + result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" write !,data - } + } } catch ex { set sc = ex.AsStatus() } diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index bc6c1d229..3ea7c633b 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -2,16 +2,16 @@ Class %IPM.Test.YamlOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - FileName As %String, - TestStatus As %String = "", - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + caseStatus As %String = "", + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(FileName)) - do fileStream.CopyFrom(..YAML(TestIndex, TestStatus)) + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + do fileStream.CopyFrom(..YAML(testIndex, caseStatus)) $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -20,12 +20,12 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - TestIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - TestStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + testStatus As %String = "") As %Status { set sc = $$$OK try { - set yamlStream = ..YAML(TestIndex, TestStatus) + set yamlStream = ..YAML(testIndex, testStatus) write ! while 'yamlStream.AtEnd { write yamlStream.Read() @@ -37,45 +37,46 @@ ClassMethod OutputToDevice( } ClassMethod YAML( - TestIndex = {$order(^UnitTest.Result(""),-1)}, - TestStatus As %String = "") As %Stream.TmpCharacter + testIndex = {$order(^UnitTest.Result(""),-1)}, + testStatus As %String = "") As %Stream.TmpCharacter { - if TestStatus'="" { - set tResult = ..FilteredTestResultsFunc(TestIndex, TestStatus) + if testStatus'="" { + set result = ..FilteredTestResultsFunc(testIndex, testStatus) } else { - set tResult = ..GetAllTestResultsFunc(TestIndex) + set result = ..GetAllTestResultsFunc(testIndex) } set yamlStream = ##class(%Stream.TmpCharacter).%New() - set (yaml,currentID,currentSuite,currentTestcase) = "" - while tResult.%Next() { + set (currentID,currentSuite,currentTestcase) = "" + while result.%Next() { if currentID = "" { - set currentID = TestIndex + set currentID = testIndex do yamlStream.WriteLine("unitTest:") - do yamlStream.WriteLine(" id: "_TestIndex) - do yamlStream.WriteLine(" namespace: """_tResult.namespace_"""") - do yamlStream.WriteLine(" duration: "_tResult.duration) - do yamlStream.WriteLine(" testDateTime: """_tResult.testDateTime_"""") + do yamlStream.WriteLine(" id: "_testIndex) + do yamlStream.WriteLine(" namespace: """_result.namespace_"""") + do yamlStream.WriteLine(" duration: "_result.duration) + do yamlStream.WriteLine(" testDateTime: """_result.testDateTime_"""") do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") } - if tResult.suiteName '= currentSuite { - set currentSuite = tResult.suiteName + if result.suiteName '= currentSuite { + set currentSuite = result.suiteName set currentTestcase = "" - do yamlStream.WriteLine(" - suiteName: """_tResult.suiteName_"""") + do yamlStream.WriteLine(" - suiteName: """_result.suiteName_"""") do yamlStream.WriteLine(" testcases:") } - if tResult.testcaseName '= currentTestcase { - set currentTestcase = tResult.testcaseName - do yamlStream.WriteLine(" - testcaseName: """_tResult.testcaseName_"""") + if result.testcaseName '= currentTestcase { + set currentTestcase = result.testcaseName + do yamlStream.WriteLine(" - testcaseName: """_result.testcaseName_"""") do yamlStream.WriteLine(" methods:") } - do yamlStream.WriteLine(" - methodName: """_tResult.methodName_"""") - do yamlStream.WriteLine(" testMethod: """_tResult.testMethod_"""") - do yamlStream.WriteLine(" assertAction: """_tResult.assertAction_"""") - do yamlStream.WriteLine(" assertCounter: "_tResult.assertCounter) + do yamlStream.WriteLine(" - methodName: """_result.methodName_"""") + do yamlStream.WriteLine(" testMethod: """_result.testMethod_"""") + do yamlStream.WriteLine(" assertAction: """_result.assertAction_"""") + do yamlStream.WriteLine(" assertCounter: "_result.assertCounter) + do yamlStream.WriteLine(" assertStatus: """_result.assertStatus_"""") do yamlStream.WriteLine(" assertDescription: |") - do yamlStream.WriteLine(" "_$replace(tResult.assertDescription, $char(10), $char(10)_" ")) - do yamlStream.WriteLine(" assertLocation: """_tResult.assertLocation_"""") + do yamlStream.WriteLine(" "_$replace(result.assertDescription, $char(10), $char(10)_" ")) + do yamlStream.WriteLine(" assertLocation: """_result.assertLocation_"""") } return yamlStream } diff --git a/tests/unit_tests/Test/PM/Unit/CLI.cls b/tests/unit_tests/Test/PM/Unit/CLI.cls index 797ce4541..993db5625 100644 --- a/tests/unit_tests/Test/PM/Unit/CLI.cls +++ b/tests/unit_tests/Test/PM/Unit/CLI.cls @@ -355,6 +355,8 @@ Method TestUninstallWithoutModuleName() /// Specifies the serialization format (json, toon, yaml) for unit and integration test results in the shell. Method TestReportFormatConfiguration() { + set originalFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + do ..RunCommand("config set TestReportFormat json") set format = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() do $$$AssertEquals(format, "json", "Verify TestReportFormat is set to JSON") @@ -362,6 +364,13 @@ Method TestReportFormatConfiguration() do ..RunCommand("config set TestReportFormat yaml") set format = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() do $$$AssertEquals(format, "yaml", "Verify TestReportFormat is set to YAML") + + // Restore original value so this test doesn't pollute subsequent tests + if originalFormat'="" { + do ..RunCommand("config set TestReportFormat "_originalFormat) + } else { + do ..RunCommand("config reset TestReportFormat") + } } } diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index 354eaa0d3..519e152c2 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -7,11 +7,14 @@ Class Test.PM.Unit.TestResultsOPFormatAndFileGenTest Extends %UnitTest.TestCase { +Method TestFail() +{ + do $$$AssertFailure("This test is designed to fail to validate the test framework's ability to capture failures.") +} + /// generate .yaml,.json,.toon files Method TestResultFileGeneration() { - #define NormalizeFilename(%file) ##class(%File).NormalizeFilename(%file) - do $$$LogMessage("This file generation picks the last or current unit test id and generate the reports") set fileDir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") @@ -21,25 +24,29 @@ Method TestResultFileGeneration() } do $$$LogMessage("Start generating the reports") - set fileName = $$$NormalizeFilename(fileDir_"/test.yaml") + + set fileName = ##class(%File).NormalizeFilename(fileDir_"test.yaml") set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName) do $$$AssertStatusOK(status,"yaml file generated successfully in "_fileDir) + do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"yaml file is non-empty") - set fileName = $$$NormalizeFilename(fileDir_"/test.Json") + set fileName = ##class(%File).NormalizeFilename(fileDir_"test.json") set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName) do $$$AssertStatusOK(status,"Json file generated successfully in "_fileDir) + do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"json file is non-empty") - set fileName = $$$NormalizeFilename(fileDir_"/test.toon") + set fileName = ##class(%File).NormalizeFilename(fileDir_"test.toon") set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName) do $$$AssertStatusOK(status,"Toon file generated successfully in "_fileDir) + do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"toon file is non-empty") do ..ShowGeneratedFilesAndCleaup(fileDir) } -Method ShowGeneratedFilesAndCleaup(FileDir As %String) +Method ShowGeneratedFilesAndCleaup(fileDir As %String) { do $$$LogMessage("Display the generated unit test report files") - set fileSet = ##class(%File).FileSetFunc(FileDir) + set fileSet = ##class(%File).FileSetFunc(fileDir) while fileSet.%Next() { set file = fileSet.Name set fileNames(file) = fileSet.ItemName From ce3dc1798c451a61f7d159fa18631f2f057d5f3a Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Apr 2026 14:29:59 -0400 Subject: [PATCH 06/18] Refactor pass two --- CHANGELOG.md | 1 + src/cls/IPM/Main.cls | 3 +- src/cls/IPM/ResourceProcessor/Test.cls | 42 ++-- src/cls/IPM/Test/Abstract.cls | 146 ++++++++---- src/cls/IPM/Test/JUnitOutput.cls | 207 +++++++++--------- src/cls/IPM/Test/JsonOutput.cls | 14 +- src/cls/IPM/Test/ToonOutput.cls | 69 +++--- src/cls/IPM/Test/YamlOutput.cls | 52 +++-- .../_data/test-output-format/module.xml | 13 ++ .../Test/Output/Format/AfterAllReturn.cls | 15 ++ .../Test/Output/Format/AfterAllThrow.cls | 15 ++ .../Test/Output/Format/AfterOneReturn.cls | 15 ++ .../Test/Output/Format/AfterOneThrow.cls | 15 ++ .../Test/Output/Format/BeforeAllReturn.cls | 15 ++ .../Test/Output/Format/BeforeAllThrow.cls | 15 ++ .../Test/Output/Format/BeforeOneReturn.cls | 20 ++ .../Test/Output/Format/BeforeOneThrow.cls | 20 ++ .../Output/Format/MethodAssertFailure.cls | 23 ++ .../Test/Output/Format/MethodThrowFailure.cls | 21 ++ .../TestResultsOPFormatAndFileGenTest.cls | 162 ++++++++++---- 20 files changed, 615 insertions(+), 268 deletions(-) create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllReturn.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllThrow.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneReturn.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneThrow.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllReturn.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllThrow.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneReturn.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneThrow.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodAssertFailure.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodThrowFailure.cls diff --git a/CHANGELOG.md b/CHANGELOG.md index 55f66de4d..e578ec59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #992: Implement automatic history purge logic - #973: Enables CORS and JWT configuration for WebApplications in module.xml +- #971: Adds support for JSON, YAML, and Toon formats via the -output-format/-f flag for outputting into the terminal and the -output-file flag for outputting to a file ### Fixed - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 2c7cc4154..fc75f520f 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -75,7 +75,8 @@ Can also specify desired version to update to. - + + diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 640de0922..520600c32 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -189,30 +189,28 @@ Method OnPhase( zkill ^UnitTestRoot $$$ThrowOnError(tSC) - if $data(pParams("UnitTest"))>1 { - set outputType="" - for { - set outputType = $order(pParams("UnitTest",outputType),1,fileName) - quit:outputType="" - set tPostfix = "-"_$$$lcase(pPhase)_"-" - if (..Package '= "") { - set tPostfix = tPostfix_$replace(..Package,".","-")_"-PKG" - } elseif (..Class '= "") { - set tPostfix = tPostfix_$replace(..Class,".","-")_"-CLS" - } - set outputClass = "%IPM.Test."_outputType - if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputType_" output format does not exist.")) - } - set extension = $select(outputType="JsonOutput":".json",outputType="ToonOutput":".toon",outputType="YamlOutput":".yaml",1:".xml") - set fileName = $piece(fileName,".",1,*-1)_tPostfix_extension - set tSC = $classmethod(outputClass,"ToFile",fileName) - $$$ThrowOnError(tSC) + set testIndex = $order(^UnitTest.Result(""),-1) + if $data(pParams("outputfile"), outputFile) { + set fileExtension = $zconvert($piece(outputFile,".",*),"L") + set outputClass = $case(fileExtension, + "json":"%IPM.Test.JsonOutput", + "yaml":"%IPM.Test.YamlOutput", + "toon":"%IPM.Test.ToonOutput", + "xml":"%IPM.Test.JUnitOutput", + :"") + if outputClass = "" { + $$$ThrowOnError($$$ERROR($$$GeneralError,"Unsupported output-file extension '."_fileExtension_"'. Use .json, .yaml, .toon, or .xml.")) + } + set outputDir = ##class(%File).GetDirectory(outputFile) + if outputDir '= "" { + $$$ThrowOnError(##class(%File).CreateDirectoryChain(outputDir)) } + set tSC = $classmethod(outputClass,"ToFile",outputFile) + $$$ThrowOnError(tSC) } write ! if $data(pParams("outputformat"),outputFormat)||('tVerbose) { - write !,"Test result summary",! + write !,"Test Result Summary",! // CLI flag takes precedence; fall back to global config, then default to Toon set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() if $get(outputFormat)="" { @@ -223,8 +221,8 @@ Method OnPhase( if '$$$defClassDefined(outputClass) { $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) } - set sc = $classmethod(outputClass,"OutputToDevice",,"failed") - $$$ThrowOnError(sc) + set tSC = $classmethod(outputClass,"OutputToDevice",testIndex) + $$$ThrowOnError(tSC) write ! } // By default, detect and report unit test failures as an error from this phase diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index f75d9799b..6ffed5522 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -10,61 +10,123 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "") As %Status [ Abstract ] + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] +{ +} + +/// Returns a summary %DynamicObject with run metadata and method/assertion counts. +/// A method is counted as "failed" if any of its assertions failed. +ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %DynamicObject { + set result = ..GetAllTestResultsFunc(testIndex) + set (ns, duration, testDateTime) = "" + set (assertTotal, assertPassed, assertFailed) = 0 + + while result.%Next() { + if ns = "" { + set ns = result.namespace + set duration = result.duration + set testDateTime = result.testDateTime + } + if result.assertStatus = "passed" { + set assertTotal = assertTotal + 1 + set assertPassed = assertPassed + 1 + } elseif result.assertStatus = "failed" { + set assertTotal = assertTotal + 1 + set assertFailed = assertFailed + 1 + } + // Track method status — once "failed", stays "failed" even if later assertions pass + set methodKey = result.suiteName_"||"_result.testcaseName_"||"_result.methodName + if '$data(seenMethods(methodKey)) { + set seenMethods(methodKey) = "" + } + if result.assertStatus = "failed" { + set seenMethods(methodKey) = "failed" + } elseif (result.assertStatus = "passed") && (seenMethods(methodKey) '= "failed") { + set seenMethods(methodKey) = "passed" + } + } + + set (methodTotal, methodPassed, methodFailed) = 0 + set key = "" + for { + set key = $order(seenMethods(key), 1, status) + quit:key="" + if status = "passed" { + set methodTotal = methodTotal + 1 + set methodPassed = methodPassed + 1 + } elseif status = "failed" { + set methodTotal = methodTotal + 1 + set methodFailed = methodFailed + 1 + } + } + + return { + "id": (testIndex), + "namespace": (ns), + "duration": (duration), + "testDateTime": (testDateTime), + "methods": { + "total": (methodTotal), + "passed": (methodPassed), + "failed": (methodFailed) + }, + "assertions": { + "total": (assertTotal), + "passed": (assertPassed), + "failed": (assertFailed) + } + } } Query FilteredTestResults( instance As %Integer, - testStatus) As %SQLQuery(ROWSPEC = "TotalCounts:%Integer,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") + testStatus) As %SQLQuery(ROWSPEC = "namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") { SELECT - count(*) as TotalCounts, - tinstance.Namespace AS namespace, - tinstance.Duration AS duration, - tinstance.DateTime AS testDateTime, - tsuite.Name AS suiteName, - tcase.Name AS testcaseName, - tmethod.Name AS methodName, - tassert.TestMethod AS testMethod, - tassert.Action AS assertAction, - tassert.Counter AS assertCounter, - tassert.Description AS assertDescription, - tassert.Location AS assertLocation, - tassert.Status AS assertStatus + inst.Namespace AS namespace, + inst.Duration AS duration, + inst.DateTime AS testDateTime, + suite.Name AS suiteName, + testCase.Name AS testcaseName, + method.Name AS methodName, + testAssert.TestMethod AS testMethod, + testAssert.Action AS assertAction, + testAssert.Counter AS assertCounter, + testAssert.Description AS assertDescription, + testAssert.Location AS assertLocation, + testAssert.Status AS assertStatus FROM -%UnitTest_Result.TestInstance tinstance -JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID -JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID -JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID -JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:instance AND tassert.Status=:testStatus +%UnitTest_Result.TestInstance inst +JOIN %UnitTest_Result.TestSuite suite ON suite.TestInstance=inst.ID +JOIN %UnitTest_Result.TestCase testCase ON testCase.TestSuite=suite.ID +JOIN %UnitTest_Result.TestMethod method ON method.TestCase=testCase.ID +JOIN %UnitTest_Result.TestAssert testAssert ON testAssert.TestMethod=method.ID +WHERE inst.ID=:instance AND testAssert.Status=:testStatus } -Query GetAllTestResults(instance As %Integer) As %SQLQuery(ROWSPEC = "TotalCounts:%String,namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") +Query GetAllTestResults(instance As %Integer) As %SQLQuery(ROWSPEC = "namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") { SELECT - count(*) as TotalCounts, - tinstance.Namespace AS namespace, - tinstance.Duration AS duration, - tinstance.DateTime AS testDateTime, - tsuite.Name AS suiteName, - tcase.Name AS testcaseName, - tmethod.Name AS methodName, - tassert.TestMethod AS testMethod, - tassert.Action AS assertAction, - tassert.Counter AS assertCounter, - tassert.Description AS assertDescription, - tassert.Location AS assertLocation, - tassert.Status AS assertStatus + inst.Namespace AS namespace, + inst.Duration AS duration, + inst.DateTime AS testDateTime, + suite.Name AS suiteName, + testCase.Name AS testcaseName, + method.Name AS methodName, + testAssert.TestMethod AS testMethod, + testAssert.Action AS assertAction, + testAssert.Counter AS assertCounter, + testAssert.Description AS assertDescription, + testAssert.Location AS assertLocation, + testAssert.Status AS assertStatus FROM -%UnitTest_Result.TestInstance tinstance -JOIN %UnitTest_Result.TestSuite tsuite ON tsuite.TestInstance=tinstance.ID -JOIN %UnitTest_Result.TestCase tcase ON tcase.TestSuite=tsuite.ID -JOIN %UnitTest_Result.TestMethod tmethod ON tmethod.TestCase=tcase.ID -JOIN %UnitTest_Result.TestAssert tassert ON tassert.TestMethod=tmethod.ID -WHERE tinstance.ID=:instance +%UnitTest_Result.TestInstance inst +JOIN %UnitTest_Result.TestSuite suite ON suite.TestInstance=inst.ID +JOIN %UnitTest_Result.TestCase testCase ON testCase.TestSuite=suite.ID +JOIN %UnitTest_Result.TestMethod method ON method.TestCase=testCase.ID +JOIN %UnitTest_Result.TestAssert testAssert ON testAssert.TestMethod=method.ID +WHERE inst.ID=:instance } } diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index 68de980dc..1a03bbb23 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -6,150 +6,149 @@ ClassMethod ToFile( caseStatus As %String = "", testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { - set tSC = $$$OK + set sc = $$$OK try { - set tFile = ##class(%Stream.FileCharacter).%New() - set tFile.TranslateTable="UTF8" - do tFile.LinkToFile(fileName) + set fileStream = ##class(%Stream.FileCharacter).%New() + set fileStream.TranslateTable = "UTF8" + $$$ThrowOnError(fileStream.LinkToFile(fileName)) - kill ^||TMP // results global - set tSuite="" + kill ^||TMP + set suite = "" for { - set tSuite=$order(^UnitTest.Result(testIndex,tSuite),1,tSuiteData) - quit:tSuite="" - set ^||TMP("S",tSuite,"time")=$listget(tSuiteData,2) + set suite = $order(^UnitTest.Result(testIndex, suite), 1, suiteData) + quit:suite="" + set ^||TMP("S", suite, "time") = $listget(suiteData, 2) - set tCase="" + set testCase = "" for { - set tCase=$order(^UnitTest.Result(testIndex,tSuite,tCase),1,tCaseData) - quit:tCase="" + set testCase = $order(^UnitTest.Result(testIndex, suite, testCase), 1, testCaseData) + quit:testCase="" - do $increment(^||TMP("S",tSuite,"tests")) - set ^||TMP("S",tSuite,"C",tCase,"time")=$listget(tCaseData,2) - set tMethod="" + do $increment(^||TMP("S", suite, "tests")) + set ^||TMP("S", suite, "C", testCase, "time") = $listget(testCaseData, 2) + set method = "" for { - set tMethod=$order(^UnitTest.Result(testIndex,tSuite,tCase,tMethod),1,tMethodData) - quit:tMethod="" + set method = $order(^UnitTest.Result(testIndex, suite, testCase, method), 1, methodData) + quit:method="" - set ^||TMP("S",tSuite,"C",tCase,"M",tMethod,"time")=$listget(tMethodData,2) - set tAssert="" + set ^||TMP("S", suite, "C", testCase, "M", method, "time") = $listget(methodData, 2) + set assert = "" for { - set tAssert=$order(^UnitTest.Result(testIndex,tSuite,tCase,tMethod,tAssert),1,tAssertData) - quit:tAssert="" - - do $increment(^||TMP("S",tSuite,"assertions")) - do $increment(^||TMP("S",tSuite,"C",tCase,"assertions")) - do $increment(^||TMP("S",tSuite,"C",tCase,"M",tMethod,"assertions")) - if $listget(tAssertData)=0 { - do $increment(^||TMP("S",tSuite,"failures")) - do $increment(^||TMP("S",tSuite,"C",tCase,"failures")) - set tIndex = $increment(^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures")) - set ^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures",tIndex) = - $listget(tAssertData,2) _ ": " _ $listget(tAssertData,3) + set assert = $order(^UnitTest.Result(testIndex, suite, testCase, method, assert), 1, assertData) + quit:assert="" + + do $increment(^||TMP("S", suite, "assertions")) + do $increment(^||TMP("S", suite, "C", testCase, "assertions")) + do $increment(^||TMP("S", suite, "C", testCase, "M", method, "assertions")) + if $listget(assertData) = 0 { + do $increment(^||TMP("S", suite, "failures")) + do $increment(^||TMP("S", suite, "C", testCase, "failures")) + set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", method, "failures")) + set ^||TMP("S", suite, "C", testCase, "M", method, "failures", failureIndex) = + $listget(assertData, 2) _ ": " _ $listget(assertData, 3) } } - if ($listget(tMethodData)=0) - && ('$data(^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures"))) { - do $increment(^||TMP("S",tSuite,"failures")) - do $increment(^||TMP("S",tSuite,"C",tCase,"failures")) - set tIndex = $increment(^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures")) - set ^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures",tIndex) = - $listget(tMethodData,3) _ ": " _ $listget(tMethodData,4) + if ($listget(methodData) = 0) + && ('$data(^||TMP("S", suite, "C", testCase, "M", method, "failures"))) { + do $increment(^||TMP("S", suite, "failures")) + do $increment(^||TMP("S", suite, "C", testCase, "failures")) + set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", method, "failures")) + set ^||TMP("S", suite, "C", testCase, "M", method, "failures", failureIndex) = + $listget(methodData, 3) _ ": " _ $listget(methodData, 4) } } - if $listget(tCaseData)=0 - && ('$data(^||TMP("S",tSuite,"C",tCase,"failures"))) { - do $increment(^||TMP("S",tSuite,"failures")) - do $increment(^||TMP("S",tSuite,"C",tCase,"failures")) - set tIndex = $increment(^||TMP("S",tSuite,"C",tCase,"M",tCase,"failures")) - set ^||TMP("S",tSuite,"C",tCase,"M",tCase,"failures",tIndex) = - $listget(tCaseData,3) _ ": " _ $listget(tCaseData,4) + if $listget(testCaseData) = 0 + && ('$data(^||TMP("S", suite, "C", testCase, "failures"))) { + do $increment(^||TMP("S", suite, "failures")) + do $increment(^||TMP("S", suite, "C", testCase, "failures")) + set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", testCase, "failures")) + set ^||TMP("S", suite, "C", testCase, "M", testCase, "failures", failureIndex) = + $listget(testCaseData, 3) _ ": " _ $listget(testCaseData, 4) } } } - do tFile.WriteLine("") - do tFile.WriteLine("") - set tSuite="" + do fileStream.WriteLine("") + do fileStream.WriteLine("") + set suite = "" for { - set tSuite=$order(^||TMP("S",tSuite)) - quit:tSuite="" + set suite = $order(^||TMP("S", suite)) + quit:suite="" - do tFile.Write("") + do fileStream.Write("") - set tCase="" + set testCase = "" for { - set tCase=$order(^||TMP("S",tSuite,"C",tCase)) - quit:tCase="" + set testCase = $order(^||TMP("S", suite, "C", testCase)) + quit:testCase="" - do tFile.Write("") + do fileStream.Write("") - set tMethod="" + set method = "" for { - set tMethod=$order(^||TMP("S",tSuite,"C",tCase,"M",tMethod)) - quit:tMethod="" - - do tFile.Write("") - set tFailureKey = "" + set method = $order(^||TMP("S", suite, "C", testCase, "M", method)) + quit:method="" + + do fileStream.Write("") + set failureKey = "" for { - set tFailureKey = $order(^||TMP("S",tSuite,"C",tCase,"M",tMethod,"failures",tFailureKey),1,tMessage) - if (tFailureKey = "") { - quit - } - set tMessage = $zstrip(tMessage,"*C") - set tMessage = $zconvert($zconvert(tMessage,"O","UTF8"),"O","XML") - // Also encode newlines - $zconvert doesn't do this. - set tMessage = $replace(tMessage,$char(10)," ") - set tMessage = $replace(tMessage,$char(13)," ") - do tFile.Write("") - do tFile.WriteLine("") + set failureKey = $order(^||TMP("S",suite,"C",testCase,"M",method,"failures",failureKey),1,message) + quit:failureKey="" + set message = $zstrip(message,"*C") + set message = $zconvert($zconvert(message,"O","UTF8"),"O","XML") + // $zconvert does not encode newlines. + set message = $replace(message,$char(10)," ") + set message = $replace(message,$char(13)," ") + do fileStream.Write("") + do fileStream.WriteLine("") } - do tFile.WriteLine("") + do fileStream.WriteLine("") } - do tFile.WriteLine("") + do fileStream.WriteLine("") } - do tFile.WriteLine("") + do fileStream.WriteLine("") } - do tFile.WriteLine("") + do fileStream.WriteLine("") kill ^||TMP - $$$ThrowOnError(tFile.%Save()) - } catch e { - set tSC = e.AsStatus() + $$$ThrowOnError(fileStream.%Save()) + } catch ex { + set sc = ex.AsStatus() } - quit tSC + return sc } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { - set result = ..GetAllTestResultsFunc(testIndex) - set currentID = "" - while result.%Next() { - if currentID = "" { - set currentID = testIndex - write !,"JUnit Test Run #"_testIndex_" ("_result.namespace_") "_result.duration_"s "_result.testDateTime - write ! + set summary = ..GetSummary(testIndex) + write !,"JUnit Test Run #"_summary.id_" ("_summary.namespace_") "_summary.duration_"s "_summary.testDateTime + write !,"Methods: "_summary.methods.total_" total, "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" + write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" + + if summary.methods.failed > 0 { + set result = ..GetAllTestResultsFunc(testIndex) + write ! + while result.%Next() { + write !,result.suiteName_"/"_result.testcaseName_"/"_result.methodName_" "_result.assertAction_" ["_result.assertStatus_"]" } - write !,result.suiteName_"/"_result.testcaseName_"/"_result.methodName_" "_result.assertAction_" ["_result.assertStatus_"]" } } catch ex { set sc = ex.AsStatus() diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 08f3ff5c4..9b973b6ec 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -23,16 +23,18 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - caseStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { - set responseJson = ..JSON(testIndex, caseStatus) - write ! - if $isobject(responseJson) { - do responseJson.%ToJSON() + set summary = ..GetSummary(testIndex) + set failures = ..JSON(testIndex, "failed") + set output = { + "summary": (summary), + "failures": (failures.results) } + write ! + do output.%ToJSON() } catch ex { set sc = ex.AsStatus() } diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index 638b6c56f..599affa4f 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -12,28 +12,35 @@ ClassMethod ToFile( set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - if caseStatus'="" { + if caseStatus '= "" { set result = ..FilteredTestResultsFunc(testIndex, caseStatus) } else { set result = ..GetAllTestResultsFunc(testIndex) } - set currentID="" + set rowCount = 0 + set (ns, duration, testDateTime) = "" + set rowStream = ##class(%Stream.TmpCharacter).%New() while result.%Next() { - if currentID = "" { - set currentID = testIndex - do fileStream.WriteLine("unitTest:") - do fileStream.WriteLine(" id: "_testIndex) - do fileStream.WriteLine(" namespace: "_result.namespace) - do fileStream.WriteLine(" duration: "_result.duration) - do fileStream.WriteLine(" testDateTime: "_result.testDateTime) - do fileStream.WriteLine() - do fileStream.WriteLine("results["_result.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + if rowCount = 0 { + set ns = result.namespace + set duration = result.duration + set testDateTime = result.testDateTime } + set rowCount = rowCount + 1 set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" - do fileStream.WriteLine(data) + do rowStream.WriteLine(data) } + + do fileStream.WriteLine("unitTest:") + do fileStream.WriteLine(" id: "_testIndex) + do fileStream.WriteLine(" namespace: "_ns) + do fileStream.WriteLine(" duration: "_duration) + do fileStream.WriteLine(" testDateTime: "_testDateTime) + do fileStream.WriteLine() + do fileStream.WriteLine("results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + do fileStream.CopyFrom(rowStream) $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -42,31 +49,27 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { - if testStatus'="" { - set result = ..FilteredTestResultsFunc(testIndex, testStatus) - } else { - set result = ..GetAllTestResultsFunc(testIndex) - } - set currentID="" - while result.%Next() { - if currentID = "" { - set currentID = testIndex - write !,"unitTest:" - write !," id: "_testIndex - write !," namespace: "_result.namespace - write !," duration: "_result.duration - write !," testDateTime: "_result.testDateTime - write ! - write !,"results["_result.TotalCounts_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + set summary = ..GetSummary(testIndex) + write ! + write !,"summary:" + write !," id: "_summary.id_" namespace: "_summary.namespace_" duration: "_summary.duration_" testDateTime: "_summary.testDateTime + write !," methods["_summary.methods.total_"]: "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" + write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" + + if summary.methods.failed > 0 { + set result = ..FilteredTestResultsFunc(testIndex, "failed") + set currentID = "" + write ! + write !,"failures["_summary.assertions.failed_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + while result.%Next() { + set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ + result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" + write !,data } - set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ - result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" - write !,data } } catch ex { set sc = ex.AsStatus() diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 3ea7c633b..40e373413 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -20,15 +20,34 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "") As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { - set yamlStream = ..YAML(testIndex, testStatus) + set summary = ..GetSummary(testIndex) write ! - while 'yamlStream.AtEnd { - write yamlStream.Read() + write !,"summary:" + write !," id: "_summary.id + write !," namespace: """_summary.namespace_"""" + write !," duration: "_summary.duration + write !," testDateTime: """_summary.testDateTime_"""" + write !," methods:" + write !," total: "_summary.methods.total + write !," passed: "_summary.methods.passed + write !," failed: "_summary.methods.failed + write !," assertions:" + write !," total: "_summary.assertions.total + write !," passed: "_summary.assertions.passed + write !," failed: "_summary.assertions.failed + + if summary.methods.failed > 0 { + write ! + write !,"failures:" + set failureStream = ..YAML(testIndex, "failed", 0) + do failureStream.Rewind() + while 'failureStream.AtEnd { + write !,failureStream.ReadLine() + } } } catch ex { set sc = ex.AsStatus() @@ -38,25 +57,28 @@ ClassMethod OutputToDevice( ClassMethod YAML( testIndex = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "") As %Stream.TmpCharacter + testStatus As %String = "", + includeHeader As %Boolean = 1) As %Stream.TmpCharacter { - if testStatus'="" { + if testStatus '= "" { set result = ..FilteredTestResultsFunc(testIndex, testStatus) } else { set result = ..GetAllTestResultsFunc(testIndex) } set yamlStream = ##class(%Stream.TmpCharacter).%New() - set (currentID,currentSuite,currentTestcase) = "" + set (currentID, currentSuite, currentTestcase) = "" while result.%Next() { if currentID = "" { set currentID = testIndex - do yamlStream.WriteLine("unitTest:") - do yamlStream.WriteLine(" id: "_testIndex) - do yamlStream.WriteLine(" namespace: """_result.namespace_"""") - do yamlStream.WriteLine(" duration: "_result.duration) - do yamlStream.WriteLine(" testDateTime: """_result.testDateTime_"""") - do yamlStream.WriteLine() - do yamlStream.WriteLine(" results:") + if includeHeader { + do yamlStream.WriteLine("unitTest:") + do yamlStream.WriteLine(" id: "_testIndex) + do yamlStream.WriteLine(" namespace: """_result.namespace_"""") + do yamlStream.WriteLine(" duration: "_result.duration) + do yamlStream.WriteLine(" testDateTime: """_result.testDateTime_"""") + do yamlStream.WriteLine() + do yamlStream.WriteLine(" results:") + } } if result.suiteName '= currentSuite { set currentSuite = result.suiteName diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml new file mode 100644 index 000000000..a1a9eb872 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml @@ -0,0 +1,13 @@ + + + + + test-output-format + 0.0.1 + module + src + + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllReturn.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllReturn.cls new file mode 100644 index 000000000..ef1a33ead --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllReturn.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnAfterAllTests returning an error status; all test methods run first. +Class Test.Output.Format.AfterAllReturn Extends %UnitTest.TestCase +{ + +Method OnAfterAllTests() As %Status +{ + return $$$ERROR($$$GeneralError, "deliberate OnAfterAllTests return-error") +} + +Method TestPassesNormally() +{ + do $$$AssertTrue(1, "this runs and passes; failure occurs only in OnAfterAllTests") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllThrow.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllThrow.cls new file mode 100644 index 000000000..5aa65734a --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterAllThrow.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnAfterAllTests throwing; all test methods run first. +Class Test.Output.Format.AfterAllThrow Extends %UnitTest.TestCase +{ + +Method OnAfterAllTests() As %Status +{ + $$$ThrowStatus($$$ERROR($$$GeneralError, "deliberate OnAfterAllTests throw")) +} + +Method TestPassesNormally() +{ + do $$$AssertTrue(1, "this runs and passes; failure occurs only in OnAfterAllTests") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneReturn.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneReturn.cls new file mode 100644 index 000000000..00322e7e2 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneReturn.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnAfterOneTest returning an error status; assertions are recorded before teardown fails. +Class Test.Output.Format.AfterOneReturn Extends %UnitTest.TestCase +{ + +Method OnAfterOneTest(testName As %String) As %Status +{ + return $$$ERROR($$$GeneralError, "deliberate OnAfterOneTest return-error for "_testName) +} + +Method TestMethodBodyPasses() +{ + do $$$AssertTrue(1, "the method body runs and this assertion is recorded before teardown fails") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneThrow.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneThrow.cls new file mode 100644 index 000000000..2cd9a4bdc --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/AfterOneThrow.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnAfterOneTest throwing; assertions are recorded before teardown throws. +Class Test.Output.Format.AfterOneThrow Extends %UnitTest.TestCase +{ + +Method OnAfterOneTest(testName As %String) As %Status +{ + $$$ThrowStatus($$$ERROR($$$GeneralError, "deliberate OnAfterOneTest throw for "_testName)) +} + +Method TestMethodBodyPasses() +{ + do $$$AssertTrue(1, "the method body runs and this assertion is recorded before teardown throws") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllReturn.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllReturn.cls new file mode 100644 index 000000000..07c4aebfe --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllReturn.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnBeforeAllTests returning an error status; no test methods run. +Class Test.Output.Format.BeforeAllReturn Extends %UnitTest.TestCase +{ + +Method OnBeforeAllTests() As %Status +{ + return $$$ERROR($$$GeneralError, "deliberate OnBeforeAllTests return-error") +} + +Method TestWouldPass() +{ + do $$$AssertTrue(1, "this would pass if OnBeforeAllTests had not failed") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllThrow.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllThrow.cls new file mode 100644 index 000000000..d6bb2d300 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeAllThrow.cls @@ -0,0 +1,15 @@ +/// Demonstrates OnBeforeAllTests throwing; no test methods run. +Class Test.Output.Format.BeforeAllThrow Extends %UnitTest.TestCase +{ + +Method OnBeforeAllTests() As %Status +{ + $$$ThrowStatus($$$ERROR($$$GeneralError, "deliberate OnBeforeAllTests throw")) +} + +Method TestWouldPass() +{ + do $$$AssertTrue(1, "this would pass if OnBeforeAllTests had not thrown") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneReturn.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneReturn.cls new file mode 100644 index 000000000..ea19c2313 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneReturn.cls @@ -0,0 +1,20 @@ +/// Demonstrates OnBeforeOneTest returning an error status; each method is individually aborted. +Class Test.Output.Format.BeforeOneReturn Extends %UnitTest.TestCase +{ + +Method OnBeforeOneTest(testName As %String) As %Status +{ + return $$$ERROR($$$GeneralError, "deliberate OnBeforeOneTest return-error for "_testName) +} + +Method TestFirstMethod() +{ + do $$$AssertTrue(1, "this would pass if OnBeforeOneTest had not failed") +} + +Method TestSecondMethod() +{ + do $$$AssertTrue(1, "this would also pass if OnBeforeOneTest had not failed") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneThrow.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneThrow.cls new file mode 100644 index 000000000..a6cabe7f2 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/BeforeOneThrow.cls @@ -0,0 +1,20 @@ +/// Demonstrates OnBeforeOneTest throwing; each method is individually aborted. +Class Test.Output.Format.BeforeOneThrow Extends %UnitTest.TestCase +{ + +Method OnBeforeOneTest(testName As %String) As %Status +{ + $$$ThrowStatus($$$ERROR($$$GeneralError, "deliberate OnBeforeOneTest throw for "_testName)) +} + +Method TestFirstMethod() +{ + do $$$AssertTrue(1, "this would pass if OnBeforeOneTest had not thrown") +} + +Method TestSecondMethod() +{ + do $$$AssertTrue(1, "this would also pass if OnBeforeOneTest had not thrown") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodAssertFailure.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodAssertFailure.cls new file mode 100644 index 000000000..4b78cd2f1 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodAssertFailure.cls @@ -0,0 +1,23 @@ +/// Demonstrates assertion failures in test method bodies. +Class Test.Output.Format.MethodAssertFailure Extends %UnitTest.TestCase +{ + +Method TestPassingAssertions() +{ + do $$$AssertTrue(1, "one is true") + do $$$AssertEquals("hello", "hello", "strings match") +} + +Method TestFailingAssertions() +{ + do $$$AssertTrue(1, "this passes") + do $$$AssertTrue(0, "deliberate failure: zero is not true") + do $$$AssertEquals("expected", "actual", "deliberate mismatch") +} + +Method TestAnotherPass() +{ + do $$$AssertNotTrue(0, "zero is not true") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodThrowFailure.cls b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodThrowFailure.cls new file mode 100644 index 000000000..a9bb2b42f --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/tests/Test/Output/Format/MethodThrowFailure.cls @@ -0,0 +1,21 @@ +/// Demonstrates a thrown error in a test method body. +Class Test.Output.Format.MethodThrowFailure Extends %UnitTest.TestCase +{ + +Method TestPassesNormally() +{ + do $$$AssertTrue(1, "this passes before the throwing method runs") +} + +Method TestThrowsFromBody() +{ + do $$$AssertTrue(1, "this assertion runs before the throw") + $$$ThrowStatus($$$ERROR($$$GeneralError, "deliberate throw from test body")) +} + +Method TestAfterThrow() +{ + do $$$AssertTrue(1, "this method runs independently after the throwing method") +} + +} diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index 519e152c2..039efcd5b 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -1,66 +1,138 @@ -/// Unit Test Class to validate the ZPM 'test' command output configuration. -/// This class ensures two primary functions work correctly: -/// 1. Console Formatting: Verifies the `-f` / `-output` flags correctly -/// format test results for terminal display (e.g., YAML, JSON). -/// 2. File Generation: Verifies the `-DUnitTest.Output` definitions -/// successfully create and populate the structured results files (e.g., .json, .yaml, .toon). +/// Unit tests for the ZPM test output format classes. +/// Verifies that each format produces structurally valid, non-trivial output +/// against the most recent test run at the time the tests execute. Class Test.PM.Unit.TestResultsOPFormatAndFileGenTest Extends %UnitTest.TestCase { -Method TestFail() +/// Temp directory for generated report files, created in OnBeforeAllTests. +Property ReportDir As %String(MAXLEN = 512); + +/// Index of the most recent completed run before this test class starts. +/// Captured in OnBeforeAllTests so TestGetSummaryHasCounts is not sensitive +/// to which test method runs first. +Property PriorRunIndex As %Integer; + +Method OnBeforeAllTests() As %Status { - do $$$AssertFailure("This test is designed to fail to validate the test framework's ability to capture failures.") + set dir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") + if '##class(%File).DirectoryExists(dir) { + set sc = ##class(%File).CreateDirectoryChain(dir) + if $$$ISERR(sc) { + return sc + } + } + set ..ReportDir = dir + // Walk back one from the current (in-progress) run to get a completed run + // with populated SQL result tables. + set currentIndex = $order(^UnitTest.Result(""), -1) + set ..PriorRunIndex = $order(^UnitTest.Result(currentIndex), -1) + return $$$OK } -/// generate .yaml,.json,.toon files -Method TestResultFileGeneration() +Method OnAfterAllTests() As %Status { - do $$$LogMessage("This file generation picks the last or current unit test id and generate the reports") - - set fileDir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") - if '##class(%File).DirectoryExists(fileDir){ - set status = ##class(%File).CreateDirectoryChain(fileDir) - do $$$AssertStatusOK(status,"Directory created: "_fileDir) + if ##class(%File).DirectoryExists(..ReportDir) { + do ##class(%File).RemoveDirectoryTree(..ReportDir) } + return $$$OK +} + +Method TestJsonOutputStructure() +{ + set testIndex = $order(^UnitTest.Result(""), -1) + set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.json") + + set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName, "", testIndex) + do $$$AssertStatusOK(status, "JsonOutput.ToFile succeeded") + do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JSON file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set parsed = ##class(%DynamicAbstractObject).%FromJSON(fileStream) + do $$$AssertTrue($isobject(parsed), "JSON output is parseable") + do $$$AssertTrue(parsed.id '= "", "JSON output has an id field") + do $$$AssertTrue(parsed.namespace '= "", "JSON output has a namespace field") + do $$$AssertTrue($isobject(parsed.results), "JSON output has a results array") +} + +Method TestYamlOutputStructure() +{ + set testIndex = $order(^UnitTest.Result(""), -1) + set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.yaml") - do $$$LogMessage("Start generating the reports") + set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName, "", testIndex) + do $$$AssertStatusOK(status, "YamlOutput.ToFile succeeded") + do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "YAML file is non-empty") - set fileName = ##class(%File).NormalizeFilename(fileDir_"test.yaml") - set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName) - do $$$AssertStatusOK(status,"yaml file generated successfully in "_fileDir) - do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"yaml file is non-empty") + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) + } + do $$$AssertTrue(content [ "unitTest:", "YAML output has unitTest header") + do $$$AssertTrue(content [ "id: ", "YAML output has id field") + do $$$AssertTrue(content [ "namespace:", "YAML output has namespace field") + do $$$AssertTrue(content [ "results:", "YAML output has results section") +} - set fileName = ##class(%File).NormalizeFilename(fileDir_"test.json") - set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName) - do $$$AssertStatusOK(status,"Json file generated successfully in "_fileDir) - do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"json file is non-empty") +Method TestToonOutputStructure() +{ + set testIndex = $order(^UnitTest.Result(""), -1) + set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.toon") - set fileName = ##class(%File).NormalizeFilename(fileDir_"test.toon") - set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName) - do $$$AssertStatusOK(status,"Toon file generated successfully in "_fileDir) - do $$$AssertTrue(##class(%File).GetFileSize(fileName)>0,"toon file is non-empty") + set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName, "", testIndex) + do $$$AssertStatusOK(status, "ToonOutput.ToFile succeeded") + do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "Toon file is non-empty") - do ..ShowGeneratedFilesAndCleaup(fileDir) + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) + } + do $$$AssertTrue(content [ "unitTest:", "Toon output has unitTest header") + do $$$AssertTrue(content [ "results[", "Toon output has results header with row count") + do $$$AssertTrue(content [ "{suiteName,", "Toon output has column header line") } -Method ShowGeneratedFilesAndCleaup(fileDir As %String) +Method TestJUnitOutputStructure() { - do $$$LogMessage("Display the generated unit test report files") - set fileSet = ##class(%File).FileSetFunc(fileDir) - while fileSet.%Next() { - set file = fileSet.Name - set fileNames(file) = fileSet.ItemName - do $$$LogMessage("Generated file "_file) + set testIndex = $order(^UnitTest.Result(""), -1) + set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.xml") + + set status = ##class(%IPM.Test.JUnitOutput).ToFile(fileName, "", testIndex) + do $$$AssertStatusOK(status, "JUnitOutput.ToFile succeeded") + do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JUnit XML file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) } - do $$$LogMessage("Started Cleanup the generated unit test report files") - set file = "" - for { - set file = $order(fileNames(file),1,fileName) - quit:file="" - set status = ##class(%File).Delete(file) - do $$$AssertStatusOK(status," File '"_fileName_"' has been deleted successfully") + do $$$AssertTrue(content [ "", "JUnit output has root element") + do $$$AssertTrue(content [ " element") + do $$$AssertTrue(content [ "", "JUnit output has closing element") +} + +Method TestGetSummaryHasCounts() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping GetSummary count check") + quit } - do $$$LogMessage("File cleanup completed") + + set summary = ##class(%IPM.Test.Abstract).GetSummary(testIndex) + do $$$AssertTrue($isobject(summary), "GetSummary returns an object") + do $$$AssertTrue($isobject(summary.methods), "Summary has a methods sub-object") + do $$$AssertTrue($isobject(summary.assertions), "Summary has an assertions sub-object") + do $$$AssertTrue(summary.methods.total > 0, "Summary reports at least one method") + do $$$AssertTrue(summary.assertions.total > 0, "Summary reports at least one assertion") + do $$$AssertEquals(summary.methods.passed + summary.methods.failed, summary.methods.total, "Method counts add up") + do $$$AssertEquals(summary.assertions.passed + summary.assertions.failed, summary.assertions.total, "Assertion counts add up") } } From 8c313a6fb6aaec9c957ef30fea186400bf677184 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Apr 2026 14:30:51 -0400 Subject: [PATCH 07/18] Update changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e578ec59a..de7a198f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #959: In ORAS repos, external name can now be used interchangeably with (default) name for `install` and `update`, i.e. a module published with its (default) name can be installed using its external name. - #951: The `unpublish` command will skip user confirmation prompt if the `-force` flag is provided. - #1018: Require module name for uninstall when not using the -all flag -- #971: Adds support for JSON, YAML, and Toon formats via the -f flag and new -DUnitTest.*Output directives. ### Changed - #316: All parameters, except developer mode, included with a `load`, `install` or `update` command will be propagated to dependencies From 365114cc5566d249d45649634241808c43c0b88a Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Apr 2026 16:41:05 -0400 Subject: [PATCH 08/18] Use tree traversal instead of SQL query to capture test failures --- src/cls/IPM/Main.cls | 4 +- src/cls/IPM/ResourceProcessor/Test.cls | 2 +- src/cls/IPM/Test/Abstract.cls | 238 +++++++++++------- src/cls/IPM/Test/JUnitOutput.cls | 178 ++++++------- src/cls/IPM/Test/JsonOutput.cls | 87 ++----- src/cls/IPM/Test/Manager.cls | 70 ++---- src/cls/IPM/Test/ToonOutput.cls | 96 ++++--- src/cls/IPM/Test/YamlOutput.cls | 176 +++++++++---- .../_data/test-output-format/module.xml | 1 - .../TestResultsOPFormatAndFileGenTest.cls | 35 ++- 10 files changed, 495 insertions(+), 392 deletions(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index fc75f520f..3b0a9a376 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -75,8 +75,8 @@ Can also specify desired version to update to. - - + + diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 520600c32..017cc78b9 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -210,7 +210,7 @@ Method OnPhase( } write ! if $data(pParams("outputformat"),outputFormat)||('tVerbose) { - write !,"Test Result Summary",! + write !,"Test Result Summary" // CLI flag takes precedence; fall back to global config, then default to Toon set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() if $get(outputFormat)="" { diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index 6ffed5522..7473b9b1d 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -1,71 +1,167 @@ -/// The class serves as the base class for all the unit test result formatting. +/// Base class for all unit test result formatters. Class %IPM.Test.Abstract Extends %RegisteredObject { ClassMethod ToFile( - fileName As %String, - caseStatus As %String = "", - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + fileName As %String, + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] { } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] { } -/// Returns a summary %DynamicObject with run metadata and method/assertion counts. -/// A method is counted as "failed" if any of its assertions failed. -ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %DynamicObject +/// Walks ^UnitTest.Result and returns a fully-populated result tree. +/// Non-assertion failures (thrown errors, lifecycle hook failures) appear +/// as non-empty "error" fields on method and case nodes rather than as +/// assert entries. +/// +/// ^UnitTest.Result global structure: +/// (testIndex, suite) → $list(status, time) +/// (testIndex, suite, case) → $list(status, time, errCategory, errMsg) +/// (testIndex, suite, case, method) → $list(status, time, errCategory, errMsg) +/// (testIndex, suite, case, method, counter) → $list(assertPassed, action, description, location) +/// status/assertPassed: 1=pass, 0=fail +/// errCategory/errMsg: non-empty only when a lifecycle hook or thrown error caused the failure +/// Instance metadata (Namespace, Duration, DateTime) lives in %UnitTest_Result.TestInstance +ClassMethod BuildResultTree(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %DynamicObject { - set result = ..GetAllTestResultsFunc(testIndex) - set (ns, duration, testDateTime) = "" - set (assertTotal, assertPassed, assertFailed) = 0 + set meta = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Namespace,Duration,DateTime FROM %UnitTest_Result.TestInstance WHERE ID=?", + testIndex) + set ns = "" + set duration = "" + set testDateTime = "" + if meta.%Next() { + set ns = meta.%Get("Namespace") + set duration = meta.%Get("Duration") + set testDateTime = meta.%Get("DateTime") + } - while result.%Next() { - if ns = "" { - set ns = result.namespace - set duration = result.duration - set testDateTime = result.testDateTime - } - if result.assertStatus = "passed" { - set assertTotal = assertTotal + 1 - set assertPassed = assertPassed + 1 - } elseif result.assertStatus = "failed" { - set assertTotal = assertTotal + 1 - set assertFailed = assertFailed + 1 - } - // Track method status — once "failed", stays "failed" even if later assertions pass - set methodKey = result.suiteName_"||"_result.testcaseName_"||"_result.methodName - if '$data(seenMethods(methodKey)) { - set seenMethods(methodKey) = "" - } - if result.assertStatus = "failed" { - set seenMethods(methodKey) = "failed" - } elseif (result.assertStatus = "passed") && (seenMethods(methodKey) '= "failed") { - set seenMethods(methodKey) = "passed" + set tree = { + "id": (testIndex), + "namespace": (ns), + "duration": (duration), + "testDateTime": (testDateTime), + "suites": [] + } + + set suite = "" + for { + set suite = $order(^UnitTest.Result(testIndex, suite), 1, suiteData) + quit:suite="" + + set suiteObj = {"name": (suite), "status": "passed", "cases": []} + do tree.suites.%Push(suiteObj) + + set testCase = "" + for { + set testCase = $order(^UnitTest.Result(testIndex, suite, testCase), 1, caseData) + quit:testCase="" + + set caseStatus = $select($listget(caseData, 1) '= 0: "passed", 1: "failed") + set caseError = "" + if caseStatus = "failed" { + set errCat = $listget(caseData, 3) + // Only set when errCat is non-empty; errMsg alone is IRIS's own aggregation + // string ("There are failed TestMethods") and not a real error message. + if errCat '= "" { + set caseError = errCat_": "_$listget(caseData, 4) + } + } + set caseObj = {"name": (testCase), "status": (caseStatus), "error": (caseError), "methods": []} + do suiteObj.cases.%Push(caseObj) + + if caseStatus = "failed" { + set suiteObj.status = "failed" + } + + set method = "" + for { + set method = $order(^UnitTest.Result(testIndex, suite, testCase, method), 1, methodData) + quit:method="" + + set methodStatus = $select($listget(methodData, 1) '= 0: "passed", 1: "failed") + set methodError = "" + if methodStatus = "failed" { + set errCat = $listget(methodData, 3) + if errCat '= "" { + set methodError = errCat_": "_$listget(methodData, 4) + } + } + set methodObj = { + "name": (method), + "status": (methodStatus), + "duration": ($listget(methodData, 2)), + "error": (methodError), + "asserts": [] + } + do caseObj.methods.%Push(methodObj) + + set assert = "" + for { + set assert = $order(^UnitTest.Result(testIndex, suite, testCase, method, assert), 1, assertData) + quit:assert="" + set assertObj = { + "counter": (assert), + "status": ($select($listget(assertData, 1) '= 0: "passed", 1: "failed")), + "action": ($listget(assertData, 2)), + "description": ($listget(assertData, 3)), + "location": ($listget(assertData, 4)) + } + do methodObj.asserts.%Push(assertObj) + } + } } } + return tree +} +/// Returns a summary %DynamicObject with run metadata and method/assertion counts. +/// Counts methods as failed if any assert failed OR if a non-assertion error occurred. +ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %DynamicObject +{ + set tree = ..BuildResultTree(testIndex) + set (assertTotal, assertPassed, assertFailed) = 0 set (methodTotal, methodPassed, methodFailed) = 0 - set key = "" - for { - set key = $order(seenMethods(key), 1, status) - quit:key="" - if status = "passed" { - set methodTotal = methodTotal + 1 - set methodPassed = methodPassed + 1 - } elseif status = "failed" { - set methodTotal = methodTotal + 1 - set methodFailed = methodFailed + 1 + + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + set methodTotal = methodTotal + 1 + if methodObj.status = "passed" { + set methodPassed = methodPassed + 1 + } else { + set methodFailed = methodFailed + 1 + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + set assertTotal = assertTotal + 1 + if assertObj.status = "passed" { + set assertPassed = assertPassed + 1 + } else { + set assertFailed = assertFailed + 1 + } + } + } + // Case-level error (e.g. OnBeforeAllTests failure) — no methods ran + if caseObj.error '= "" { + set methodTotal = methodTotal + 1 + set methodFailed = methodFailed + 1 + } } } return { "id": (testIndex), - "namespace": (ns), - "duration": (duration), - "testDateTime": (testDateTime), + "namespace": (tree.namespace), + "duration": (tree.duration), + "testDateTime": (tree.testDateTime), "methods": { "total": (methodTotal), "passed": (methodPassed), @@ -79,54 +175,4 @@ ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)} } } -Query FilteredTestResults( - instance As %Integer, - testStatus) As %SQLQuery(ROWSPEC = "namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") -{ -SELECT - inst.Namespace AS namespace, - inst.Duration AS duration, - inst.DateTime AS testDateTime, - suite.Name AS suiteName, - testCase.Name AS testcaseName, - method.Name AS methodName, - testAssert.TestMethod AS testMethod, - testAssert.Action AS assertAction, - testAssert.Counter AS assertCounter, - testAssert.Description AS assertDescription, - testAssert.Location AS assertLocation, - testAssert.Status AS assertStatus -FROM -%UnitTest_Result.TestInstance inst -JOIN %UnitTest_Result.TestSuite suite ON suite.TestInstance=inst.ID -JOIN %UnitTest_Result.TestCase testCase ON testCase.TestSuite=suite.ID -JOIN %UnitTest_Result.TestMethod method ON method.TestCase=testCase.ID -JOIN %UnitTest_Result.TestAssert testAssert ON testAssert.TestMethod=method.ID -WHERE inst.ID=:instance AND testAssert.Status=:testStatus -} - -Query GetAllTestResults(instance As %Integer) As %SQLQuery(ROWSPEC = "namespace:%String,duration:%String,testDateTime:%String,suiteName:%String,testcaseName:%String,methodName:%String,testMethod:%String,assertAction:%String,assertCounter:%Integer,assertDescription:%String,assertLocation:%String,assertStatus:%String", SELECTMODE = "DISPLAY") -{ -SELECT - inst.Namespace AS namespace, - inst.Duration AS duration, - inst.DateTime AS testDateTime, - suite.Name AS suiteName, - testCase.Name AS testcaseName, - method.Name AS methodName, - testAssert.TestMethod AS testMethod, - testAssert.Action AS assertAction, - testAssert.Counter AS assertCounter, - testAssert.Description AS assertDescription, - testAssert.Location AS assertLocation, - testAssert.Status AS assertStatus -FROM -%UnitTest_Result.TestInstance inst -JOIN %UnitTest_Result.TestSuite suite ON suite.TestInstance=inst.ID -JOIN %UnitTest_Result.TestCase testCase ON testCase.TestSuite=suite.ID -JOIN %UnitTest_Result.TestMethod method ON method.TestCase=testCase.ID -JOIN %UnitTest_Result.TestAssert testAssert ON testAssert.TestMethod=method.ID -WHERE inst.ID=:instance -} - } diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index 1a03bbb23..baa5384f6 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -2,9 +2,8 @@ Class %IPM.Test.JUnitOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - fileName As %String, - caseStatus As %String = "", - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { @@ -12,109 +11,74 @@ ClassMethod ToFile( set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - kill ^||TMP - set suite = "" - for { - set suite = $order(^UnitTest.Result(testIndex, suite), 1, suiteData) - quit:suite="" - set ^||TMP("S", suite, "time") = $listget(suiteData, 2) + set tree = ..BuildResultTree(testIndex) - set testCase = "" - for { - set testCase = $order(^UnitTest.Result(testIndex, suite, testCase), 1, testCaseData) - quit:testCase="" - - do $increment(^||TMP("S", suite, "tests")) - set ^||TMP("S", suite, "C", testCase, "time") = $listget(testCaseData, 2) - set method = "" - for { - set method = $order(^UnitTest.Result(testIndex, suite, testCase, method), 1, methodData) - quit:method="" - - set ^||TMP("S", suite, "C", testCase, "M", method, "time") = $listget(methodData, 2) - set assert = "" - for { - set assert = $order(^UnitTest.Result(testIndex, suite, testCase, method, assert), 1, assertData) - quit:assert="" + do fileStream.WriteLine("") + do fileStream.WriteLine("") - do $increment(^||TMP("S", suite, "assertions")) - do $increment(^||TMP("S", suite, "C", testCase, "assertions")) - do $increment(^||TMP("S", suite, "C", testCase, "M", method, "assertions")) - if $listget(assertData) = 0 { - do $increment(^||TMP("S", suite, "failures")) - do $increment(^||TMP("S", suite, "C", testCase, "failures")) - set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", method, "failures")) - set ^||TMP("S", suite, "C", testCase, "M", method, "failures", failureIndex) = - $listget(assertData, 2) _ ": " _ $listget(assertData, 3) - } - } - if ($listget(methodData) = 0) - && ('$data(^||TMP("S", suite, "C", testCase, "M", method, "failures"))) { - do $increment(^||TMP("S", suite, "failures")) - do $increment(^||TMP("S", suite, "C", testCase, "failures")) - set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", method, "failures")) - set ^||TMP("S", suite, "C", testCase, "M", method, "failures", failureIndex) = - $listget(methodData, 3) _ ": " _ $listget(methodData, 4) - } + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set suiteTests = 0 + set suiteFailures = 0 + set suiteAssertions = 0 + set suiteDuration = 0 + set caseIter2 = suiteObj.cases.%GetIterator() + while caseIter2.%GetNext(, .caseObj2) { + set methodIter2 = caseObj2.methods.%GetIterator() + while methodIter2.%GetNext(, .methodObj2) { + set suiteTests = suiteTests + 1 + if methodObj2.status = "failed" { set suiteFailures = suiteFailures + 1 } + set suiteDuration = suiteDuration + methodObj2.duration + set assertIter2 = methodObj2.asserts.%GetIterator() + while assertIter2.%GetNext(, .unused) { set suiteAssertions = suiteAssertions + 1 } } - - if $listget(testCaseData) = 0 - && ('$data(^||TMP("S", suite, "C", testCase, "failures"))) { - do $increment(^||TMP("S", suite, "failures")) - do $increment(^||TMP("S", suite, "C", testCase, "failures")) - set failureIndex = $increment(^||TMP("S", suite, "C", testCase, "M", testCase, "failures")) - set ^||TMP("S", suite, "C", testCase, "M", testCase, "failures", failureIndex) = - $listget(testCaseData, 3) _ ": " _ $listget(testCaseData, 4) + if caseObj2.error '= "" { + set suiteTests = suiteTests + 1 + set suiteFailures = suiteFailures + 1 } } - } - - do fileStream.WriteLine("") - do fileStream.WriteLine("") - set suite = "" - for { - set suite = $order(^||TMP("S", suite)) - quit:suite="" do fileStream.Write("") - set testCase = "" - for { - set testCase = $order(^||TMP("S", suite, "C", testCase)) - quit:testCase="" - + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { do fileStream.Write("") - set method = "" - for { - set method = $order(^||TMP("S", suite, "C", testCase, "M", method)) - quit:method="" + if caseObj.error '= "" { + do fileStream.Write("") + set msg = ..EncodeXMLAttr(caseObj.error) + do fileStream.Write("") + do fileStream.WriteLine("") + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + set methodAssertions = methodObj.asserts.%Size() do fileStream.Write("") - set failureKey = "" - for { - set failureKey = $order(^||TMP("S",suite,"C",testCase,"M",method,"failures",failureKey),1,message) - quit:failureKey="" - set message = $zstrip(message,"*C") - set message = $zconvert($zconvert(message,"O","UTF8"),"O","XML") - // $zconvert does not encode newlines. - set message = $replace(message,$char(10)," ") - set message = $replace(message,$char(13)," ") - do fileStream.Write("") + + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + set msg = ..EncodeXMLAttr(assertObj.action_": "_assertObj.description) + do fileStream.Write("") + do fileStream.WriteLine("") + } + } + if (methodObj.status = "failed") && (methodObj.error '= "") { + set msg = ..EncodeXMLAttr(methodObj.error) + do fileStream.Write("") do fileStream.WriteLine("") } do fileStream.WriteLine("") @@ -124,8 +88,6 @@ ClassMethod ToFile( do fileStream.WriteLine("") } do fileStream.WriteLine("") - kill ^||TMP - $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -134,7 +96,7 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { @@ -144,10 +106,23 @@ ClassMethod OutputToDevice( write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" if summary.methods.failed > 0 { - set result = ..GetAllTestResultsFunc(testIndex) + set tree = ..BuildResultTree(testIndex) write ! - while result.%Next() { - write !,result.suiteName_"/"_result.testcaseName_"/"_result.methodName_" "_result.assertAction_" ["_result.assertStatus_"]" + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" [failed]" + if methodObj.error '= "" { write " "_methodObj.error } + } + } + } } } } catch ex { @@ -156,4 +131,13 @@ ClassMethod OutputToDevice( return sc } +ClassMethod EncodeXMLAttr(msg As %String) As %String [ Private ] +{ + set msg = $zstrip(msg, "*C") + set msg = $zconvert($zconvert(msg, "O", "UTF8"), "O", "XML") + set msg = $replace(msg, $char(10), " ") + set msg = $replace(msg, $char(13), " ") + return msg +} + } diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 9b973b6ec..94570ac84 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -2,19 +2,15 @@ Class %IPM.Test.JsonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - fileName As %String, - caseStatus As %String = "", - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - set responseJson = ..JSON(testIndex, caseStatus) - if $isobject(responseJson) { - do fileStream.Write(responseJson.%ToJSON()) - } + do fileStream.Write(..BuildResultTree(testIndex).%ToJSON()) $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -23,16 +19,31 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set summary = ..GetSummary(testIndex) - set failures = ..JSON(testIndex, "failed") - set output = { - "summary": (summary), - "failures": (failures.results) + set failedMethods = [] + if summary.methods.failed > 0 { + set tree = ..BuildResultTree(testIndex) + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "error": (caseObj.error)}) + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error), "asserts": (methodObj.asserts)}) + } + } + } + } } + set output = {"summary": (summary), "failures": (failedMethods)} write ! do output.%ToJSON() } catch ex { @@ -41,56 +52,4 @@ ClassMethod OutputToDevice( return sc } -ClassMethod JSON( - testIndex As %Integer, - testStatus As %String) As %DynamicObject -{ - if testStatus'="" { - set result = ..FilteredTestResultsFunc(testIndex, testStatus) - } else { - set result = ..GetAllTestResultsFunc(testIndex) - } - set unitTest = {} - set unitTest.results = [] - set (previousID,currentSuite,currentTestcase,suiteObj,testcaseObj) = "" - - while result.%Next() { - if previousID = "" { - set unitTest.id = testIndex - set unitTest.namespace = result.namespace - set unitTest.duration = result.duration - set unitTest.testDateTime = result.testDateTime - } - set previousID = testIndex - if result.suiteName '= currentSuite { - set currentSuite = result.suiteName - set suiteObj = { - "suiteName": (currentSuite), - "testcases": [] - } - do unitTest.results.%Push(suiteObj) - set currentTestcase = "" - } - if result.testcaseName '= currentTestcase { - set currentTestcase = result.testcaseName - set testcaseObj = { - "testcaseName": (currentTestcase), - "methods": [] - } - do suiteObj.testcases.%Push(testcaseObj) - } - set methodObj = { - "methodName": (result.methodName), - "testMethod": (result.testMethod), - "assertAction": (result.assertAction), - "assertCounter": (result.assertCounter), - "assertDescription": (result.assertDescription), - "assertLocation": (result.assertLocation), - "assertStatus": (result.assertStatus) - } - do testcaseObj.methods.%Push(methodObj) - } - return unitTest -} - } diff --git a/src/cls/IPM/Test/Manager.cls b/src/cls/IPM/Test/Manager.cls index bd3b95bbe..29e09aa94 100644 --- a/src/cls/IPM/Test/Manager.cls +++ b/src/cls/IPM/Test/Manager.cls @@ -49,6 +49,7 @@ ClassMethod LoadTestDirectory( } /// Returns $$$OK if the last unit test run was successful, or an error if it was unsuccessful. +/// Uses ^UnitTest.Result directly so that lifecycle/hook/throw failures are counted alongside assertion failures. ClassMethod GetLastStatus(Output pFailureCount As %Integer) As %Status { set tSC = $$$OK @@ -58,31 +59,10 @@ ClassMethod GetLastStatus(Output pFailureCount As %Integer) As %Status } kill ^||%UnitTest.Manager.LastResult // Clean up if tLogIndex { - set tRes = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestAssert where Status = 0 "_ - "and TestMethod->TestCase->TestSuite->TestInstance->InstanceIndex = ?",tLogIndex) - if (tRes.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(tRes.%SQLCODE,tRes.%Message) - } - do tRes.%Next(.tSC) - $$$ThrowOnError(tSC) - set pFailureCount = tRes.%GetData(1) + set summary = ##class(%IPM.Test.Abstract).GetSummary(tLogIndex) + set pFailureCount = summary.methods.failed if (pFailureCount > 0) { - set tSC = $$$ERROR($$$GeneralError,$$$FormatText("%1 assertion(s) failed.",pFailureCount)) - } else { - // Double check that no other failures were reported - e.g., failures loading that would lead to no assertions passing or failing! - set tRes = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestSuite where Status = 0 "_ - "and TestInstance->InstanceIndex = ?",tLogIndex) - if (tRes.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(tRes.%SQLCODE,tRes.%Message) - } - do tRes.%Next(.tSC) - $$$ThrowOnError(tSC) - set pFailureCount = tRes.%GetData(1) - if (pFailureCount > 0) { - set tSC = $$$ERROR($$$GeneralError,$$$FormatText("%1 test suite(s) failed.",pFailureCount)) - } + set tSC = $$$ERROR($$$GeneralError,$$$FormatText("%1 failure(s).",pFailureCount)) } } else { set tSC = $$$ERROR($$$GeneralError,"No unit test results recorded.") @@ -104,27 +84,27 @@ ClassMethod OutputFailures() if 'tLogIndex { quit } - set tLogGN = $name(^UnitTest.Result(tLogIndex)) - set tRoot = "" - for { - set tRoot = $order(@tLogGN@(tRoot)) - quit:tRoot="" - set tSuite = "" - for { - set tSuite = $order(@tLogGN@(tRoot, tSuite)) - quit:tSuite="" - set tMethod = "" - for { - set tMethod = $order(@tLogGN@(tRoot, tSuite, tMethod)) - quit:tMethod="" - - set tAssert = "" - for { - set tAssert = $order(@tLogGN@(tRoot, tSuite, tMethod, tAssert), 1, tAssertInfo) - quit:tAssert="" - set $listbuild(status, type, text) = tAssertInfo - continue:status - write !,$$$FormattedLine($$$Red, "FAILED " _ tSuite _ ":" _ tMethod), ": " _ type _ " - " _ text + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(tLogIndex) + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + // Case-level failure (e.g. OnBeforeAllTests) — no methods ran + if caseObj.error '= "" { + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ caseObj.name), ": " _ caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status '= "failed" { continue } + // Method-level lifecycle/throw failure (no assert row) + if methodObj.error '= "" { + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ methodObj.name), ": " _ methodObj.error + } + // Individual assertion failures + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status '= "failed" { continue } + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ methodObj.name), ": " _ assertObj.action _ " - " _ assertObj.description } } } diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index 599affa4f..d9115e5b4 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -2,9 +2,8 @@ Class %IPM.Test.ToonOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - fileName As %String, - caseStatus As %String = "", - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { @@ -12,34 +11,42 @@ ClassMethod ToFile( set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - if caseStatus '= "" { - set result = ..FilteredTestResultsFunc(testIndex, caseStatus) - } else { - set result = ..GetAllTestResultsFunc(testIndex) - } - - set rowCount = 0 - set (ns, duration, testDateTime) = "" + set tree = ..BuildResultTree(testIndex) set rowStream = ##class(%Stream.TmpCharacter).%New() - while result.%Next() { - if rowCount = 0 { - set ns = result.namespace - set duration = result.duration - set testDateTime = result.testDateTime + set rowCount = 0 + + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + set rowCount = rowCount + 1 + set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" + do rowStream.WriteLine(row) + } + } } - set rowCount = rowCount + 1 - set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ - result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" - do rowStream.WriteLine(data) } do fileStream.WriteLine("unitTest:") - do fileStream.WriteLine(" id: "_testIndex) - do fileStream.WriteLine(" namespace: "_ns) - do fileStream.WriteLine(" duration: "_duration) - do fileStream.WriteLine(" testDateTime: "_testDateTime) + do fileStream.WriteLine(" id: "_tree.id) + do fileStream.WriteLine(" namespace: "_tree.namespace) + do fileStream.WriteLine(" duration: "_tree.duration) + do fileStream.WriteLine(" testDateTime: "_tree.testDateTime) do fileStream.WriteLine() do fileStream.WriteLine("results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + do rowStream.Rewind() do fileStream.CopyFrom(rowStream) $$$ThrowOnError(fileStream.%Save()) } catch ex { @@ -49,7 +56,7 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { @@ -61,14 +68,39 @@ ClassMethod OutputToDevice( write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" if summary.methods.failed > 0 { - set result = ..FilteredTestResultsFunc(testIndex, "failed") - set currentID = "" + set tree = ..BuildResultTree(testIndex) + set failCount = 0 + set failStream = ##class(%Stream.TmpCharacter).%New() + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + set failCount = failCount + 1 + do failStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.error '= "" { + set failCount = failCount + 1 + do failStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + set failCount = failCount + 1 + set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,"_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" + do failStream.WriteLine(row) + } + } + } + } + } write ! - write !,"failures["_summary.assertions.failed_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" - while result.%Next() { - set data = " "_result.suiteName_","_result.testcaseName_","_result.methodName_","_result.assertStatus_","_ - result.assertAction_","_result.assertCounter_","""_$translate(result.assertDescription,"""")_""","""_result.assertLocation_"""" - write !,data + write !,"failures["_failCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + do failStream.Rewind() + while 'failStream.AtEnd { + write !,failStream.ReadLine() } } } catch ex { diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 40e373413..54c05e566 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -2,16 +2,15 @@ Class %IPM.Test.YamlOutput Extends %IPM.Test.Abstract { ClassMethod ToFile( - fileName As %String, - caseStatus As %String = "", - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + fileName As %String, + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - do fileStream.CopyFrom(..YAML(testIndex, caseStatus)) + do fileStream.CopyFrom(..YAML(testIndex)) $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -20,7 +19,7 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status { set sc = $$$OK try { @@ -43,10 +42,10 @@ ClassMethod OutputToDevice( if summary.methods.failed > 0 { write ! write !,"failures:" - set failureStream = ..YAML(testIndex, "failed", 0) - do failureStream.Rewind() - while 'failureStream.AtEnd { - write !,failureStream.ReadLine() + set failStream = ..YAMLFailures(testIndex) + do failStream.Rewind() + while 'failStream.AtEnd { + write !,failStream.ReadLine() } } } catch ex { @@ -55,52 +54,131 @@ ClassMethod OutputToDevice( return sc } -ClassMethod YAML( - testIndex = {$order(^UnitTest.Result(""),-1)}, - testStatus As %String = "", - includeHeader As %Boolean = 1) As %Stream.TmpCharacter +/// Returns a stream with the full YAML document (header + all results). +ClassMethod YAML(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Stream.TmpCharacter { - if testStatus '= "" { - set result = ..FilteredTestResultsFunc(testIndex, testStatus) - } else { - set result = ..GetAllTestResultsFunc(testIndex) - } + set tree = ..BuildResultTree(testIndex) set yamlStream = ##class(%Stream.TmpCharacter).%New() - set (currentID, currentSuite, currentTestcase) = "" - while result.%Next() { - if currentID = "" { - set currentID = testIndex - if includeHeader { - do yamlStream.WriteLine("unitTest:") - do yamlStream.WriteLine(" id: "_testIndex) - do yamlStream.WriteLine(" namespace: """_result.namespace_"""") - do yamlStream.WriteLine(" duration: "_result.duration) - do yamlStream.WriteLine(" testDateTime: """_result.testDateTime_"""") - do yamlStream.WriteLine() - do yamlStream.WriteLine(" results:") + + do yamlStream.WriteLine("unitTest:") + do yamlStream.WriteLine(" id: "_tree.id) + do yamlStream.WriteLine(" namespace: """_tree.namespace_"""") + do yamlStream.WriteLine(" duration: "_tree.duration) + do yamlStream.WriteLine(" testDateTime: """_tree.testDateTime_"""") + do yamlStream.WriteLine() + do yamlStream.WriteLine(" results:") + do yamlStream.CopyFrom(..YAMLResults(tree)) + return yamlStream +} + +/// Returns a stream of YAML result rows for all entries in the tree (no header). +/// Used by ToFile via YAML, and by OutputToDevice for the failures section. +ClassMethod YAMLResults(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] +{ + set resultStream = ##class(%Stream.TmpCharacter).%New() + set (currentSuite, currentCase) = "" + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + if suiteObj.name '= currentSuite { + set currentSuite = suiteObj.name + set currentCase = "" + do resultStream.WriteLine(" - suiteName: """_suiteObj.name_"""") + do resultStream.WriteLine(" testcases:") + } + do resultStream.WriteLine(" - testcaseName: """_caseObj.name_"""") + do resultStream.WriteLine(" error: |") + do resultStream.WriteLine(" "_$replace(caseObj.error, $char(10), $char(10)_" ")) + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if suiteObj.name '= currentSuite { + set currentSuite = suiteObj.name + set currentCase = "" + do resultStream.WriteLine(" - suiteName: """_suiteObj.name_"""") + do resultStream.WriteLine(" testcases:") + } + if caseObj.name '= currentCase { + set currentCase = caseObj.name + do resultStream.WriteLine(" - testcaseName: """_caseObj.name_"""") + do resultStream.WriteLine(" methods:") + } + do resultStream.WriteLine(" - methodName: """_methodObj.name_"""") + do resultStream.WriteLine(" status: """_methodObj.status_"""") + if methodObj.error '= "" { + do resultStream.WriteLine(" error: |") + do resultStream.WriteLine(" "_$replace(methodObj.error, $char(10), $char(10)_" ")) + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + do resultStream.WriteLine(" assertAction: """_assertObj.action_"""") + do resultStream.WriteLine(" assertCounter: "_assertObj.counter) + do resultStream.WriteLine(" assertStatus: """_assertObj.status_"""") + do resultStream.WriteLine(" assertDescription: |") + do resultStream.WriteLine(" "_$replace(assertObj.description, $char(10), $char(10)_" ")) + do resultStream.WriteLine(" assertLocation: """_assertObj.location_"""") + } } } - if result.suiteName '= currentSuite { - set currentSuite = result.suiteName - set currentTestcase = "" - do yamlStream.WriteLine(" - suiteName: """_result.suiteName_"""") - do yamlStream.WriteLine(" testcases:") - } - if result.testcaseName '= currentTestcase { - set currentTestcase = result.testcaseName - do yamlStream.WriteLine(" - testcaseName: """_result.testcaseName_"""") - do yamlStream.WriteLine(" methods:") + } + return resultStream +} + +/// Returns a stream of YAML failure rows only (no header). Used by OutputToDevice. +ClassMethod YAMLFailures(testIndex As %Integer) As %Stream.TmpCharacter [ Private ] +{ + set tree = ..BuildResultTree(testIndex) + set failStream = ##class(%Stream.TmpCharacter).%New() + set (currentSuite, currentCase) = "" + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + if suiteObj.name '= currentSuite { + set currentSuite = suiteObj.name + set currentCase = "" + do failStream.WriteLine(" - suiteName: """_suiteObj.name_"""") + do failStream.WriteLine(" testcases:") + } + do failStream.WriteLine(" - testcaseName: """_caseObj.name_"""") + do failStream.WriteLine(" error: |") + do failStream.WriteLine(" "_$replace(caseObj.error, $char(10), $char(10)_" ")) + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status '= "failed" { continue } + if suiteObj.name '= currentSuite { + set currentSuite = suiteObj.name + set currentCase = "" + do failStream.WriteLine(" - suiteName: """_suiteObj.name_"""") + do failStream.WriteLine(" testcases:") + } + if caseObj.name '= currentCase { + set currentCase = caseObj.name + do failStream.WriteLine(" - testcaseName: """_caseObj.name_"""") + do failStream.WriteLine(" methods:") + } + do failStream.WriteLine(" - methodName: """_methodObj.name_"""") + if methodObj.error '= "" { + do failStream.WriteLine(" error: |") + do failStream.WriteLine(" "_$replace(methodObj.error, $char(10), $char(10)_" ")) + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status '= "failed" { continue } + do failStream.WriteLine(" assertAction: """_assertObj.action_"""") + do failStream.WriteLine(" assertCounter: "_assertObj.counter) + do failStream.WriteLine(" assertDescription: |") + do failStream.WriteLine(" "_$replace(assertObj.description, $char(10), $char(10)_" ")) + do failStream.WriteLine(" assertLocation: """_assertObj.location_"""") + } + } } - do yamlStream.WriteLine(" - methodName: """_result.methodName_"""") - do yamlStream.WriteLine(" testMethod: """_result.testMethod_"""") - do yamlStream.WriteLine(" assertAction: """_result.assertAction_"""") - do yamlStream.WriteLine(" assertCounter: "_result.assertCounter) - do yamlStream.WriteLine(" assertStatus: """_result.assertStatus_"""") - do yamlStream.WriteLine(" assertDescription: |") - do yamlStream.WriteLine(" "_$replace(result.assertDescription, $char(10), $char(10)_" ")) - do yamlStream.WriteLine(" assertLocation: """_result.assertLocation_"""") } - return yamlStream + return failStream } } diff --git a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml index a1a9eb872..effe0d57a 100644 --- a/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml +++ b/tests/integration_tests/Test/PM/Integration/_data/test-output-format/module.xml @@ -7,7 +7,6 @@ module src - diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index 039efcd5b..ef65e4cf7 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -42,7 +42,7 @@ Method TestJsonOutputStructure() set testIndex = $order(^UnitTest.Result(""), -1) set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.json") - set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName, "", testIndex) + set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName, testIndex) do $$$AssertStatusOK(status, "JsonOutput.ToFile succeeded") do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JSON file is non-empty") @@ -52,7 +52,7 @@ Method TestJsonOutputStructure() do $$$AssertTrue($isobject(parsed), "JSON output is parseable") do $$$AssertTrue(parsed.id '= "", "JSON output has an id field") do $$$AssertTrue(parsed.namespace '= "", "JSON output has a namespace field") - do $$$AssertTrue($isobject(parsed.results), "JSON output has a results array") + do $$$AssertTrue($isobject(parsed.suites), "JSON output has a suites array") } Method TestYamlOutputStructure() @@ -60,7 +60,7 @@ Method TestYamlOutputStructure() set testIndex = $order(^UnitTest.Result(""), -1) set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.yaml") - set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName, "", testIndex) + set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName, testIndex) do $$$AssertStatusOK(status, "YamlOutput.ToFile succeeded") do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "YAML file is non-empty") @@ -81,7 +81,7 @@ Method TestToonOutputStructure() set testIndex = $order(^UnitTest.Result(""), -1) set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.toon") - set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName, "", testIndex) + set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName, testIndex) do $$$AssertStatusOK(status, "ToonOutput.ToFile succeeded") do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "Toon file is non-empty") @@ -101,7 +101,7 @@ Method TestJUnitOutputStructure() set testIndex = $order(^UnitTest.Result(""), -1) set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.xml") - set status = ##class(%IPM.Test.JUnitOutput).ToFile(fileName, "", testIndex) + set status = ##class(%IPM.Test.JUnitOutput).ToFile(fileName, testIndex) do $$$AssertStatusOK(status, "JUnitOutput.ToFile succeeded") do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JUnit XML file is non-empty") @@ -135,4 +135,29 @@ Method TestGetSummaryHasCounts() do $$$AssertEquals(summary.assertions.passed + summary.assertions.failed, summary.assertions.total, "Assertion counts add up") } +Method TestBuildResultTreeStructure() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping tree structure check") + quit + } + + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(testIndex) + do $$$AssertTrue($isobject(tree), "BuildResultTree returns an object") + do $$$AssertTrue(tree.id '= "", "Tree has an id") + do $$$AssertTrue(tree.namespace '= "", "Tree has a namespace") + do $$$AssertTrue($isobject(tree.suites), "Tree has a suites array") + do $$$AssertTrue(tree.suites.%Size() > 0, "Tree has at least one suite") + + set suiteObj = tree.suites.%Get(0) + do $$$AssertTrue($isobject(suiteObj), "First suite is an object") + do $$$AssertTrue(suiteObj.name '= "", "Suite has a name") + do $$$AssertTrue($isobject(suiteObj.cases), "Suite has a cases array") + + set caseObj = suiteObj.cases.%Get(0) + do $$$AssertTrue($isobject(caseObj), "First case is an object") + do $$$AssertTrue($isobject(caseObj.methods), "Case has a methods array") +} + } From 887b35545d888dcc9b1f637a60488bf53c1243dd Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Apr 2026 13:33:07 -0400 Subject: [PATCH 09/18] Initial fix of verbose and quiet modes --- src/cls/IPM/ResourceProcessor/Test.cls | 28 ++++- src/cls/IPM/Test/Abstract.cls | 3 +- src/cls/IPM/Test/JUnitOutput.cls | 22 +++- src/cls/IPM/Test/JsonOutput.cls | 35 ++++-- src/cls/IPM/Test/ToonOutput.cls | 42 ++++++- src/cls/IPM/Test/YamlOutput.cls | 14 ++- .../TestResultsOPFormatAndFileGenTest.cls | 112 +++++++++++++++++- 7 files changed, 233 insertions(+), 23 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 017cc78b9..c4acc5f76 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -101,6 +101,12 @@ Method OnPhase( // In test/verify phase, run unit tests. set tVerbose = $get(pParams("Verbose"), 0) set tFlags = $select(tVerbose:"/display=all",1:"/display=none") + set explicitQuiet = ($data(pParams("Verbose")) && (pParams("Verbose") = 0)) + set capturing = 0 + if explicitQuiet { + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + set capturing = 1 + } // Ensure unit tests and related classes are loaded. set tUnitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root_..ResourceReference.Name) @@ -189,7 +195,9 @@ Method OnPhase( zkill ^UnitTestRoot $$$ThrowOnError(tSC) - set testIndex = $order(^UnitTest.Result(""),-1) + if '$data(^||%UnitTest.Manager.LastResult, testIndex)#2 { + set testIndex = $order(^UnitTest.Result(""),-1) + } if $data(pParams("outputfile"), outputFile) { set fileExtension = $zconvert($piece(outputFile,".",*),"L") set outputClass = $case(fileExtension, @@ -208,20 +216,27 @@ Method OnPhase( set tSC = $classmethod(outputClass,"ToFile",outputFile) $$$ThrowOnError(tSC) } + if capturing { + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .discarded) + set capturing = 0 + } + + set hasOutputFormat = $data(pParams("outputformat"),outputFormat) write ! - if $data(pParams("outputformat"),outputFormat)||('tVerbose) { - write !,"Test Result Summary" + if explicitQuiet || hasOutputFormat || ('tVerbose) { + if 'explicitQuiet { + write !,"Test Result Summary" + } // CLI flag takes precedence; fall back to global config, then default to Toon set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() if $get(outputFormat)="" { set outputFormat = $select(defaultOutputFormat'="":defaultOutputFormat,1:"Toon") } - set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" if '$$$defClassDefined(outputClass) { $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) } - set tSC = $classmethod(outputClass,"OutputToDevice",testIndex) + set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose) $$$ThrowOnError(tSC) write ! } @@ -234,6 +249,9 @@ Method OnPhase( write ! } } catch e { + if capturing { + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .discarded) + } set tSC = e.AsStatus() } if $data(tOldUnitTestRoot,^UnitTestRoot) // Restore ^UnitTestRoot diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index 7473b9b1d..b8f9d0b2f 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -9,7 +9,8 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + verbose As %Boolean = 0) As %Status [ Abstract ] { } diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index baa5384f6..bb7e8dec3 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -96,7 +96,8 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + verbose As %Boolean = 0) As %Status { set sc = $$$OK try { @@ -105,7 +106,24 @@ ClassMethod OutputToDevice( write !,"Methods: "_summary.methods.total_" total, "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - if summary.methods.failed > 0 { + if verbose { + set tree = ..BuildResultTree(testIndex) + write ! + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" ["_methodObj.status_"]" + if methodObj.error '= "" { write " "_methodObj.error } + } + } + } + } elseif summary.methods.failed > 0 { set tree = ..BuildResultTree(testIndex) write ! set suiteIter = tree.suites.%GetIterator() diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 94570ac84..1dc5ee0aa 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -19,31 +19,50 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + verbose As %Boolean = 0) As %Status { set sc = $$$OK try { set summary = ..GetSummary(testIndex) - set failedMethods = [] - if summary.methods.failed > 0 { - set tree = ..BuildResultTree(testIndex) + set tree = ..BuildResultTree(testIndex) + if verbose { + set allMethods = [] set suiteIter = tree.suites.%GetIterator() while suiteIter.%GetNext(, .suiteObj) { set caseIter = suiteObj.cases.%GetIterator() while caseIter.%GetNext(, .caseObj) { if caseObj.error '= "" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "error": (caseObj.error)}) + do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "status": "failed", "error": (caseObj.error)}) } set methodIter = caseObj.methods.%GetIterator() while methodIter.%GetNext(, .methodObj) { - if methodObj.status = "failed" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error), "asserts": (methodObj.asserts)}) + do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "status": (methodObj.status), "error": (methodObj.error)}) + } + } + } + set output = {"summary": (summary), "methods": (allMethods)} + } else { + set failedMethods = [] + if summary.methods.failed > 0 { + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "error": (caseObj.error)}) + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error), "asserts": (methodObj.asserts)}) + } } } } } + set output = {"summary": (summary), "failures": (failedMethods)} } - set output = {"summary": (summary), "failures": (failedMethods)} write ! do output.%ToJSON() } catch ex { diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index d9115e5b4..8f11cd23b 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -56,7 +56,8 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + verbose As %Boolean = 0) As %Status { set sc = $$$OK try { @@ -67,7 +68,44 @@ ClassMethod OutputToDevice( write !," methods["_summary.methods.total_"]: "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - if summary.methods.failed > 0 { + if verbose { + set tree = ..BuildResultTree(testIndex) + set rowCount = 0 + set rowStream = ##class(%Stream.TmpCharacter).%New() + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + set rowCount = rowCount + 1 + set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" + do rowStream.WriteLine(row) + } + if methodObj.asserts.%Size() = 0 { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_methodObj.status_",,,,""""") + } + } + } + } + write ! + write !,"results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + do rowStream.Rewind() + while 'rowStream.AtEnd { + write !,rowStream.ReadLine() + } + } elseif summary.methods.failed > 0 { set tree = ..BuildResultTree(testIndex) set failCount = 0 set failStream = ##class(%Stream.TmpCharacter).%New() diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 54c05e566..2432deca8 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -19,7 +19,8 @@ ClassMethod ToFile( } ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, + verbose As %Boolean = 0) As %Status { set sc = $$$OK try { @@ -39,7 +40,16 @@ ClassMethod OutputToDevice( write !," passed: "_summary.assertions.passed write !," failed: "_summary.assertions.failed - if summary.methods.failed > 0 { + if verbose { + write ! + write !,"results:" + set tree = ..BuildResultTree(testIndex) + set resultStream = ..YAMLResults(tree) + do resultStream.Rewind() + while 'resultStream.AtEnd { + write !,resultStream.ReadLine() + } + } elseif summary.methods.failed > 0 { write ! write !,"failures:" set failStream = ..YAMLFailures(testIndex) diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index ef65e4cf7..ef9286cb1 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -22,10 +22,18 @@ Method OnBeforeAllTests() As %Status } } set ..ReportDir = dir - // Walk back one from the current (in-progress) run to get a completed run - // with populated SQL result tables. + // Walk back from the current (in-progress) run to find a completed run that + // has at least one suite with test cases. Runs with no suites (e.g. aborted + // or setup-only runs) produce empty result trees that break tree-structure tests. set currentIndex = $order(^UnitTest.Result(""), -1) - set ..PriorRunIndex = $order(^UnitTest.Result(currentIndex), -1) + set priorIndex = $order(^UnitTest.Result(currentIndex), -1) + while priorIndex '= "" { + if $order(^UnitTest.Result(priorIndex, "")) '= "" { + quit + } + set priorIndex = $order(^UnitTest.Result(priorIndex), -1) + } + set ..PriorRunIndex = priorIndex return $$$OK } @@ -160,4 +168,102 @@ Method TestBuildResultTreeStructure() do $$$AssertTrue($isobject(caseObj.methods), "Case has a methods array") } +Method TestOutputToDeviceVerboseJson() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping") + quit + } + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + set sc = ##class(%IPM.Test.JsonOutput).OutputToDevice(testIndex, 1) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) + $$$ThrowOnError(sc) + + set jsonStr = "" + set i = "" + for { + set i = $order(captured(i)) + quit:i="" + set jsonStr = jsonStr _ captured(i) + } + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed.methods), "Verbose JSON output has 'methods' key") + do $$$AssertTrue(parsed.methods.%Size() > 0, "Verbose JSON 'methods' array is non-empty") + do $$$AssertTrue('$isobject(parsed.failures), "Verbose JSON output does not have 'failures' key") +} + +Method TestOutputToDeviceDefaultJson() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping") + quit + } + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + set sc = ##class(%IPM.Test.JsonOutput).OutputToDevice(testIndex, 0) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) + $$$ThrowOnError(sc) + + set jsonStr = "" + set i = "" + for { + set i = $order(captured(i)) + quit:i="" + set jsonStr = jsonStr _ captured(i) + } + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed.failures), "Default JSON output has 'failures' key") + do $$$AssertTrue('$isobject(parsed.methods), "Default JSON output does not have 'methods' key") +} + +Method TestOutputToDeviceVerboseYaml() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping") + quit + } + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + set sc = ##class(%IPM.Test.YamlOutput).OutputToDevice(testIndex, 1) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) + $$$ThrowOnError(sc) + + set content = "" + set i = "" + for { + set i = $order(captured(i)) + quit:i="" + set content = content _ captured(i) _ $char(10) + } + do $$$AssertTrue(content [ "results:", "Verbose YAML output contains 'results:' block") + do $$$AssertTrue('(content [ "failures:"), "Verbose YAML output does not contain 'failures:' block") +} + +Method TestOutputToDeviceDefaultYaml() +{ + set testIndex = ..PriorRunIndex + if testIndex = "" { + do $$$LogMessage("No prior test run available; skipping") + quit + } + set summary = ##class(%IPM.Test.Abstract).GetSummary(testIndex) + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + set sc = ##class(%IPM.Test.YamlOutput).OutputToDevice(testIndex, 0) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) + $$$ThrowOnError(sc) + + set content = "" + set i = "" + for { + set i = $order(captured(i)) + quit:i="" + set content = content _ captured(i) _ $char(10) + } + do $$$AssertTrue('(content [ "results:"), "Default YAML output does not contain 'results:' block") + if summary.methods.failed > 0 { + do $$$AssertTrue(content [ "failures:", "Default YAML output contains 'failures:' block when failures exist") + } +} + } From fc26ba5ee64123dc04d9104a72517dd56f22f093 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Apr 2026 13:48:34 -0400 Subject: [PATCH 10/18] Add suppress output I/O redirect --- src/cls/IPM/ResourceProcessor/Test.cls | 6 +- src/cls/IPM/Test/ToonOutput.cls | 4 +- src/cls/IPM/Test/YamlOutput.cls | 4 +- src/cls/IPM/Utils/Module.cls | 14 ++++- src/cls/IPM/Utils/OutputSuppressor.cls | 59 +++++++++++++++++++ .../TestResultsOPFormatAndFileGenTest.cls | 8 +-- 6 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 src/cls/IPM/Utils/OutputSuppressor.cls diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index c4acc5f76..2968f40d6 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -104,7 +104,7 @@ Method OnPhase( set explicitQuiet = ($data(pParams("Verbose")) && (pParams("Verbose") = 0)) set capturing = 0 if explicitQuiet { - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginSuppressOutput(.cookie)) set capturing = 1 } @@ -217,7 +217,7 @@ Method OnPhase( $$$ThrowOnError(tSC) } if capturing { - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .discarded) + $$$ThrowOnError(##class(%IPM.Utils.Module).EndSuppressOutput(cookie)) set capturing = 0 } @@ -250,7 +250,7 @@ Method OnPhase( } } catch e { if capturing { - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .discarded) + do ##class(%IPM.Utils.Module).EndSuppressOutput(cookie) } set tSC = e.AsStatus() } diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index 8f11cd23b..0d69eb924 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -42,7 +42,7 @@ ClassMethod ToFile( do fileStream.WriteLine("unitTest:") do fileStream.WriteLine(" id: "_tree.id) do fileStream.WriteLine(" namespace: "_tree.namespace) - do fileStream.WriteLine(" duration: "_tree.duration) + do fileStream.WriteLine(" duration: "_tree.duration_"s") do fileStream.WriteLine(" testDateTime: "_tree.testDateTime) do fileStream.WriteLine() do fileStream.WriteLine("results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") @@ -64,7 +64,7 @@ ClassMethod OutputToDevice( set summary = ..GetSummary(testIndex) write ! write !,"summary:" - write !," id: "_summary.id_" namespace: "_summary.namespace_" duration: "_summary.duration_" testDateTime: "_summary.testDateTime + write !," id: "_summary.id_" namespace: "_summary.namespace_" duration: "_summary.duration_"s testDateTime: "_summary.testDateTime write !," methods["_summary.methods.total_"]: "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 2432deca8..afb9c3b78 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -29,7 +29,7 @@ ClassMethod OutputToDevice( write !,"summary:" write !," id: "_summary.id write !," namespace: """_summary.namespace_"""" - write !," duration: "_summary.duration + write !," duration: "_summary.duration_"s" write !," testDateTime: """_summary.testDateTime_"""" write !," methods:" write !," total: "_summary.methods.total @@ -73,7 +73,7 @@ ClassMethod YAML(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As % do yamlStream.WriteLine("unitTest:") do yamlStream.WriteLine(" id: "_tree.id) do yamlStream.WriteLine(" namespace: """_tree.namespace_"""") - do yamlStream.WriteLine(" duration: "_tree.duration) + do yamlStream.WriteLine(" duration: "_tree.duration_"s") do yamlStream.WriteLine(" testDateTime: """_tree.testDateTime_"""") do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls index b2903e36d..6832704bf 100644 --- a/src/cls/IPM/Utils/Module.cls +++ b/src/cls/IPM/Utils/Module.cls @@ -948,7 +948,7 @@ ClassMethod GetModuleNameFromXML( /// 1 /// /// ``` -/// +/// /// Returns results as multidimensional array ClassMethod GetModuleDefaultsFromXML( pDirectory As %String, @@ -1822,6 +1822,18 @@ getPackage(itemlist,package) ; } +/// Suppresses all output written during the enclosed block (discards it). +/// Safe to call while a BeginCaptureOutput capture is already active. +ClassMethod BeginSuppressOutput(Output pCookie As %String) As %Status +{ + quit ##class(%IPM.Utils.OutputSuppressor).Begin(.pCookie) +} + +ClassMethod EndSuppressOutput(pCookie As %String) As %Status +{ + quit ##class(%IPM.Utils.OutputSuppressor).End(pCookie) +} + /// This method enables I/O redirection (see EndCaptureOutput for retrieval). pCookie has the previous I/O redirection info. ClassMethod BeginCaptureOutput(Output pCookie As %String) As %Status [ ProcedureBlock = 0 ] { diff --git a/src/cls/IPM/Utils/OutputSuppressor.cls b/src/cls/IPM/Utils/OutputSuppressor.cls new file mode 100644 index 000000000..d5b6c49f4 --- /dev/null +++ b/src/cls/IPM/Utils/OutputSuppressor.cls @@ -0,0 +1,59 @@ +/// I/O redirect routine that discards all writes. +/// Used as the mnemonic device routine for BeginSuppressOutput / EndSuppressOutput. +/// Unlike BeginCaptureOutput, suppress does not use ^||%capture, so it is safe +/// to call while a capture is already active. +Class %IPM.Utils.OutputSuppressor +{ + +ClassMethod Begin(Output pCookie As %String) As %Status [ ProcedureBlock = 0 ] +{ + new tSC,e + + #dim tSC As %Status = $$$OK + #dim e As %Exception.AbstractException + + try { + if $zutil(82,12) { + set pCookie=$zutil(96,12) + } else { + set pCookie="" + } + + use $io::("^"_$zname) + + do $zutil(82,12,1) + + } catch (e) { + set tSC=e.AsStatus() + } + quit tSC + +rstr(sz,to) [rt] public { + new rt set vr="rt" + set rd=$zutil(82,12,0) + set:$data(sz) vr=vr_"#"_sz set:$data(to) vr=vr_":"_to + read @vr + do:$data(to) $zutil(96,4,$test) + do $zutil(82,12,rd) + quit rt + } +wchr(s) public { } +wff() public { } +wnl() public { } +wstr(s) public { } +wtab(s) public { } +write(s) public { } +} + +ClassMethod End(pCookie As %String) As %Status +{ + if pCookie '= "" { + use $io::("^"_pCookie) + } else { + do $zutil(82,12,0) + use $io::("") + } + quit $$$OK +} + +} diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls index ef9286cb1..f6cf34eb0 100644 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls @@ -236,8 +236,8 @@ Method TestOutputToDeviceVerboseYaml() quit:i="" set content = content _ captured(i) _ $char(10) } - do $$$AssertTrue(content [ "results:", "Verbose YAML output contains 'results:' block") - do $$$AssertTrue('(content [ "failures:"), "Verbose YAML output does not contain 'failures:' block") + do $$$AssertTrue(content [ ($char(10)_"results:"), "Verbose YAML output contains 'results:' block") + do $$$AssertTrue('(content [ ($char(10)_"failures:")), "Verbose YAML output does not contain 'failures:' block") } Method TestOutputToDeviceDefaultYaml() @@ -260,9 +260,9 @@ Method TestOutputToDeviceDefaultYaml() quit:i="" set content = content _ captured(i) _ $char(10) } - do $$$AssertTrue('(content [ "results:"), "Default YAML output does not contain 'results:' block") + do $$$AssertTrue('(content [ ($char(10)_"results:")), "Default YAML output does not contain 'results:' block") if summary.methods.failed > 0 { - do $$$AssertTrue(content [ "failures:", "Default YAML output contains 'failures:' block when failures exist") + do $$$AssertTrue(content [ ($char(10)_"failures:"), "Default YAML output contains 'failures:' block when failures exist") } } From 1c8efd75f156fb3ea8afad28194af819bbfedd56 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Apr 2026 14:41:00 -0400 Subject: [PATCH 11/18] Fix incomplete merge --- src/cls/IPM/Test/Manager.cls | 135 +++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/src/cls/IPM/Test/Manager.cls b/src/cls/IPM/Test/Manager.cls index b9f8bbebe..d83ba01bb 100644 --- a/src/cls/IPM/Test/Manager.cls +++ b/src/cls/IPM/Test/Manager.cls @@ -62,39 +62,49 @@ ClassMethod GetAllTestsStatus( set sc = $$$OK set failureCount = 0 try { - if '$data(^||%UnitTest.Manager.LastResult,tLogIndex)#2 { - set tLogIndex = $order(^UnitTest.Result(""),-1) - } - kill ^||%UnitTest.Manager.LastResult // Clean up - if tLogIndex { - set tRes = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestAssert where Status = 0 "_ - "and TestMethod->TestCase->TestSuite->TestInstance->InstanceIndex = ?",tLogIndex) - if (tRes.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(tRes.%SQLCODE,tRes.%Message) - } - do tRes.%Next(.tSC) - $$$ThrowOnError(tSC) - set pFailureCount = tRes.%GetData(1) - if (pFailureCount > 0) { - set tSC = $$$ERROR($$$GeneralError,$$$FormatText("%1 assertion(s) failed.",pFailureCount)) - } else { - // Double check that no other failures were reported - e.g., failures loading that would lead to no assertions passing or failing! - set tRes = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestSuite where Status = 0 "_ - "and TestInstance->InstanceIndex = ?",tLogIndex) - if (tRes.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(tRes.%SQLCODE,tRes.%Message) + set testCount = $get(^||%UnitTest.Manager.AllResultsCount, 0) + + // Check tracked test LogIndexes from startIndex onwards + // This ensures nested phases only see their own results, not parent's + for i=(startIndex+1):1:testCount { + set logIndex = $get(^||%UnitTest.Manager.AllResults(i)) + if (logIndex '= "") { + // Query for assertion failures in this test run + set res = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ + "from %UnitTest_Result.TestAssert where Status = 0 "_ + "and TestMethod->TestCase->TestSuite->TestInstance->InstanceIndex = ?",logIndex) + if (res.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(res.%SQLCODE,res.%Message) } - do tRes.%Next(.tSC) - $$$ThrowOnError(tSC) - set pFailureCount = tRes.%GetData(1) - if (pFailureCount > 0) { - set tSC = $$$ERROR($$$GeneralError,$$$FormatText("%1 test suite(s) failed.",pFailureCount)) + do res.%Next(.sc) + $$$ThrowOnError(sc) + set failures = res.%GetData(1) + set failureCount = failureCount + failures + + // Also check for test suite failures (e.g., loading errors) + if (failures = 0) { + set res = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ + "from %UnitTest_Result.TestSuite where Status = 0 "_ + "and TestInstance->InstanceIndex = ?",logIndex) + if (res.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(res.%SQLCODE,res.%Message) + } + do res.%Next(.sc) + $$$ThrowOnError(sc) + set failures = res.%GetData(1) + set failureCount = failureCount + failures } } - } else { - set tSC = $$$ERROR($$$GeneralError,"No unit test results recorded.") + } + + if (failureCount > 0) { + set sc = $$$ERROR($$$GeneralError, failureCount_" assertion(s) failed.") + } + + // Only clean up AllResults at top level (startIndex=0), not in nested phases + if (startIndex = 0) { + kill ^||%UnitTest.Manager.AllResults + kill ^||%UnitTest.Manager.AllResultsCount } } catch e { set sc = e.AsStatus() @@ -108,35 +118,50 @@ ClassMethod OutputFailures(startIndex As %Integer = 0) { set sc = $$$OK try { - if '$data(^||%UnitTest.Manager.LastResult,tLogIndex)#2 { - set tLogIndex = $order(^UnitTest.Result(""),-1) - } - kill ^||%UnitTest.Manager.LastResult // Clean up - if 'tLogIndex { - quit + set testCount = $get(^||%UnitTest.Manager.AllResultsCount, 0) + + // Output failures from all tracked test LogIndexes from startIndex onwards + // This ensures parent phase outputs failures from both parent and nested tests + for i=(startIndex+1):1:testCount { + set logIndex = $get(^||%UnitTest.Manager.AllResults(i)) + if (logIndex '= "") { + do ..OutputFailuresForLogIndex(logIndex) + } } - set tLogGN = $name(^UnitTest.Result(tLogIndex)) - set tRoot = "" + + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Helper method to output failures for a single LogIndex +ClassMethod OutputFailuresForLogIndex(logIndex As %Integer) +{ + if 'logIndex { + quit + } + set logGN = $name(^UnitTest.Result(logIndex)) + set root = "" + for { + set root = $order(@logGN@(root)) + quit:root="" + set suite = "" for { - set tRoot = $order(@tLogGN@(tRoot)) - quit:tRoot="" - set tSuite = "" + set suite = $order(@logGN@(root, suite)) + quit:suite="" + set method = "" for { - set tSuite = $order(@tLogGN@(tRoot, tSuite)) - quit:tSuite="" - set tMethod = "" + set method = $order(@logGN@(root, suite, method)) + quit:method="" + + set assert = "" for { - set tMethod = $order(@tLogGN@(tRoot, tSuite, tMethod)) - quit:tMethod="" - - set tAssert = "" - for { - set tAssert = $order(@tLogGN@(tRoot, tSuite, tMethod, tAssert), 1, tAssertInfo) - quit:tAssert="" - set $listbuild(status, type, text) = tAssertInfo - continue:status - write !,$$$FormattedLine($$$Red, "FAILED " _ tSuite _ ":" _ tMethod), ": " _ type _ " - " _ text - } + set assert = $order(@logGN@(root, suite, method, assert), 1, assertInfo) + quit:assert="" + set $listbuild(status, type, text) = assertInfo + continue:status + write !,$$$FormattedLine($$$Red, "FAILED " _ suite _ ":" _ method), ": " _ type _ " - " _ text } } } From 98153d0dccfde86f854924b997f70bc086cfba97 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Apr 2026 16:07:33 -0400 Subject: [PATCH 12/18] Add integration tests --- src/cls/IPM/Test/JsonOutput.cls | 10 +- src/cls/IPM/Test/Manager.cls | 92 +++--- .../Test/PM/Integration/TestOutputFormat.cls | 264 +++++++++++++++++ .../TestResultsOPFormatAndFileGenTest.cls | 269 ------------------ 4 files changed, 317 insertions(+), 318 deletions(-) create mode 100644 tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls delete mode 100644 tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 1dc5ee0aa..52132ba57 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -55,7 +55,15 @@ ClassMethod OutputToDevice( set methodIter = caseObj.methods.%GetIterator() while methodIter.%GetNext(, .methodObj) { if methodObj.status = "failed" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error), "asserts": (methodObj.asserts)}) + if methodObj.error '= "" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error)}) + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "action": (assertObj.action), "description": (assertObj.description), "location": (assertObj.location)}) + } + } } } } diff --git a/src/cls/IPM/Test/Manager.cls b/src/cls/IPM/Test/Manager.cls index d83ba01bb..9b3cb668e 100644 --- a/src/cls/IPM/Test/Manager.cls +++ b/src/cls/IPM/Test/Manager.cls @@ -68,41 +68,40 @@ ClassMethod GetAllTestsStatus( // This ensures nested phases only see their own results, not parent's for i=(startIndex+1):1:testCount { set logIndex = $get(^||%UnitTest.Manager.AllResults(i)) - if (logIndex '= "") { - // Query for assertion failures in this test run - set res = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestAssert where Status = 0 "_ - "and TestMethod->TestCase->TestSuite->TestInstance->InstanceIndex = ?",logIndex) - if (res.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(res.%SQLCODE,res.%Message) - } - do res.%Next(.sc) - $$$ThrowOnError(sc) - set failures = res.%GetData(1) - set failureCount = failureCount + failures - - // Also check for test suite failures (e.g., loading errors) - if (failures = 0) { - set res = ##class(%SQL.Statement).%ExecDirect(,"select count(*) "_ - "from %UnitTest_Result.TestSuite where Status = 0 "_ - "and TestInstance->InstanceIndex = ?",logIndex) - if (res.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(res.%SQLCODE,res.%Message) + if logIndex '= "" { + // Count failure rows to match what OutputFailuresForLogIndex displays: + // one row per case error, one per method error, one per failed assertion + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(logIndex) + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + set failureCount = failureCount + 1 + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.error '= "" { + set failureCount = failureCount + 1 + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + set failureCount = failureCount + 1 + } + } + } } - do res.%Next(.sc) - $$$ThrowOnError(sc) - set failures = res.%GetData(1) - set failureCount = failureCount + failures } } } - if (failureCount > 0) { - set sc = $$$ERROR($$$GeneralError, failureCount_" assertion(s) failed.") + if failureCount > 0 { + set sc = $$$ERROR($$$GeneralError, failureCount_" failure(s).") } // Only clean up AllResults at top level (startIndex=0), not in nested phases - if (startIndex = 0) { + if startIndex = 0 { kill ^||%UnitTest.Manager.AllResults kill ^||%UnitTest.Manager.AllResultsCount } @@ -141,27 +140,24 @@ ClassMethod OutputFailuresForLogIndex(logIndex As %Integer) if 'logIndex { quit } - set logGN = $name(^UnitTest.Result(logIndex)) - set root = "" - for { - set root = $order(@logGN@(root)) - quit:root="" - set suite = "" - for { - set suite = $order(@logGN@(root, suite)) - quit:suite="" - set method = "" - for { - set method = $order(@logGN@(root, suite, method)) - quit:method="" - - set assert = "" - for { - set assert = $order(@logGN@(root, suite, method, assert), 1, assertInfo) - quit:assert="" - set $listbuild(status, type, text) = assertInfo - continue:status - write !,$$$FormattedLine($$$Red, "FAILED " _ suite _ ":" _ method), ": " _ type _ " - " _ text + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(logIndex) + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ caseObj.name), ": " _ caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status '= "failed" { continue } + if methodObj.error '= "" { + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ methodObj.name), ": " _ methodObj.error + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status '= "failed" { continue } + write !,$$$FormattedLine($$$Red, "FAILED " _ suiteObj.name _ ":" _ methodObj.name), ": " _ assertObj.action _ " - " _ assertObj.description } } } diff --git a/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls new file mode 100644 index 000000000..19857dbeb --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls @@ -0,0 +1,264 @@ +Class Test.PM.Integration.TestOutputFormat Extends Test.PM.Integration.Base +{ + +Method OnBeforeAllTests() As %Status +{ + set sc = ##class(%IPM.Main).Shell("load " _ ..GetModuleDir("test-output-format")) + do $$$AssertStatusOK(sc, "Loaded test-output-format module") + return sc +} + +Method OnAfterAllTests() As %Status +{ + do ##class(%IPM.Main).Shell("uninstall test-output-format") + return $$$OK +} + +/// Helper: join captured output array into a single string +ClassMethod JoinOutput(ByRef pOutput) As %String [ Private ] +{ + set result = "" + set i = "" + for { + set i = $order(pOutput(i)) + quit:i="" + set result = result _ pOutput(i) _ $char(10) + } + return result +} + + +/// Default (non-verbose) output: summary + failures section, no results section. +/// Expects 12 failed methods and 13 failure rows. +Method TestDefaultOutput() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") + do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows") + do $$$AssertNotTrue(content [ "results[", "no results section in default mode") +} + +/// Verbose output requires -f to route through OutputToDevice. +/// With -verbose -f toon: results section with all methods, no failures section. +Method TestVerboseOutput() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f toon") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") + do $$$AssertTrue(content [ "results[", "results section present in verbose mode") + do $$$AssertNotTrue(content [ "failures[", "no failures section in verbose mode") +} + +/// Quiet output: suppresses test runner noise, still shows summary and failures. +/// Known gap: two pre-phase lines ([USER|ZPM] Test START, Building dependency graph) still escape suppression. +Method TestQuietOutput() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -quiet") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods in quiet mode") + do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows in quiet mode") +} + +/// -f json: summary + failures as JSON object (not verbose, so failures key not methods). +Method TestOutputFormatJson() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f json") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + + // Extract just the JSON object from the output: first "{" to last "}" + // OutputFailures writes "FAILED ..." lines after the JSON, so we must not include them + set raw = ..JoinOutput(.output) + set jsonStart = $find(raw, "{") - 1 + set jsonEnd = $length(raw) - $find($reverse(raw), "}") + 2 + set jsonStr = $extract(raw, jsonStart, jsonEnd) + + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed), "JSON output is parseable") + do $$$AssertTrue($isobject(parsed.failures), "JSON output has failures array") + do $$$AssertEquals(parsed.failures.%Size(), 13, "JSON failures array has 13 entries") + do $$$AssertNotTrue($isobject(parsed.methods), "JSON output does not have methods array in default mode") +} + +/// -verbose -f json: methods array with all methods, no failures array. +Method TestVerboseOutputFormatJson() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f json") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + + set raw = ..JoinOutput(.output) + set jsonStart = $find(raw, "{") - 1 + set jsonEnd = $length(raw) - $find($reverse(raw), "}") + 2 + set jsonStr = $extract(raw, jsonStart, jsonEnd) + + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed), "Verbose JSON output is parseable") + do $$$AssertTrue($isobject(parsed.methods), "Verbose JSON output has methods array") + do $$$AssertTrue(parsed.methods.%Size() > 0, "Verbose JSON methods array is non-empty") + do $$$AssertNotTrue($isobject(parsed.failures), "Verbose JSON output does not have failures array") +} + +/// -f yaml: summary + failures block (not verbose). +/// YAML summary uses nested "failed: N" structure, not inline "N failed". +Method TestOutputFormatYaml() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f yaml") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "failed: 12", "YAML summary reports 12 failed methods") + do $$$AssertTrue(content [ ($char(10)_"failures:"), "YAML output has failures block") + do $$$AssertNotTrue(content [ ($char(10)_"results:"), "YAML output has no results block in default mode") +} + +/// -verbose -f yaml: results block present, no failures block. +Method TestVerboseOutputFormatYaml() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f yaml") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ ($char(10)_"results:"), "Verbose YAML output has results block") + do $$$AssertNotTrue(content [ ($char(10)_"failures:"), "Verbose YAML output has no failures block") +} + +/// -output-file .toon: ToFile always writes all rows under results[N], not failures-only. +Method TestOutputFileToon() +{ + set tmpFile = ##class(%Library.File).TempFilename() _ ".toon" + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "Toon output file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(tmpFile)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) + } + do $$$AssertTrue(content [ "unitTest:", "Toon file has unitTest header") + do $$$AssertTrue(content [ "results[", "Toon file has results header") + do $$$AssertTrue(content [ "deliberate failure", "Toon file contains known failure text") + do ##class(%File).Delete(tmpFile) +} + +/// -output-file .yaml: file contains unitTest header and results section. +Method TestOutputFileYaml() +{ + set tmpFile = ##class(%Library.File).TempFilename() _ ".yaml" + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "YAML output file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(tmpFile)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) + } + do $$$AssertTrue(content [ "unitTest:", "YAML file has unitTest header") + do $$$AssertTrue(content [ "id: ", "YAML file has id field") + do $$$AssertTrue(content [ "namespace:", "YAML file has namespace field") + do $$$AssertTrue(content [ "results:", "YAML file has results section") + do ##class(%File).Delete(tmpFile) +} + +/// -output-file .json: file is valid JSON with expected structure. +Method TestOutputFileJson() +{ + set tmpFile = ##class(%Library.File).TempFilename() _ ".json" + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "JSON output file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(tmpFile)) + set parsed = ##class(%DynamicAbstractObject).%FromJSON(fileStream) + do $$$AssertTrue($isobject(parsed), "JSON file is parseable") + do $$$AssertTrue(parsed.id '= "", "JSON file has id field") + do $$$AssertTrue(parsed.namespace '= "", "JSON file has namespace field") + do $$$AssertTrue($isobject(parsed.suites), "JSON file has suites array") + do ##class(%File).Delete(tmpFile) +} + +/// -output-file .xml: file is valid JUnit XML. +Method TestOutputFileXml() +{ + set tmpFile = ##class(%Library.File).TempFilename() _ ".xml" + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "XML output file is non-empty") + + set fileStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(fileStream.LinkToFile(tmpFile)) + set content = "" + while 'fileStream.AtEnd { + set content = content _ fileStream.ReadLine() _ $char(10) + } + do $$$AssertTrue(content [ "", "XML file has root element") + do $$$AssertTrue(content [ "", "XML file has closing ") + do ##class(%File).Delete(tmpFile) +} + +/// GetSummary returns correct counts for the test-output-format module run. +Method TestGetSummary() +{ + set sc = ##class(%IPM.Main).Shell("test test-output-format -only") + set testIndex = $order(^UnitTest.Result(""), -1) + + set summary = ##class(%IPM.Test.Abstract).GetSummary(testIndex) + do $$$AssertTrue($isobject(summary), "GetSummary returns an object") + do $$$AssertTrue($isobject(summary.methods), "Summary has a methods sub-object") + do $$$AssertTrue($isobject(summary.assertions), "Summary has an assertions sub-object") + do $$$AssertEquals(summary.methods.failed, 12, "Summary reports 12 failed methods") + do $$$AssertEquals(summary.methods.passed, 6, "Summary reports 6 passed methods") + do $$$AssertEquals(summary.methods.passed + summary.methods.failed, summary.methods.total, "Method counts add up") + do $$$AssertEquals(summary.assertions.passed + summary.assertions.failed, summary.assertions.total, "Assertion counts add up") +} + +/// BuildResultTree returns correct structure for the test-output-format module run. +Method TestBuildResultTree() +{ + set sc = ##class(%IPM.Main).Shell("test test-output-format -only") + set testIndex = $order(^UnitTest.Result(""), -1) + + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(testIndex) + do $$$AssertTrue($isobject(tree), "BuildResultTree returns an object") + do $$$AssertTrue(tree.id '= "", "Tree has an id") + do $$$AssertTrue(tree.namespace '= "", "Tree has a namespace") + do $$$AssertTrue($isobject(tree.suites), "Tree has a suites array") + do $$$AssertTrue(tree.suites.%Size() > 0, "Tree has at least one suite") + + set suiteObj = tree.suites.%Get(0) + do $$$AssertTrue($isobject(suiteObj), "First suite is an object") + do $$$AssertTrue(suiteObj.name '= "", "Suite has a name") + do $$$AssertTrue($isobject(suiteObj.cases), "Suite has a cases array") + + set caseObj = suiteObj.cases.%Get(0) + do $$$AssertTrue($isobject(caseObj), "First case is an object") + do $$$AssertTrue($isobject(caseObj.methods), "Case has a methods array") +} + +} diff --git a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls b/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls deleted file mode 100644 index f6cf34eb0..000000000 --- a/tests/unit_tests/Test/PM/Unit/TestResultsOPFormatAndFileGenTest.cls +++ /dev/null @@ -1,269 +0,0 @@ -/// Unit tests for the ZPM test output format classes. -/// Verifies that each format produces structurally valid, non-trivial output -/// against the most recent test run at the time the tests execute. -Class Test.PM.Unit.TestResultsOPFormatAndFileGenTest Extends %UnitTest.TestCase -{ - -/// Temp directory for generated report files, created in OnBeforeAllTests. -Property ReportDir As %String(MAXLEN = 512); - -/// Index of the most recent completed run before this test class starts. -/// Captured in OnBeforeAllTests so TestGetSummaryHasCounts is not sensitive -/// to which test method runs first. -Property PriorRunIndex As %Integer; - -Method OnBeforeAllTests() As %Status -{ - set dir = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)_"/test-reports/") - if '##class(%File).DirectoryExists(dir) { - set sc = ##class(%File).CreateDirectoryChain(dir) - if $$$ISERR(sc) { - return sc - } - } - set ..ReportDir = dir - // Walk back from the current (in-progress) run to find a completed run that - // has at least one suite with test cases. Runs with no suites (e.g. aborted - // or setup-only runs) produce empty result trees that break tree-structure tests. - set currentIndex = $order(^UnitTest.Result(""), -1) - set priorIndex = $order(^UnitTest.Result(currentIndex), -1) - while priorIndex '= "" { - if $order(^UnitTest.Result(priorIndex, "")) '= "" { - quit - } - set priorIndex = $order(^UnitTest.Result(priorIndex), -1) - } - set ..PriorRunIndex = priorIndex - return $$$OK -} - -Method OnAfterAllTests() As %Status -{ - if ##class(%File).DirectoryExists(..ReportDir) { - do ##class(%File).RemoveDirectoryTree(..ReportDir) - } - return $$$OK -} - -Method TestJsonOutputStructure() -{ - set testIndex = $order(^UnitTest.Result(""), -1) - set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.json") - - set status = ##class(%IPM.Test.JsonOutput).ToFile(fileName, testIndex) - do $$$AssertStatusOK(status, "JsonOutput.ToFile succeeded") - do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JSON file is non-empty") - - set fileStream = ##class(%Stream.FileCharacter).%New() - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - set parsed = ##class(%DynamicAbstractObject).%FromJSON(fileStream) - do $$$AssertTrue($isobject(parsed), "JSON output is parseable") - do $$$AssertTrue(parsed.id '= "", "JSON output has an id field") - do $$$AssertTrue(parsed.namespace '= "", "JSON output has a namespace field") - do $$$AssertTrue($isobject(parsed.suites), "JSON output has a suites array") -} - -Method TestYamlOutputStructure() -{ - set testIndex = $order(^UnitTest.Result(""), -1) - set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.yaml") - - set status = ##class(%IPM.Test.YamlOutput).ToFile(fileName, testIndex) - do $$$AssertStatusOK(status, "YamlOutput.ToFile succeeded") - do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "YAML file is non-empty") - - set fileStream = ##class(%Stream.FileCharacter).%New() - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - set content = "" - while 'fileStream.AtEnd { - set content = content _ fileStream.ReadLine() _ $char(10) - } - do $$$AssertTrue(content [ "unitTest:", "YAML output has unitTest header") - do $$$AssertTrue(content [ "id: ", "YAML output has id field") - do $$$AssertTrue(content [ "namespace:", "YAML output has namespace field") - do $$$AssertTrue(content [ "results:", "YAML output has results section") -} - -Method TestToonOutputStructure() -{ - set testIndex = $order(^UnitTest.Result(""), -1) - set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.toon") - - set status = ##class(%IPM.Test.ToonOutput).ToFile(fileName, testIndex) - do $$$AssertStatusOK(status, "ToonOutput.ToFile succeeded") - do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "Toon file is non-empty") - - set fileStream = ##class(%Stream.FileCharacter).%New() - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - set content = "" - while 'fileStream.AtEnd { - set content = content _ fileStream.ReadLine() _ $char(10) - } - do $$$AssertTrue(content [ "unitTest:", "Toon output has unitTest header") - do $$$AssertTrue(content [ "results[", "Toon output has results header with row count") - do $$$AssertTrue(content [ "{suiteName,", "Toon output has column header line") -} - -Method TestJUnitOutputStructure() -{ - set testIndex = $order(^UnitTest.Result(""), -1) - set fileName = ##class(%File).NormalizeFilename(..ReportDir_"test.xml") - - set status = ##class(%IPM.Test.JUnitOutput).ToFile(fileName, testIndex) - do $$$AssertStatusOK(status, "JUnitOutput.ToFile succeeded") - do $$$AssertTrue(##class(%File).GetFileSize(fileName) > 0, "JUnit XML file is non-empty") - - set fileStream = ##class(%Stream.FileCharacter).%New() - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - set content = "" - while 'fileStream.AtEnd { - set content = content _ fileStream.ReadLine() _ $char(10) - } - do $$$AssertTrue(content [ "", "JUnit output has root element") - do $$$AssertTrue(content [ " element") - do $$$AssertTrue(content [ "", "JUnit output has closing element") -} - -Method TestGetSummaryHasCounts() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping GetSummary count check") - quit - } - - set summary = ##class(%IPM.Test.Abstract).GetSummary(testIndex) - do $$$AssertTrue($isobject(summary), "GetSummary returns an object") - do $$$AssertTrue($isobject(summary.methods), "Summary has a methods sub-object") - do $$$AssertTrue($isobject(summary.assertions), "Summary has an assertions sub-object") - do $$$AssertTrue(summary.methods.total > 0, "Summary reports at least one method") - do $$$AssertTrue(summary.assertions.total > 0, "Summary reports at least one assertion") - do $$$AssertEquals(summary.methods.passed + summary.methods.failed, summary.methods.total, "Method counts add up") - do $$$AssertEquals(summary.assertions.passed + summary.assertions.failed, summary.assertions.total, "Assertion counts add up") -} - -Method TestBuildResultTreeStructure() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping tree structure check") - quit - } - - set tree = ##class(%IPM.Test.Abstract).BuildResultTree(testIndex) - do $$$AssertTrue($isobject(tree), "BuildResultTree returns an object") - do $$$AssertTrue(tree.id '= "", "Tree has an id") - do $$$AssertTrue(tree.namespace '= "", "Tree has a namespace") - do $$$AssertTrue($isobject(tree.suites), "Tree has a suites array") - do $$$AssertTrue(tree.suites.%Size() > 0, "Tree has at least one suite") - - set suiteObj = tree.suites.%Get(0) - do $$$AssertTrue($isobject(suiteObj), "First suite is an object") - do $$$AssertTrue(suiteObj.name '= "", "Suite has a name") - do $$$AssertTrue($isobject(suiteObj.cases), "Suite has a cases array") - - set caseObj = suiteObj.cases.%Get(0) - do $$$AssertTrue($isobject(caseObj), "First case is an object") - do $$$AssertTrue($isobject(caseObj.methods), "Case has a methods array") -} - -Method TestOutputToDeviceVerboseJson() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping") - quit - } - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) - set sc = ##class(%IPM.Test.JsonOutput).OutputToDevice(testIndex, 1) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) - $$$ThrowOnError(sc) - - set jsonStr = "" - set i = "" - for { - set i = $order(captured(i)) - quit:i="" - set jsonStr = jsonStr _ captured(i) - } - set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) - do $$$AssertTrue($isobject(parsed.methods), "Verbose JSON output has 'methods' key") - do $$$AssertTrue(parsed.methods.%Size() > 0, "Verbose JSON 'methods' array is non-empty") - do $$$AssertTrue('$isobject(parsed.failures), "Verbose JSON output does not have 'failures' key") -} - -Method TestOutputToDeviceDefaultJson() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping") - quit - } - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) - set sc = ##class(%IPM.Test.JsonOutput).OutputToDevice(testIndex, 0) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) - $$$ThrowOnError(sc) - - set jsonStr = "" - set i = "" - for { - set i = $order(captured(i)) - quit:i="" - set jsonStr = jsonStr _ captured(i) - } - set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) - do $$$AssertTrue($isobject(parsed.failures), "Default JSON output has 'failures' key") - do $$$AssertTrue('$isobject(parsed.methods), "Default JSON output does not have 'methods' key") -} - -Method TestOutputToDeviceVerboseYaml() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping") - quit - } - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) - set sc = ##class(%IPM.Test.YamlOutput).OutputToDevice(testIndex, 1) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) - $$$ThrowOnError(sc) - - set content = "" - set i = "" - for { - set i = $order(captured(i)) - quit:i="" - set content = content _ captured(i) _ $char(10) - } - do $$$AssertTrue(content [ ($char(10)_"results:"), "Verbose YAML output contains 'results:' block") - do $$$AssertTrue('(content [ ($char(10)_"failures:")), "Verbose YAML output does not contain 'failures:' block") -} - -Method TestOutputToDeviceDefaultYaml() -{ - set testIndex = ..PriorRunIndex - if testIndex = "" { - do $$$LogMessage("No prior test run available; skipping") - quit - } - set summary = ##class(%IPM.Test.Abstract).GetSummary(testIndex) - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie)) - set sc = ##class(%IPM.Test.YamlOutput).OutputToDevice(testIndex, 0) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .captured) - $$$ThrowOnError(sc) - - set content = "" - set i = "" - for { - set i = $order(captured(i)) - quit:i="" - set content = content _ captured(i) _ $char(10) - } - do $$$AssertTrue('(content [ ($char(10)_"results:")), "Default YAML output does not contain 'results:' block") - if summary.methods.failed > 0 { - do $$$AssertTrue(content [ ($char(10)_"failures:"), "Default YAML output contains 'failures:' block when failures exist") - } -} - -} From 54e62c43df0ca581efa54f00a4aa8f142d759ecd Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Apr 2026 16:33:10 -0400 Subject: [PATCH 13/18] Make quiet flag work for verify and improve CLI docs --- src/cls/IPM/Lifecycle/Base.cls | 19 +++++++++++++++++++ src/cls/IPM/Main.cls | 8 ++++---- src/cls/IPM/ResourceProcessor/PythonWheel.cls | 9 ++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 74bade7c7..7071add8a 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -1307,10 +1307,20 @@ Method %Package(ByRef pParams) As %Status [ Abstract ] Method %Verify(ByRef pParams) As %Status { set tSC = $$$OK + set capturing = 0 try { new $namespace set tInitNS = $select($namespace="%SYS": "USER", 1: $namespace) set tVerbose = $get(pParams("Verbose")) + set explicitQuiet = ($data(pParams("Verbose")) && (pParams("Verbose") = 0)) + + // Suppress module install/load noise in quiet mode; ended before resource processors + // run so each processor can manage its own output (Test.cls still shows summary + failures). + // BeginSuppressOutput is device-level and survives the namespace switch below. + if explicitQuiet { + $$$ThrowOnError(##class(%IPM.Utils.Module).BeginSuppressOutput(.tVerifyCookie)) + set capturing = 1 + } if '$get(pParams("Verify","InCurrentNamespace"),0) { set tMustCreate = 1 @@ -1394,6 +1404,12 @@ Method %Verify(ByRef pParams) As %Status // Load dependencies scoped to the "Verify" phase that are not yet installed. do ##class(%IPM.Utils.Module).LoadDependencies(..Module, ..PhaseList, .pParams) + // End suppression before resource processors run — each handles its own output. + if capturing { + $$$ThrowOnError(##class(%IPM.Utils.Module).EndSuppressOutput(tVerifyCookie)) + set capturing = 0 + } + set orderedResourceList = ..Module.GetOrderedResourceList() set tKey = "" for { @@ -1407,6 +1423,9 @@ Method %Verify(ByRef pParams) As %Status } } } catch e { + if capturing { + do ##class(%IPM.Utils.Module).EndSuppressOutput(tVerifyCookie) + } set tSC = e.AsStatus() } quit tSC diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 3b0a9a376..88c277dee 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -73,10 +73,10 @@ Can also specify desired version to update to. - - - - + + + + diff --git a/src/cls/IPM/ResourceProcessor/PythonWheel.cls b/src/cls/IPM/ResourceProcessor/PythonWheel.cls index a1346cc9b..0cb67a5a4 100644 --- a/src/cls/IPM/ResourceProcessor/PythonWheel.cls +++ b/src/cls/IPM/ResourceProcessor/PythonWheel.cls @@ -52,7 +52,14 @@ Method OnPhase( if verbose { write !,"Running command: ",command } - $$$ThrowOnError(##class(%IPM.Utils.Module).RunCommand(, command)) + set explicitQuiet = ($data(pParams("Verbose")) && (pParams("Verbose") = 0)) + if explicitQuiet { + // Redirect pip stdout/stderr to a sink stream so subprocess output doesn't bypass device suppression + set pipOutput = ##class(%Stream.TmpCharacter).%New() + $$$ThrowOnError(##class(%IPM.Utils.Module).RunCommand(, command, .pipOutput, .pipOutput)) + } else { + $$$ThrowOnError(##class(%IPM.Utils.Module).RunCommand(, command)) + } } catch ex { set pResourceHandled = 0 // Special case: we want the installation of IPM to continue, even if the wheel package fails to install From ca7cce40554bb0a88eb8d5a3f6595cf345239395 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Apr 2026 10:50:49 -0400 Subject: [PATCH 14/18] Refactor UX and flags --- CHANGELOG.md | 2 +- src/cls/IPM/Main.cls | 4 +- src/cls/IPM/Repo/UniversalSettings.cls | 8 +- src/cls/IPM/ResourceProcessor/Test.cls | 41 +++--- src/cls/IPM/Test/Abstract.cls | 18 ++- src/cls/IPM/Test/JsonOutput.cls | 11 +- src/cls/IPM/Test/Manager.cls | 13 +- src/cls/IPM/Test/ToonOutput.cls | 137 +++++++----------- src/cls/IPM/Test/YamlOutput.cls | 26 ++-- src/cls/IPM/Utils/OutputSuppressor.cls | 25 ++-- .../Test/PM/Integration/TestOutputFormat.cls | 92 +++++++++--- 11 files changed, 204 insertions(+), 173 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d22caad7a..aa531f67b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #992: Implement automatic history purge logic - #973: Enables CORS and JWT configuration for WebApplications in module.xml -- #971: Adds support for JSON, YAML, and Toon formats via the -output-format/-f flag for outputting into the terminal and the -output-file flag for outputting to a file - #1110: Add `iriscli` and `ipm` container utility scripts that are auto-installed to `~/.local/bin/` and `~/bin/` so they work both inside and outside of containers (Unix/Linux only) +- #971: Adds support for JSON, YAML, and Toon formats via the -output-format/-f flag for outputting into the terminal, adds the -output-file flag for outputting to a file, and improves the -quiet flag to suppress most output ### Fixed - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 88c277dee..8bd8dfdd7 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -74,8 +74,8 @@ Can also specify desired version to update to. - - + + diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls index f644c3dc7..8c0999896 100644 --- a/src/cls/IPM/Repo/UniversalSettings.cls +++ b/src/cls/IPM/Repo/UniversalSettings.cls @@ -1,6 +1,6 @@ /// IPM settings are placed in ^IPM.settings global in %SYS namespace /// Use this class to set or get settings -/// +/// /// Available settings /// default_registry (string) - default registry url /// analytics_tracking_id @@ -197,12 +197,16 @@ ClassMethod SetTestReportFormat( val As %String, overwrite As %Boolean = 1) As %Boolean { + // Validate format value + if val '= "" && ('$listfind($listfromstring(##class(%IPM.Test.Abstract).#VALIDFORMATS), $zconvert(val, "l"))) { + return 0 + } return ..SetValue(..#TestReportFormat, val, overwrite) } ClassMethod GetTestReportFormat() As %String { - return ..GetValue(..#TestReportFormat) + return ..GetValue(..#TestReportFormat) } } diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 3ae221a3d..5a169ca50 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -232,28 +232,29 @@ Method OnPhase( set capturing = 0 } - set hasOutputFormat = $data(pParams("outputformat"),outputFormat) - write ! - if explicitQuiet || hasOutputFormat || ('tVerbose) { - if 'explicitQuiet { - write !,"Test Result Summary" - } - // CLI flag takes precedence; fall back to global config, then default to Toon - set defaultOutputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() - if $get(outputFormat)="" { - set outputFormat = $select(defaultOutputFormat'="":defaultOutputFormat,1:"Toon") - } - set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" - if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) - } - set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose) - $$$ThrowOnError(tSC) - write ! + set hasOutputFormat = $data(pParams("outputformat")) + write !!,"Test Result Summary" + // -f enables the failures/results detail section. The format is driven by + // 'config set TestReportFormat'; falls back to Toon if not configured. + // Without -f only the summary line is shown, matching pre-existing behavior. + set showDetail = hasOutputFormat + set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + if outputFormat = "" { + set outputFormat = "Toon" } - // By default, detect and report unit test failures as an error from this phase + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) + } + set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose,showDetail) + $$$ThrowOnError(tSC) + write ! + // By default, detect and report unit test failures as an error from this phase. + // OutputFailures is skipped when detail is shown since the formatter already includes failures. if $get(pParams("UnitTest","FailuresAreFatal"),1) { - do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) + if 'showDetail { + do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) + } set tSC = ##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex) $$$ThrowOnError(tSC) } diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index b8f9d0b2f..8ac45574f 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -2,15 +2,22 @@ Class %IPM.Test.Abstract Extends %RegisteredObject { +/// Comma-separated list of valid output format names (lowercase). +/// Must stay in sync with the valueList on the output-format modifier in %IPM.Main. +Parameter VALIDFORMATS = "json,yaml,toon"; + ClassMethod ToFile( fileName As %String, testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] { } +/// verbose: show all methods (not just failures) in the detail section. +/// showDetail: show the results/failures section at all; when 0 only the summary is written. ClassMethod OutputToDevice( testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - verbose As %Boolean = 0) As %Status [ Abstract ] + verbose As %Boolean = 0, + showDetail As %Boolean = 1) As %Status [ Abstract ] { } @@ -122,9 +129,12 @@ ClassMethod BuildResultTree(testIndex As %Integer = {$order(^UnitTest.Result("") /// Returns a summary %DynamicObject with run metadata and method/assertion counts. /// Counts methods as failed if any assert failed OR if a non-assertion error occurred. -ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %DynamicObject +/// Pass a pre-built tree to avoid a second ^UnitTest.Result traversal. +ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, tree As %DynamicObject = "") As %DynamicObject { - set tree = ..BuildResultTree(testIndex) + if tree = "" { + set tree = ..BuildResultTree(testIndex) + } set (assertTotal, assertPassed, assertFailed) = 0 set (methodTotal, methodPassed, methodFailed) = 0 @@ -150,7 +160,7 @@ ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)} } } } - // Case-level error (e.g. OnBeforeAllTests failure) — no methods ran + // Case-level error from a lifecycle hook (OnBeforeAllTests, OnAfterAllTests) — count it as one failed method if caseObj.error '= "" { set methodTotal = methodTotal + 1 set methodFailed = methodFailed + 1 diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 52132ba57..7bbf36d3f 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -20,13 +20,14 @@ ClassMethod ToFile( ClassMethod OutputToDevice( testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - verbose As %Boolean = 0) As %Status + verbose As %Boolean = 0, + showDetail As %Boolean = 1) As %Status { set sc = $$$OK try { - set summary = ..GetSummary(testIndex) set tree = ..BuildResultTree(testIndex) - if verbose { + set summary = ..GetSummary(testIndex, tree) + if showDetail && verbose { set allMethods = [] set suiteIter = tree.suites.%GetIterator() while suiteIter.%GetNext(, .suiteObj) { @@ -42,7 +43,7 @@ ClassMethod OutputToDevice( } } set output = {"summary": (summary), "methods": (allMethods)} - } else { + } elseif showDetail { set failedMethods = [] if summary.methods.failed > 0 { set suiteIter = tree.suites.%GetIterator() @@ -70,6 +71,8 @@ ClassMethod OutputToDevice( } } set output = {"summary": (summary), "failures": (failedMethods)} + } else { + set output = {"summary": (summary)} } write ! do output.%ToJSON() diff --git a/src/cls/IPM/Test/Manager.cls b/src/cls/IPM/Test/Manager.cls index 9b3cb668e..5957bc949 100644 --- a/src/cls/IPM/Test/Manager.cls +++ b/src/cls/IPM/Test/Manager.cls @@ -100,10 +100,19 @@ ClassMethod GetAllTestsStatus( set sc = $$$ERROR($$$GeneralError, failureCount_" failure(s).") } - // Only clean up AllResults at top level (startIndex=0), not in nested phases + // Always consume the entries this phase owned. + // At top level (startIndex=0) kill everything; for nested phases, roll the count + // back so the outer phase does not re-count entries the inner phase already handled. + // Outer phases learn about inner failures via the Shell return status, not by + // re-inspecting these entries. if startIndex = 0 { kill ^||%UnitTest.Manager.AllResults kill ^||%UnitTest.Manager.AllResultsCount + } else { + for i=(startIndex+1):1:testCount { + kill ^||%UnitTest.Manager.AllResults(i) + } + set ^||%UnitTest.Manager.AllResultsCount = startIndex } } catch e { set sc = e.AsStatus() @@ -119,8 +128,6 @@ ClassMethod OutputFailures(startIndex As %Integer = 0) try { set testCount = $get(^||%UnitTest.Manager.AllResultsCount, 0) - // Output failures from all tracked test LogIndexes from startIndex onwards - // This ensures parent phase outputs failures from both parent and nested tests for i=(startIndex+1):1:testCount { set logIndex = $get(^||%UnitTest.Manager.AllResults(i)) if (logIndex '= "") { diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index 0d69eb924..a07adeebb 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -12,32 +12,7 @@ ClassMethod ToFile( $$$ThrowOnError(fileStream.LinkToFile(fileName)) set tree = ..BuildResultTree(testIndex) - set rowStream = ##class(%Stream.TmpCharacter).%New() - set rowCount = 0 - - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if methodObj.error '= "" { - set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") - } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - set rowCount = rowCount + 1 - set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" - do rowStream.WriteLine(row) - } - } - } - } + set rowStream = ..BuildToonRows(tree, 0, .rowCount) do fileStream.WriteLine("unitTest:") do fileStream.WriteLine(" id: "_tree.id) @@ -57,83 +32,29 @@ ClassMethod ToFile( ClassMethod OutputToDevice( testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - verbose As %Boolean = 0) As %Status + verbose As %Boolean = 0, + showDetail As %Boolean = 1) As %Status { set sc = $$$OK try { - set summary = ..GetSummary(testIndex) + set tree = ..BuildResultTree(testIndex) + set summary = ..GetSummary(testIndex, tree) write ! write !,"summary:" write !," id: "_summary.id_" namespace: "_summary.namespace_" duration: "_summary.duration_"s testDateTime: "_summary.testDateTime write !," methods["_summary.methods.total_"]: "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - if verbose { - set tree = ..BuildResultTree(testIndex) - set rowCount = 0 - set rowStream = ##class(%Stream.TmpCharacter).%New() - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if methodObj.error '= "" { - set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") - } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - set rowCount = rowCount + 1 - set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" - do rowStream.WriteLine(row) - } - if methodObj.asserts.%Size() = 0 { - set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_methodObj.status_",,,,""""") - } - } - } - } + if showDetail && verbose { + set rowStream = ..BuildToonRows(tree, 0, .rowCount) write ! write !,"results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" do rowStream.Rewind() while 'rowStream.AtEnd { write !,rowStream.ReadLine() } - } elseif summary.methods.failed > 0 { - set tree = ..BuildResultTree(testIndex) - set failCount = 0 - set failStream = ##class(%Stream.TmpCharacter).%New() - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - set failCount = failCount + 1 - do failStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if methodObj.error '= "" { - set failCount = failCount + 1 - do failStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") - } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - if assertObj.status = "failed" { - set failCount = failCount + 1 - set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,"_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" - do failStream.WriteLine(row) - } - } - } - } - } + } elseif showDetail && (summary.methods.failed > 0) { + set failStream = ..BuildToonRows(tree, 1, .failCount) write ! write !,"failures["_failCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" do failStream.Rewind() @@ -147,4 +68,44 @@ ClassMethod OutputToDevice( return sc } +/// Returns a %Stream.TmpCharacter containing Toon result rows. +/// failuresOnly=0: all methods (for ToFile and verbose OutputToDevice). +/// failuresOnly=1: failed methods and assert failures only (for non-verbose OutputToDevice). +ClassMethod BuildToonRows(tree As %DynamicObject, failuresOnly As %Boolean, Output rowCount As %Integer) As %Stream.TmpCharacter [ Private ] +{ + set rowStream = ##class(%Stream.TmpCharacter).%New() + set rowCount = 0 + + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if failuresOnly && (methodObj.status '= "failed") { continue } + if methodObj.error '= "" { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if failuresOnly && (assertObj.status '= "failed") { continue } + set rowCount = rowCount + 1 + set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" + do rowStream.WriteLine(row) + } + if 'failuresOnly && (methodObj.asserts.%Size() = 0) { + set rowCount = rowCount + 1 + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_methodObj.status_",,,,""""") + } + } + } + } + return rowStream +} + } diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index afb9c3b78..8f52b524c 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -10,7 +10,7 @@ ClassMethod ToFile( set fileStream = ##class(%Stream.FileCharacter).%New() set fileStream.TranslateTable = "UTF8" $$$ThrowOnError(fileStream.LinkToFile(fileName)) - do fileStream.CopyFrom(..YAML(testIndex)) + do fileStream.CopyFrom(..BuildYaml(testIndex)) $$$ThrowOnError(fileStream.%Save()) } catch ex { set sc = ex.AsStatus() @@ -20,11 +20,13 @@ ClassMethod ToFile( ClassMethod OutputToDevice( testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - verbose As %Boolean = 0) As %Status + verbose As %Boolean = 0, + showDetail As %Boolean = 1) As %Status { set sc = $$$OK try { - set summary = ..GetSummary(testIndex) + set tree = ..BuildResultTree(testIndex) + set summary = ..GetSummary(testIndex, tree) write ! write !,"summary:" write !," id: "_summary.id @@ -40,19 +42,18 @@ ClassMethod OutputToDevice( write !," passed: "_summary.assertions.passed write !," failed: "_summary.assertions.failed - if verbose { + if showDetail && verbose { write ! write !,"results:" - set tree = ..BuildResultTree(testIndex) - set resultStream = ..YAMLResults(tree) + set resultStream = ..BuildYamlResults(tree) do resultStream.Rewind() while 'resultStream.AtEnd { write !,resultStream.ReadLine() } - } elseif summary.methods.failed > 0 { + } elseif showDetail && (summary.methods.failed > 0) { write ! write !,"failures:" - set failStream = ..YAMLFailures(testIndex) + set failStream = ..BuildYamlFailures(tree) do failStream.Rewind() while 'failStream.AtEnd { write !,failStream.ReadLine() @@ -65,7 +66,7 @@ ClassMethod OutputToDevice( } /// Returns a stream with the full YAML document (header + all results). -ClassMethod YAML(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Stream.TmpCharacter +ClassMethod BuildYaml(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Stream.TmpCharacter { set tree = ..BuildResultTree(testIndex) set yamlStream = ##class(%Stream.TmpCharacter).%New() @@ -77,13 +78,13 @@ ClassMethod YAML(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As % do yamlStream.WriteLine(" testDateTime: """_tree.testDateTime_"""") do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") - do yamlStream.CopyFrom(..YAMLResults(tree)) + do yamlStream.CopyFrom(..BuildYamlResults(tree)) return yamlStream } /// Returns a stream of YAML result rows for all entries in the tree (no header). /// Used by ToFile via YAML, and by OutputToDevice for the failures section. -ClassMethod YAMLResults(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] +ClassMethod BuildYamlResults(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] { set resultStream = ##class(%Stream.TmpCharacter).%New() set (currentSuite, currentCase) = "" @@ -137,9 +138,8 @@ ClassMethod YAMLResults(tree As %DynamicObject) As %Stream.TmpCharacter [ Privat } /// Returns a stream of YAML failure rows only (no header). Used by OutputToDevice. -ClassMethod YAMLFailures(testIndex As %Integer) As %Stream.TmpCharacter [ Private ] +ClassMethod BuildYamlFailures(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] { - set tree = ..BuildResultTree(testIndex) set failStream = ##class(%Stream.TmpCharacter).%New() set (currentSuite, currentCase) = "" set suiteIter = tree.suites.%GetIterator() diff --git a/src/cls/IPM/Utils/OutputSuppressor.cls b/src/cls/IPM/Utils/OutputSuppressor.cls index d5b6c49f4..f3ca13524 100644 --- a/src/cls/IPM/Utils/OutputSuppressor.cls +++ b/src/cls/IPM/Utils/OutputSuppressor.cls @@ -1,32 +1,33 @@ /// I/O redirect routine that discards all writes. +/// Modeled after CaptureOutput, but instead of capturing output to a variable, it simply discards it. /// Used as the mnemonic device routine for BeginSuppressOutput / EndSuppressOutput. /// Unlike BeginCaptureOutput, suppress does not use ^||%capture, so it is safe /// to call while a capture is already active. Class %IPM.Utils.OutputSuppressor { -ClassMethod Begin(Output pCookie As %String) As %Status [ ProcedureBlock = 0 ] +ClassMethod Begin(Output cookie As %String) As %Status [ ProcedureBlock = 0 ] { - new tSC,e + new sc,ex - #dim tSC As %Status = $$$OK - #dim e As %Exception.AbstractException + #dim sc As %Status = $$$OK + #dim ex As %Exception.AbstractException try { if $zutil(82,12) { - set pCookie=$zutil(96,12) + set cookie=$zutil(96,12) } else { - set pCookie="" + set cookie="" } use $io::("^"_$zname) do $zutil(82,12,1) - } catch (e) { - set tSC=e.AsStatus() + } catch (ex) { + set sc=ex.AsStatus() } - quit tSC + quit sc rstr(sz,to) [rt] public { new rt set vr="rt" @@ -45,10 +46,10 @@ wtab(s) public { } write(s) public { } } -ClassMethod End(pCookie As %String) As %Status +ClassMethod End(cookie As %String) As %Status { - if pCookie '= "" { - use $io::("^"_pCookie) + if cookie '= "" { + use $io::("^"_cookie) } else { do $zutil(82,12,0) use $io::("") diff --git a/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls index 19857dbeb..b655e507b 100644 --- a/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls +++ b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls @@ -1,6 +1,14 @@ Class Test.PM.Integration.TestOutputFormat Extends Test.PM.Integration.Base { +/// The module is loaded once for the class rather than per-method because each `test` run +/// is fast and re-loading would dominate test time. This is safe because: +/// (1) each test method triggers its own `test` run, producing a new ^UnitTest.Result index, +/// (2) all tests that need the latest index use $order(^UnitTest.Result(""),-1), so they +/// always query their own run rather than a stale one, +/// (3) the test module has no side effects outside ^UnitTest.Result. +/// Adding new test methods: trigger a fresh `test` run or re-use the latest index — do not +/// rely on a specific index value from a previous test method. Method OnBeforeAllTests() As %Status { set sc = ##class(%IPM.Main).Shell("load " _ ..GetModuleDir("test-output-format")) @@ -10,10 +18,18 @@ Method OnBeforeAllTests() As %Status Method OnAfterAllTests() As %Status { + do ##class(%IPM.Main).Shell("config delete TestReportFormat") do ##class(%IPM.Main).Shell("uninstall test-output-format") return $$$OK } +/// Reset format config before each test so format-specific tests don't pollute each other. +Method OnBeforeOneTest(testName As %String) As %Status +{ + do ##class(%IPM.Main).Shell("config delete TestReportFormat") + return $$$OK +} + /// Helper: join captured output array into a single string ClassMethod JoinOutput(ByRef pOutput) As %String [ Private ] { @@ -28,8 +44,7 @@ ClassMethod JoinOutput(ByRef pOutput) As %String [ Private ] } -/// Default (non-verbose) output: summary + failures section, no results section. -/// Expects 12 failed methods and 13 failure rows. +/// Default (non-verbose, no -f) output: summary only, no failures or results section. Method TestDefaultOutput() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) @@ -39,26 +54,39 @@ Method TestDefaultOutput() do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") - do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows") - do $$$AssertNotTrue(content [ "results[", "no results section in default mode") + do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") + do $$$AssertNotTrue(content [ "results[", "no results section without -f") } -/// Verbose output requires -f to route through OutputToDevice. -/// With -verbose -f toon: results section with all methods, no failures section. +/// Verbose without -f: summary only (no results/failures section). Method TestVerboseOutput() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f toon") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") + do $$$AssertNotTrue(content [ "results[", "no results section without -f") + do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") +} + +/// Verbose with -f: results section with all methods, no failures section. +Method TestVerboseOutputFormat() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") - do $$$AssertTrue(content [ "results[", "results section present in verbose mode") + do $$$AssertTrue(content [ "results[", "results section present with -verbose -f") do $$$AssertNotTrue(content [ "failures[", "no failures section in verbose mode") } -/// Quiet output: suppresses test runner noise, still shows summary and failures. +/// Quiet output: suppresses test runner noise, shows summary only (no failures section without -f). /// Known gap: two pre-phase lines ([USER|ZPM] Test START, Building dependency graph) still escape suppression. Method TestQuietOutput() { @@ -69,20 +97,33 @@ Method TestQuietOutput() do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods in quiet mode") - do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows in quiet mode") + do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") +} + +/// -f with toon format (default): summary + failures section with 13 rows. +Method TestOutputFormatToon() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") + do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows") + do $$$AssertNotTrue(content [ "results[", "no results section in default mode") } -/// -f json: summary + failures as JSON object (not verbose, so failures key not methods). +/// -f with json format configured: summary + failures as JSON object. Method TestOutputFormatJson() { + do ##class(%IPM.Main).Shell("config set TestReportFormat json") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f json") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") - // Extract just the JSON object from the output: first "{" to last "}" - // OutputFailures writes "FAILED ..." lines after the JSON, so we must not include them set raw = ..JoinOutput(.output) set jsonStart = $find(raw, "{") - 1 set jsonEnd = $length(raw) - $find($reverse(raw), "}") + 2 @@ -95,11 +136,12 @@ Method TestOutputFormatJson() do $$$AssertNotTrue($isobject(parsed.methods), "JSON output does not have methods array in default mode") } -/// -verbose -f json: methods array with all methods, no failures array. +/// -verbose -f with json format configured: methods array with all methods, no failures array. Method TestVerboseOutputFormatJson() { + do ##class(%IPM.Main).Shell("config set TestReportFormat json") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f json") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -116,12 +158,13 @@ Method TestVerboseOutputFormatJson() do $$$AssertNotTrue($isobject(parsed.failures), "Verbose JSON output does not have failures array") } -/// -f yaml: summary + failures block (not verbose). +/// -f with yaml format configured: summary + failures block. /// YAML summary uses nested "failed: N" structure, not inline "N failed". Method TestOutputFormatYaml() { + do ##class(%IPM.Main).Shell("config set TestReportFormat yaml") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f yaml") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -131,11 +174,12 @@ Method TestOutputFormatYaml() do $$$AssertNotTrue(content [ ($char(10)_"results:"), "YAML output has no results block in default mode") } -/// -verbose -f yaml: results block present, no failures block. +/// -verbose -f with yaml format configured: results block present, no failures block. Method TestVerboseOutputFormatYaml() { + do ##class(%IPM.Main).Shell("config set TestReportFormat yaml") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f yaml") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -148,7 +192,7 @@ Method TestVerboseOutputFormatYaml() Method TestOutputFileToon() { set tmpFile = ##class(%Library.File).TempFilename() _ ".toon" - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file """ _ tmpFile _ """") do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "Toon output file is non-empty") @@ -168,7 +212,7 @@ Method TestOutputFileToon() Method TestOutputFileYaml() { set tmpFile = ##class(%Library.File).TempFilename() _ ".yaml" - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file """ _ tmpFile _ """") do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "YAML output file is non-empty") @@ -189,7 +233,7 @@ Method TestOutputFileYaml() Method TestOutputFileJson() { set tmpFile = ##class(%Library.File).TempFilename() _ ".json" - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file """ _ tmpFile _ """") do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "JSON output file is non-empty") @@ -207,7 +251,7 @@ Method TestOutputFileJson() Method TestOutputFileXml() { set tmpFile = ##class(%Library.File).TempFilename() _ ".xml" - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file " _ tmpFile) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -output-file """ _ tmpFile _ """") do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") do $$$AssertTrue(##class(%File).GetFileSize(tmpFile) > 0, "XML output file is non-empty") From b43113817fb5a2b748ecd3c16f14d29b88396943 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Apr 2026 12:21:57 -0400 Subject: [PATCH 15/18] Refactor, change -f back to a value flag, and improve behavior of config setting --- src/cls/IPM/Main.cls | 2 +- src/cls/IPM/Repo/UniversalSettings.cls | 12 +- src/cls/IPM/ResourceProcessor/Test.cls | 33 +-- src/cls/IPM/Test/Abstract.cls | 70 ++++++- src/cls/IPM/Test/JUnitOutput.cls | 191 ++++++------------ src/cls/IPM/Test/JsonOutput.cls | 18 +- src/cls/IPM/Test/ToonOutput.cls | 39 ++-- src/cls/IPM/Test/YamlOutput.cls | 136 ++++--------- .../Test/PM/Integration/TestOutputFormat.cls | 114 +++++++---- tests/unit_tests/Test/PM/Unit/CLI.cls | 2 +- 10 files changed, 286 insertions(+), 331 deletions(-) diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 8bd8dfdd7..3871ecdbf 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -75,7 +75,7 @@ Can also specify desired version to update to. - + diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls index 8c0999896..e1ec5ee3d 100644 --- a/src/cls/IPM/Repo/UniversalSettings.cls +++ b/src/cls/IPM/Repo/UniversalSettings.cls @@ -89,11 +89,13 @@ ClassMethod ResetToDefault(key As %String) As %Status write "Config key = """_key_""" not found",! quit } - set sc = ..SetValue($parameter(..%ClassName(1),key), ..GetDefaultValue($parameter(..%ClassName(1),key))) + // TestReportFormat has no factory default; empty means "use legacy output" + set defaultValue = $select(key = "TestReportFormat": "", 1: ..GetDefaultValue($parameter(..%ClassName(1),key))) + set sc = ..SetValue($parameter(..%ClassName(1),key), defaultValue) if $$$ISOK(sc) { - write "Value for """_key_""" succesfully reset to default",! + write !,"Value for """_key_""" succesfully reset to default",! } else { - write "Error reseting value for """_key_"""",! + write !,"Error reseting value for """_key_"""",! } return sc } @@ -109,9 +111,9 @@ ClassMethod UpdateOne( } set sc = ..SetValue($parameter(..%ClassName(1),key), value) if $$$ISOK(sc) { - write "Key """_key_""" succesfully updated",! + write !,"Key """_key_""" succesfully updated",! } else { - write "Error updating """_key_"""",! + write !,"Error updating """_key_"""",! } return sc } diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 5a169ca50..37228258f 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -232,27 +232,28 @@ Method OnPhase( set capturing = 0 } - set hasOutputFormat = $data(pParams("outputformat")) - write !!,"Test Result Summary" - // -f enables the failures/results detail section. The format is driven by - // 'config set TestReportFormat'; falls back to Toon if not configured. - // Without -f only the summary line is shown, matching pre-existing behavior. - set showDetail = hasOutputFormat - set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + set outputFormat = $get(pParams("outputformat")) if outputFormat = "" { - set outputFormat = "Toon" + set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() } - set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" - if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"The requested "_outputFormat_" output format does not exist.")) + + write !!,"Test Results:" + if outputFormat '= "" { + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"Unknown output format: "_outputFormat)) + } + set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose,1) + $$$ThrowOnError(tSC) + } else { + set tSC = ##class(%IPM.Test.Abstract).OutputToDevice(testIndex,tVerbose,0) + $$$ThrowOnError(tSC) } - set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose,showDetail) - $$$ThrowOnError(tSC) write ! - // By default, detect and report unit test failures as an error from this phase. - // OutputFailures is skipped when detail is shown since the formatter already includes failures. + // Detect and report unit test failures as an error from this phase. + // OutputFailures shows legacy red FAILED lines only when no format is active. if $get(pParams("UnitTest","FailuresAreFatal"),1) { - if 'showDetail { + if outputFormat = "" { do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) } set tSC = ##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex) diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index 8ac45574f..6bf81b4bf 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -3,12 +3,29 @@ Class %IPM.Test.Abstract Extends %RegisteredObject { /// Comma-separated list of valid output format names (lowercase). -/// Must stay in sync with the valueList on the output-format modifier in %IPM.Main. +/// Must stay in sync with the valueList on the format modifier in %IPM.Main. Parameter VALIDFORMATS = "json,yaml,toon"; ClassMethod ToFile( fileName As %String, - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status [ Abstract ] + testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +{ + set sc = $$$OK + try { + set fileStream = ##class(%Stream.FileCharacter).%New() + set fileStream.TranslateTable = "UTF8" + $$$ThrowOnError(fileStream.LinkToFile(fileName)) + do ..WriteToFileStream(fileStream, testIndex) + $$$ThrowOnError(fileStream.%Save()) + } catch ex { + set sc = ex.AsStatus() + } + return sc +} + +ClassMethod WriteToFileStream( + fileStream As %Stream.FileCharacter, + testIndex As %Integer) [ Abstract ] { } @@ -17,8 +34,55 @@ ClassMethod ToFile( ClassMethod OutputToDevice( testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, verbose As %Boolean = 0, - showDetail As %Boolean = 1) As %Status [ Abstract ] + showDetail As %Boolean = 1) As %Status { + set sc = $$$OK + try { + set tree = ..BuildResultTree(testIndex) + set summary = ..GetSummary(testIndex, tree) + write !!,"Test Run #"_summary.id_" ("_summary.namespace_") "_summary.duration_"s "_summary.testDateTime + write !,"Methods: "_summary.methods.total_" total, "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" + write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" + + if showDetail && verbose { + write ! + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" ["_methodObj.status_"]" + if methodObj.error '= "" { write " "_methodObj.error } + } + } + } + } elseif showDetail && (summary.methods.failed > 0) { + write ! + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" [failed]" + if methodObj.error '= "" { write " "_methodObj.error } + } + } + } + } + } + } catch ex { + set sc = ex.AsStatus() + } + return sc } /// Walks ^UnitTest.Result and returns a fully-populated result tree. diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index bb7e8dec3..52cd2d2b1 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -1,152 +1,87 @@ Class %IPM.Test.JUnitOutput Extends %IPM.Test.Abstract { -ClassMethod ToFile( - fileName As %String, - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +ClassMethod WriteToFileStream( + fileStream As %Stream.FileCharacter, + testIndex As %Integer) { - set sc = $$$OK - try { - set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(fileName)) + set tree = ..BuildResultTree(testIndex) - set tree = ..BuildResultTree(testIndex) + do fileStream.WriteLine("") + do fileStream.WriteLine("") - do fileStream.WriteLine("") - do fileStream.WriteLine("") - - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set suiteTests = 0 - set suiteFailures = 0 - set suiteAssertions = 0 - set suiteDuration = 0 - set caseIter2 = suiteObj.cases.%GetIterator() - while caseIter2.%GetNext(, .caseObj2) { - set methodIter2 = caseObj2.methods.%GetIterator() - while methodIter2.%GetNext(, .methodObj2) { - set suiteTests = suiteTests + 1 - if methodObj2.status = "failed" { set suiteFailures = suiteFailures + 1 } - set suiteDuration = suiteDuration + methodObj2.duration - set assertIter2 = methodObj2.asserts.%GetIterator() - while assertIter2.%GetNext(, .unused) { set suiteAssertions = suiteAssertions + 1 } - } - if caseObj2.error '= "" { - set suiteTests = suiteTests + 1 - set suiteFailures = suiteFailures + 1 - } + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set suiteTests = 0 + set suiteFailures = 0 + set suiteAssertions = 0 + set suiteDuration = 0 + set caseIter2 = suiteObj.cases.%GetIterator() + while caseIter2.%GetNext(, .caseObj2) { + set methodIter2 = caseObj2.methods.%GetIterator() + while methodIter2.%GetNext(, .methodObj2) { + set suiteTests = suiteTests + 1 + if methodObj2.status = "failed" { set suiteFailures = suiteFailures + 1 } + set suiteDuration = suiteDuration + methodObj2.duration + set assertIter2 = methodObj2.asserts.%GetIterator() + while assertIter2.%GetNext(, .unused) { set suiteAssertions = suiteAssertions + 1 } + } + if caseObj2.error '= "" { + set suiteTests = suiteTests + 1 + set suiteFailures = suiteFailures + 1 } + } + + do fileStream.Write("") + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { do fileStream.Write("") - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - do fileStream.Write("") - - if caseObj.error '= "" { - do fileStream.Write("") - set msg = ..EncodeXMLAttr(caseObj.error) - do fileStream.Write("") - do fileStream.WriteLine("") - } + if caseObj.error '= "" { + do fileStream.Write("") + set msg = ..EncodeXMLAttr(caseObj.error) + do fileStream.Write("") + do fileStream.WriteLine("") + } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - set methodAssertions = methodObj.asserts.%Size() - do fileStream.Write("") + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + set methodAssertions = methodObj.asserts.%Size() + do fileStream.Write("") - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - if assertObj.status = "failed" { - set msg = ..EncodeXMLAttr(assertObj.action_": "_assertObj.description) - do fileStream.Write("") - do fileStream.WriteLine("") - } - } - if (methodObj.status = "failed") && (methodObj.error '= "") { - set msg = ..EncodeXMLAttr(methodObj.error) + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + set msg = ..EncodeXMLAttr(assertObj.action_": "_assertObj.description) do fileStream.Write("") do fileStream.WriteLine("") } - do fileStream.WriteLine("") } - do fileStream.WriteLine("") - } - do fileStream.WriteLine("") - } - do fileStream.WriteLine("") - $$$ThrowOnError(fileStream.%Save()) - } catch ex { - set sc = ex.AsStatus() - } - return sc -} - -ClassMethod OutputToDevice( - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, - verbose As %Boolean = 0) As %Status -{ - set sc = $$$OK - try { - set summary = ..GetSummary(testIndex) - write !,"JUnit Test Run #"_summary.id_" ("_summary.namespace_") "_summary.duration_"s "_summary.testDateTime - write !,"Methods: "_summary.methods.total_" total, "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" - write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - - if verbose { - set tree = ..BuildResultTree(testIndex) - write ! - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" ["_methodObj.status_"]" - if methodObj.error '= "" { write " "_methodObj.error } - } - } - } - } elseif summary.methods.failed > 0 { - set tree = ..BuildResultTree(testIndex) - write ! - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if methodObj.status = "failed" { - write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" [failed]" - if methodObj.error '= "" { write " "_methodObj.error } - } - } + if (methodObj.status = "failed") && (methodObj.error '= "") { + set msg = ..EncodeXMLAttr(methodObj.error) + do fileStream.Write("") + do fileStream.WriteLine("") } + do fileStream.WriteLine("") } + do fileStream.WriteLine("") } - } catch ex { - set sc = ex.AsStatus() + do fileStream.WriteLine("") } - return sc + do fileStream.WriteLine("") } ClassMethod EncodeXMLAttr(msg As %String) As %String [ Private ] diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 7bbf36d3f..7c6a01242 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -1,21 +1,11 @@ Class %IPM.Test.JsonOutput Extends %IPM.Test.Abstract { -ClassMethod ToFile( - fileName As %String, - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +ClassMethod WriteToFileStream( + fileStream As %Stream.FileCharacter, + testIndex As %Integer) { - set sc = $$$OK - try { - set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - do fileStream.Write(..BuildResultTree(testIndex).%ToJSON()) - $$$ThrowOnError(fileStream.%Save()) - } catch ex { - set sc = ex.AsStatus() - } - return sc + do fileStream.Write(..BuildResultTree(testIndex).%ToJSON()) } ClassMethod OutputToDevice( diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index a07adeebb..fbdc27566 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -1,33 +1,22 @@ Class %IPM.Test.ToonOutput Extends %IPM.Test.Abstract { -ClassMethod ToFile( - fileName As %String, - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +ClassMethod WriteToFileStream( + fileStream As %Stream.FileCharacter, + testIndex As %Integer) { - set sc = $$$OK - try { - set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - - set tree = ..BuildResultTree(testIndex) - set rowStream = ..BuildToonRows(tree, 0, .rowCount) + set tree = ..BuildResultTree(testIndex) + set rowStream = ..BuildToonRows(tree, 0, .rowCount) - do fileStream.WriteLine("unitTest:") - do fileStream.WriteLine(" id: "_tree.id) - do fileStream.WriteLine(" namespace: "_tree.namespace) - do fileStream.WriteLine(" duration: "_tree.duration_"s") - do fileStream.WriteLine(" testDateTime: "_tree.testDateTime) - do fileStream.WriteLine() - do fileStream.WriteLine("results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") - do rowStream.Rewind() - do fileStream.CopyFrom(rowStream) - $$$ThrowOnError(fileStream.%Save()) - } catch ex { - set sc = ex.AsStatus() - } - return sc + do fileStream.WriteLine("unitTest:") + do fileStream.WriteLine(" id: "_tree.id) + do fileStream.WriteLine(" namespace: "_tree.namespace) + do fileStream.WriteLine(" duration: "_tree.duration_"s") + do fileStream.WriteLine(" testDateTime: "_tree.testDateTime) + do fileStream.WriteLine() + do fileStream.WriteLine("results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:") + do rowStream.Rewind() + do fileStream.CopyFrom(rowStream) } ClassMethod OutputToDevice( diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 8f52b524c..073cfd516 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -1,21 +1,11 @@ Class %IPM.Test.YamlOutput Extends %IPM.Test.Abstract { -ClassMethod ToFile( - fileName As %String, - testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) As %Status +ClassMethod WriteToFileStream( + fileStream As %Stream.FileCharacter, + testIndex As %Integer) { - set sc = $$$OK - try { - set fileStream = ##class(%Stream.FileCharacter).%New() - set fileStream.TranslateTable = "UTF8" - $$$ThrowOnError(fileStream.LinkToFile(fileName)) - do fileStream.CopyFrom(..BuildYaml(testIndex)) - $$$ThrowOnError(fileStream.%Save()) - } catch ex { - set sc = ex.AsStatus() - } - return sc + do fileStream.CopyFrom(..BuildYaml(testIndex)) } ClassMethod OutputToDevice( @@ -45,7 +35,7 @@ ClassMethod OutputToDevice( if showDetail && verbose { write ! write !,"results:" - set resultStream = ..BuildYamlResults(tree) + set resultStream = ..BuildYamlRows(tree, 0) do resultStream.Rewind() while 'resultStream.AtEnd { write !,resultStream.ReadLine() @@ -53,7 +43,7 @@ ClassMethod OutputToDevice( } elseif showDetail && (summary.methods.failed > 0) { write ! write !,"failures:" - set failStream = ..BuildYamlFailures(tree) + set failStream = ..BuildYamlRows(tree, 1) do failStream.Rewind() while 'failStream.AtEnd { write !,failStream.ReadLine() @@ -78,69 +68,16 @@ ClassMethod BuildYaml(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) do yamlStream.WriteLine(" testDateTime: """_tree.testDateTime_"""") do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") - do yamlStream.CopyFrom(..BuildYamlResults(tree)) + do yamlStream.CopyFrom(..BuildYamlRows(tree, 0)) return yamlStream } -/// Returns a stream of YAML result rows for all entries in the tree (no header). -/// Used by ToFile via YAML, and by OutputToDevice for the failures section. -ClassMethod BuildYamlResults(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] -{ - set resultStream = ##class(%Stream.TmpCharacter).%New() - set (currentSuite, currentCase) = "" - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - if suiteObj.name '= currentSuite { - set currentSuite = suiteObj.name - set currentCase = "" - do resultStream.WriteLine(" - suiteName: """_suiteObj.name_"""") - do resultStream.WriteLine(" testcases:") - } - do resultStream.WriteLine(" - testcaseName: """_caseObj.name_"""") - do resultStream.WriteLine(" error: |") - do resultStream.WriteLine(" "_$replace(caseObj.error, $char(10), $char(10)_" ")) - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if suiteObj.name '= currentSuite { - set currentSuite = suiteObj.name - set currentCase = "" - do resultStream.WriteLine(" - suiteName: """_suiteObj.name_"""") - do resultStream.WriteLine(" testcases:") - } - if caseObj.name '= currentCase { - set currentCase = caseObj.name - do resultStream.WriteLine(" - testcaseName: """_caseObj.name_"""") - do resultStream.WriteLine(" methods:") - } - do resultStream.WriteLine(" - methodName: """_methodObj.name_"""") - do resultStream.WriteLine(" status: """_methodObj.status_"""") - if methodObj.error '= "" { - do resultStream.WriteLine(" error: |") - do resultStream.WriteLine(" "_$replace(methodObj.error, $char(10), $char(10)_" ")) - } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - do resultStream.WriteLine(" assertAction: """_assertObj.action_"""") - do resultStream.WriteLine(" assertCounter: "_assertObj.counter) - do resultStream.WriteLine(" assertStatus: """_assertObj.status_"""") - do resultStream.WriteLine(" assertDescription: |") - do resultStream.WriteLine(" "_$replace(assertObj.description, $char(10), $char(10)_" ")) - do resultStream.WriteLine(" assertLocation: """_assertObj.location_"""") - } - } - } - } - return resultStream -} - -/// Returns a stream of YAML failure rows only (no header). Used by OutputToDevice. -ClassMethod BuildYamlFailures(tree As %DynamicObject) As %Stream.TmpCharacter [ Private ] +/// Returns a stream of YAML rows. failuresOnly=1 skips passing methods/assertions. +/// Each method's assertions are emitted under an `asserts:` list to produce valid YAML. +ClassMethod BuildYamlRows(tree As %DynamicObject, failuresOnly As %Boolean) As %Stream.TmpCharacter [ Private ] { - set failStream = ##class(%Stream.TmpCharacter).%New() + set indent = $select(failuresOnly: " ", 1: " ") + set stream = ##class(%Stream.TmpCharacter).%New() set (currentSuite, currentCase) = "" set suiteIter = tree.suites.%GetIterator() while suiteIter.%GetNext(, .suiteObj) { @@ -150,45 +87,50 @@ ClassMethod BuildYamlFailures(tree As %DynamicObject) As %Stream.TmpCharacter [ if suiteObj.name '= currentSuite { set currentSuite = suiteObj.name set currentCase = "" - do failStream.WriteLine(" - suiteName: """_suiteObj.name_"""") - do failStream.WriteLine(" testcases:") + do stream.WriteLine(indent_"- suiteName: """_suiteObj.name_"""") + do stream.WriteLine(indent_" testcases:") } - do failStream.WriteLine(" - testcaseName: """_caseObj.name_"""") - do failStream.WriteLine(" error: |") - do failStream.WriteLine(" "_$replace(caseObj.error, $char(10), $char(10)_" ")) + do stream.WriteLine(indent_" - testcaseName: """_caseObj.name_"""") + do stream.WriteLine(indent_" error: |") + do stream.WriteLine(indent_" "_$replace(caseObj.error, $char(10), $char(10)_indent_" ")) } set methodIter = caseObj.methods.%GetIterator() while methodIter.%GetNext(, .methodObj) { - if methodObj.status '= "failed" { continue } + if failuresOnly && (methodObj.status '= "failed") { continue } if suiteObj.name '= currentSuite { set currentSuite = suiteObj.name set currentCase = "" - do failStream.WriteLine(" - suiteName: """_suiteObj.name_"""") - do failStream.WriteLine(" testcases:") + do stream.WriteLine(indent_"- suiteName: """_suiteObj.name_"""") + do stream.WriteLine(indent_" testcases:") } if caseObj.name '= currentCase { set currentCase = caseObj.name - do failStream.WriteLine(" - testcaseName: """_caseObj.name_"""") - do failStream.WriteLine(" methods:") + do stream.WriteLine(indent_" - testcaseName: """_caseObj.name_"""") + do stream.WriteLine(indent_" methods:") } - do failStream.WriteLine(" - methodName: """_methodObj.name_"""") + do stream.WriteLine(indent_" - methodName: """_methodObj.name_"""") + do stream.WriteLine(indent_" status: """_methodObj.status_"""") if methodObj.error '= "" { - do failStream.WriteLine(" error: |") - do failStream.WriteLine(" "_$replace(methodObj.error, $char(10), $char(10)_" ")) + do stream.WriteLine(indent_" error: |") + do stream.WriteLine(indent_" "_$replace(methodObj.error, $char(10), $char(10)_indent_" ")) } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - if assertObj.status '= "failed" { continue } - do failStream.WriteLine(" assertAction: """_assertObj.action_"""") - do failStream.WriteLine(" assertCounter: "_assertObj.counter) - do failStream.WriteLine(" assertDescription: |") - do failStream.WriteLine(" "_$replace(assertObj.description, $char(10), $char(10)_" ")) - do failStream.WriteLine(" assertLocation: """_assertObj.location_"""") + if methodObj.asserts.%Size() > 0 { + do stream.WriteLine(indent_" asserts:") + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if failuresOnly && (assertObj.status '= "failed") { continue } + do stream.WriteLine(indent_" - action: """_assertObj.action_"""") + do stream.WriteLine(indent_" counter: "_assertObj.counter) + do stream.WriteLine(indent_" status: """_assertObj.status_"""") + do stream.WriteLine(indent_" description: |") + do stream.WriteLine(indent_" "_$replace(assertObj.description, $char(10), $char(10)_indent_" ")) + do stream.WriteLine(indent_" location: """_assertObj.location_"""") + } } } } } - return failStream + return stream } } diff --git a/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls index b655e507b..d761dc316 100644 --- a/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls +++ b/tests/integration_tests/Test/PM/Integration/TestOutputFormat.cls @@ -44,7 +44,7 @@ ClassMethod JoinOutput(ByRef pOutput) As %String [ Private ] } -/// Default (non-verbose, no -f) output: summary only, no failures or results section. +/// Default (no -f, no config): summary + legacy FAILED lines, no formatted sections. Method TestDefaultOutput() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) @@ -54,11 +54,11 @@ Method TestDefaultOutput() do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") - do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") - do $$$AssertNotTrue(content [ "results[", "no results section without -f") + do $$$AssertNotTrue(content [ "failures[", "no toon failures section without -f") + do $$$AssertNotTrue(content [ "results[", "no toon results section without -f") } -/// Verbose without -f: summary only (no results/failures section). +/// Verbose without -f: summary only (no results/failures section), plus legacy FAILED lines. Method TestVerboseOutput() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) @@ -72,54 +72,39 @@ Method TestVerboseOutput() do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") } -/// Verbose with -f: results section with all methods, no failures section. -Method TestVerboseOutputFormat() +/// -f toon: summary + failures section with 13 rows. +Method TestOutputFormatToon() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f toon") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") - do $$$AssertTrue(content [ "results[", "results section present with -verbose -f") - do $$$AssertNotTrue(content [ "failures[", "no failures section in verbose mode") -} - -/// Quiet output: suppresses test runner noise, shows summary only (no failures section without -f). -/// Known gap: two pre-phase lines ([USER|ZPM] Test START, Building dependency graph) still escape suppression. -Method TestQuietOutput() -{ - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -quiet") - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - - do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") - set content = ..JoinOutput(.output) - do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods in quiet mode") - do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") + do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows") + do $$$AssertNotTrue(content [ "results[", "no results section in default mode") } -/// -f with toon format (default): summary + failures section with 13 rows. -Method TestOutputFormatToon() +/// -f toon -verbose: results section with all rows, no failures section. +Method TestVerboseOutputFormatToon() { do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f toon") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") set content = ..JoinOutput(.output) do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods") - do $$$AssertTrue(content [ "failures[13]", "failures section has 13 rows") - do $$$AssertNotTrue(content [ "results[", "no results section in default mode") + do $$$AssertTrue(content [ "results[", "results section present with -verbose -f toon") + do $$$AssertNotTrue(content [ "failures[", "no failures section in verbose mode") } -/// -f with json format configured: summary + failures as JSON object. +/// -f json: summary + failures as JSON object. Method TestOutputFormatJson() { - do ##class(%IPM.Main).Shell("config set TestReportFormat json") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f json") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -136,12 +121,11 @@ Method TestOutputFormatJson() do $$$AssertNotTrue($isobject(parsed.methods), "JSON output does not have methods array in default mode") } -/// -verbose -f with json format configured: methods array with all methods, no failures array. +/// -f json -verbose: methods array with all methods, no failures array. Method TestVerboseOutputFormatJson() { - do ##class(%IPM.Main).Shell("config set TestReportFormat json") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f json") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -158,13 +142,11 @@ Method TestVerboseOutputFormatJson() do $$$AssertNotTrue($isobject(parsed.failures), "Verbose JSON output does not have failures array") } -/// -f with yaml format configured: summary + failures block. -/// YAML summary uses nested "failed: N" structure, not inline "N failed". +/// -f yaml: summary + failures block. Method TestOutputFormatYaml() { - do ##class(%IPM.Main).Shell("config set TestReportFormat yaml") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f yaml") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -174,12 +156,11 @@ Method TestOutputFormatYaml() do $$$AssertNotTrue(content [ ($char(10)_"results:"), "YAML output has no results block in default mode") } -/// -verbose -f with yaml format configured: results block present, no failures block. +/// -f yaml -verbose: results block present, no failures block. Method TestVerboseOutputFormatYaml() { - do ##class(%IPM.Main).Shell("config set TestReportFormat yaml") do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f") + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -verbose -f yaml") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") @@ -188,6 +169,57 @@ Method TestVerboseOutputFormatYaml() do $$$AssertNotTrue(content [ ($char(10)_"failures:"), "Verbose YAML output has no failures block") } +/// Config default: setting TestReportFormat=json makes default output use json. +Method TestConfigDefault() +{ + do ##class(%IPM.Main).Shell("config set TestReportFormat json") + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set raw = ..JoinOutput(.output) + set jsonStart = $find(raw, "{") - 1 + set jsonEnd = $length(raw) - $find($reverse(raw), "}") + 2 + set jsonStr = $extract(raw, jsonStart, jsonEnd) + + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed), "Config default JSON output is parseable") + do $$$AssertTrue($isobject(parsed.failures), "Config default shows failures") +} + +/// -f overrides config: config=yaml but -f json should produce JSON. +Method TestFormatOverridesConfig() +{ + do ##class(%IPM.Main).Shell("config set TestReportFormat yaml") + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -f json") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set raw = ..JoinOutput(.output) + set jsonStart = $find(raw, "{") - 1 + set jsonEnd = $length(raw) - $find($reverse(raw), "}") + 2 + set jsonStr = $extract(raw, jsonStart, jsonEnd) + + set parsed = ##class(%DynamicAbstractObject).%FromJSON(jsonStr) + do $$$AssertTrue($isobject(parsed), "-f json overrides config=yaml") +} + +/// Quiet output: suppresses test runner noise, shows summary only (no failures section without -f). +/// Known gap: two pre-phase lines ([USER|ZPM] Test START, Building dependency graph) still escape suppression. +Method TestQuietOutput() +{ + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("test test-output-format -only -quiet") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusNotOK(sc, "test run returns error when failures exist") + set content = ..JoinOutput(.output) + do $$$AssertTrue(content [ "12 failed", "summary reports 12 failed methods in quiet mode") + do $$$AssertNotTrue(content [ "failures[", "no failures section without -f") +} + /// -output-file .toon: ToFile always writes all rows under results[N], not failures-only. Method TestOutputFileToon() { diff --git a/tests/unit_tests/Test/PM/Unit/CLI.cls b/tests/unit_tests/Test/PM/Unit/CLI.cls index 993db5625..15b5e7254 100644 --- a/tests/unit_tests/Test/PM/Unit/CLI.cls +++ b/tests/unit_tests/Test/PM/Unit/CLI.cls @@ -369,7 +369,7 @@ Method TestReportFormatConfiguration() if originalFormat'="" { do ..RunCommand("config set TestReportFormat "_originalFormat) } else { - do ..RunCommand("config reset TestReportFormat") + do ..RunCommand("config delete TestReportFormat") } } From 5a49fb7feea40a1500e8c13e3ff33a0e620df7f1 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Apr 2026 13:00:19 -0400 Subject: [PATCH 16/18] Minor fixes --- CHANGELOG.md | 2 +- preload/cls/IPM/Installer.cls | 1 - src/cls/IPM/Repo/UniversalSettings.cls | 14 +++++++----- src/cls/IPM/Test/Abstract.cls | 2 +- src/cls/IPM/Test/JUnitOutput.cls | 4 +++- src/cls/IPM/Test/ToonOutput.cls | 15 ++++++++++--- src/cls/IPM/Test/YamlOutput.cls | 30 ++++++++++++++++---------- 7 files changed, 45 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa531f67b..010e44b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #992: Implement automatic history purge logic - #973: Enables CORS and JWT configuration for WebApplications in module.xml - #1110: Add `iriscli` and `ipm` container utility scripts that are auto-installed to `~/.local/bin/` and `~/bin/` so they work both inside and outside of containers (Unix/Linux only) -- #971: Adds support for JSON, YAML, and Toon formats via the -output-format/-f flag for outputting into the terminal, adds the -output-file flag for outputting to a file, and improves the -quiet flag to suppress most output +- #971: Adds structured test output formats (JSON, YAML, Toon). Use `-f ` for a one-shot override or `config set TestReportFormat ` for a persistent default. Without either, legacy output is shown. Also adds `-output-file` for writing results to a file (including JUnit XML via `.xml` extension) and improves `-quiet` to suppress build noise. ### Fixed - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls index e2f323d9f..44361f71e 100644 --- a/preload/cls/IPM/Installer.cls +++ b/preload/cls/IPM/Installer.cls @@ -117,7 +117,6 @@ ClassMethod ZPMInit( $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("UseStandalonePip", "", 0)) $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0, 0)) $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("DefaultLogEntryLimit",20, 0)) - $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("TestReportFormat","toon", 0)) quit $$$OK } diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls index e1ec5ee3d..e5f79a85a 100644 --- a/src/cls/IPM/Repo/UniversalSettings.cls +++ b/src/cls/IPM/Repo/UniversalSettings.cls @@ -109,11 +109,16 @@ ClassMethod UpdateOne( write "Config key = """_key_""" not found",! quit } - set sc = ..SetValue($parameter(..%ClassName(1),key), value) + if key = "TestReportFormat" { + // Validate format value before saving it + set sc = ..SetTestReportFormat(value) + } else { + set sc = ..SetValue($parameter(..%ClassName(1),key), value) + } if $$$ISOK(sc) { write !,"Key """_key_""" succesfully updated",! } else { - write !,"Error updating """_key_"""",! + write !,$system.Status.GetErrorText(sc),! } return sc } @@ -197,11 +202,10 @@ ClassMethod GetHistoryRetain() As %Integer ClassMethod SetTestReportFormat( val As %String, - overwrite As %Boolean = 1) As %Boolean + overwrite As %Boolean = 1) As %Status { - // Validate format value if val '= "" && ('$listfind($listfromstring(##class(%IPM.Test.Abstract).#VALIDFORMATS), $zconvert(val, "l"))) { - return 0 + return $$$ERROR($$$GeneralError, "Unknown format '"_val_"'. Valid formats: "_##class(%IPM.Test.Abstract).#VALIDFORMATS) } return ..SetValue(..#TestReportFormat, val, overwrite) } diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index 6bf81b4bf..1e86f0c1f 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -196,7 +196,7 @@ ClassMethod BuildResultTree(testIndex As %Integer = {$order(^UnitTest.Result("") /// Pass a pre-built tree to avoid a second ^UnitTest.Result traversal. ClassMethod GetSummary(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}, tree As %DynamicObject = "") As %DynamicObject { - if tree = "" { + if '$isobject(tree) { set tree = ..BuildResultTree(testIndex) } set (assertTotal, assertPassed, assertFailed) = 0 diff --git a/src/cls/IPM/Test/JUnitOutput.cls b/src/cls/IPM/Test/JUnitOutput.cls index 52cd2d2b1..843e0f4ac 100644 --- a/src/cls/IPM/Test/JUnitOutput.cls +++ b/src/cls/IPM/Test/JUnitOutput.cls @@ -86,8 +86,10 @@ ClassMethod WriteToFileStream( ClassMethod EncodeXMLAttr(msg As %String) As %String [ Private ] { - set msg = $zstrip(msg, "*C") + // Strip control chars but preserve LF/CR for entity encoding below + set msg = $translate(msg, $char(0,1,2,3,4,5,6,7,8,11,12,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)) set msg = $zconvert($zconvert(msg, "O", "UTF8"), "O", "XML") + set msg = $replace(msg, $char(13,10), " ") set msg = $replace(msg, $char(10), " ") set msg = $replace(msg, $char(13), " ") return msg diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index fbdc27566..b474699ed 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -57,6 +57,15 @@ ClassMethod OutputToDevice( return sc } +/// RFC 4180 quoting: doubles internal quotes, wraps in quotes if value contains comma/quote/newline. +ClassMethod EscapeToonField(val As %String) As %String [ Private ] +{ + if val [ """" || (val [ ",") || (val [ $char(10)) { + return """"_$replace(val, """", """""")_"""" + } + return """"_val_"""" +} + /// Returns a %Stream.TmpCharacter containing Toon result rows. /// failuresOnly=0: all methods (for ToFile and verbose OutputToDevice). /// failuresOnly=1: failed methods and assert failures only (for non-verbose OutputToDevice). @@ -71,20 +80,20 @@ ClassMethod BuildToonRows(tree As %DynamicObject, failuresOnly As %Boolean, Outp while caseIter.%GetNext(, .caseObj) { if caseObj.error '= "" { set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"""_$translate(caseObj.error,"""")_""",""""") + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_caseObj.name_",failed,OnBeforeAllTests,0,"_..EscapeToonField(caseObj.error)_",""""") } set methodIter = caseObj.methods.%GetIterator() while methodIter.%GetNext(, .methodObj) { if failuresOnly && (methodObj.status '= "failed") { continue } if methodObj.error '= "" { set rowCount = rowCount + 1 - do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"""_$translate(methodObj.error,"""")_""",""""") + do rowStream.WriteLine(" "_suiteObj.name_","_caseObj.name_","_methodObj.name_",failed,error,0,"_..EscapeToonField(methodObj.error)_",""""") } set assertIter = methodObj.asserts.%GetIterator() while assertIter.%GetNext(, .assertObj) { if failuresOnly && (assertObj.status '= "failed") { continue } set rowCount = rowCount + 1 - set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","""_$translate(assertObj.description,"""")_""","""_assertObj.location_"""" + set row = " "_suiteObj.name_","_caseObj.name_","_methodObj.name_","_assertObj.status_","_assertObj.action_","_assertObj.counter_","_..EscapeToonField(assertObj.description)_","_..EscapeToonField(assertObj.location) do rowStream.WriteLine(row) } if 'failuresOnly && (methodObj.asserts.%Size() = 0) { diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 073cfd516..042d28ccd 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -20,9 +20,9 @@ ClassMethod OutputToDevice( write ! write !,"summary:" write !," id: "_summary.id - write !," namespace: """_summary.namespace_"""" + write !," namespace: """_..EscapeYamlString(summary.namespace)_"""" write !," duration: "_summary.duration_"s" - write !," testDateTime: """_summary.testDateTime_"""" + write !," testDateTime: """_..EscapeYamlString(summary.testDateTime)_"""" write !," methods:" write !," total: "_summary.methods.total write !," passed: "_summary.methods.passed @@ -63,15 +63,23 @@ ClassMethod BuildYaml(testIndex As %Integer = {$order(^UnitTest.Result(""),-1)}) do yamlStream.WriteLine("unitTest:") do yamlStream.WriteLine(" id: "_tree.id) - do yamlStream.WriteLine(" namespace: """_tree.namespace_"""") + do yamlStream.WriteLine(" namespace: """_..EscapeYamlString(tree.namespace)_"""") do yamlStream.WriteLine(" duration: "_tree.duration_"s") - do yamlStream.WriteLine(" testDateTime: """_tree.testDateTime_"""") + do yamlStream.WriteLine(" testDateTime: """_..EscapeYamlString(tree.testDateTime)_"""") do yamlStream.WriteLine() do yamlStream.WriteLine(" results:") do yamlStream.CopyFrom(..BuildYamlRows(tree, 0)) return yamlStream } +/// Escapes backslashes and double-quotes for use inside YAML double-quoted strings. +ClassMethod EscapeYamlString(val As %String) As %String [ Private ] +{ + set val = $replace(val, "\", "\\") + set val = $replace(val, """", "\""") + return val +} + /// Returns a stream of YAML rows. failuresOnly=1 skips passing methods/assertions. /// Each method's assertions are emitted under an `asserts:` list to produce valid YAML. ClassMethod BuildYamlRows(tree As %DynamicObject, failuresOnly As %Boolean) As %Stream.TmpCharacter [ Private ] @@ -87,10 +95,10 @@ ClassMethod BuildYamlRows(tree As %DynamicObject, failuresOnly As %Boolean) As % if suiteObj.name '= currentSuite { set currentSuite = suiteObj.name set currentCase = "" - do stream.WriteLine(indent_"- suiteName: """_suiteObj.name_"""") + do stream.WriteLine(indent_"- suiteName: """_..EscapeYamlString(suiteObj.name)_"""") do stream.WriteLine(indent_" testcases:") } - do stream.WriteLine(indent_" - testcaseName: """_caseObj.name_"""") + do stream.WriteLine(indent_" - testcaseName: """_..EscapeYamlString(caseObj.name)_"""") do stream.WriteLine(indent_" error: |") do stream.WriteLine(indent_" "_$replace(caseObj.error, $char(10), $char(10)_indent_" ")) } @@ -100,15 +108,15 @@ ClassMethod BuildYamlRows(tree As %DynamicObject, failuresOnly As %Boolean) As % if suiteObj.name '= currentSuite { set currentSuite = suiteObj.name set currentCase = "" - do stream.WriteLine(indent_"- suiteName: """_suiteObj.name_"""") + do stream.WriteLine(indent_"- suiteName: """_..EscapeYamlString(suiteObj.name)_"""") do stream.WriteLine(indent_" testcases:") } if caseObj.name '= currentCase { set currentCase = caseObj.name - do stream.WriteLine(indent_" - testcaseName: """_caseObj.name_"""") + do stream.WriteLine(indent_" - testcaseName: """_..EscapeYamlString(caseObj.name)_"""") do stream.WriteLine(indent_" methods:") } - do stream.WriteLine(indent_" - methodName: """_methodObj.name_"""") + do stream.WriteLine(indent_" - methodName: """_..EscapeYamlString(methodObj.name)_"""") do stream.WriteLine(indent_" status: """_methodObj.status_"""") if methodObj.error '= "" { do stream.WriteLine(indent_" error: |") @@ -119,12 +127,12 @@ ClassMethod BuildYamlRows(tree As %DynamicObject, failuresOnly As %Boolean) As % set assertIter = methodObj.asserts.%GetIterator() while assertIter.%GetNext(, .assertObj) { if failuresOnly && (assertObj.status '= "failed") { continue } - do stream.WriteLine(indent_" - action: """_assertObj.action_"""") + do stream.WriteLine(indent_" - action: """_..EscapeYamlString(assertObj.action)_"""") do stream.WriteLine(indent_" counter: "_assertObj.counter) do stream.WriteLine(indent_" status: """_assertObj.status_"""") do stream.WriteLine(indent_" description: |") do stream.WriteLine(indent_" "_$replace(assertObj.description, $char(10), $char(10)_indent_" ")) - do stream.WriteLine(indent_" location: """_assertObj.location_"""") + do stream.WriteLine(indent_" location: """_..EscapeYamlString(assertObj.location)_"""") } } } From f5fc3185b8655e887bd6bbcde75f38784d5ee09e Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Apr 2026 13:12:57 -0400 Subject: [PATCH 17/18] Change formatting --- src/cls/IPM/Test/Abstract.cls | 58 ++++++++++++++++--------------- src/cls/IPM/Test/JsonOutput.cls | 60 +++++++++++++++++---------------- src/cls/IPM/Test/ToonOutput.cls | 32 +++++++++--------- src/cls/IPM/Test/YamlOutput.cls | 32 +++++++++--------- 4 files changed, 95 insertions(+), 87 deletions(-) diff --git a/src/cls/IPM/Test/Abstract.cls b/src/cls/IPM/Test/Abstract.cls index 1e86f0c1f..5621c0091 100644 --- a/src/cls/IPM/Test/Abstract.cls +++ b/src/cls/IPM/Test/Abstract.cls @@ -44,36 +44,38 @@ ClassMethod OutputToDevice( write !,"Methods: "_summary.methods.total_" total, "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !,"Assertions: "_summary.assertions.total_" total, "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - if showDetail && verbose { - write ! - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" ["_methodObj.status_"]" - if methodObj.error '= "" { write " "_methodObj.error } + if showDetail { + if verbose { + write ! + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" ["_methodObj.status_"]" + if methodObj.error '= "" { write " "_methodObj.error } + } } } - } - } elseif showDetail && (summary.methods.failed > 0) { - write ! - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - if methodObj.status = "failed" { - write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" [failed]" - if methodObj.error '= "" { write " "_methodObj.error } + } elseif summary.methods.failed > 0 { + write ! + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + write !,suiteObj.name_"/"_caseObj.name_" [failed] "_caseObj.error + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + write !,suiteObj.name_"/"_caseObj.name_"/"_methodObj.name_" [failed]" + if methodObj.error '= "" { write " "_methodObj.error } + } } } } diff --git a/src/cls/IPM/Test/JsonOutput.cls b/src/cls/IPM/Test/JsonOutput.cls index 7c6a01242..1f56a0da4 100644 --- a/src/cls/IPM/Test/JsonOutput.cls +++ b/src/cls/IPM/Test/JsonOutput.cls @@ -17,50 +17,52 @@ ClassMethod OutputToDevice( try { set tree = ..BuildResultTree(testIndex) set summary = ..GetSummary(testIndex, tree) - if showDetail && verbose { - set allMethods = [] - set suiteIter = tree.suites.%GetIterator() - while suiteIter.%GetNext(, .suiteObj) { - set caseIter = suiteObj.cases.%GetIterator() - while caseIter.%GetNext(, .caseObj) { - if caseObj.error '= "" { - do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "status": "failed", "error": (caseObj.error)}) - } - set methodIter = caseObj.methods.%GetIterator() - while methodIter.%GetNext(, .methodObj) { - do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "status": (methodObj.status), "error": (methodObj.error)}) - } - } - } - set output = {"summary": (summary), "methods": (allMethods)} - } elseif showDetail { - set failedMethods = [] - if summary.methods.failed > 0 { + if showDetail { + if verbose { + set allMethods = [] set suiteIter = tree.suites.%GetIterator() while suiteIter.%GetNext(, .suiteObj) { set caseIter = suiteObj.cases.%GetIterator() while caseIter.%GetNext(, .caseObj) { if caseObj.error '= "" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "error": (caseObj.error)}) + do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "status": "failed", "error": (caseObj.error)}) } set methodIter = caseObj.methods.%GetIterator() while methodIter.%GetNext(, .methodObj) { - if methodObj.status = "failed" { - if methodObj.error '= "" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error)}) - } - set assertIter = methodObj.asserts.%GetIterator() - while assertIter.%GetNext(, .assertObj) { - if assertObj.status = "failed" { - do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "action": (assertObj.action), "description": (assertObj.description), "location": (assertObj.location)}) + do allMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "status": (methodObj.status), "error": (methodObj.error)}) + } + } + } + set output = {"summary": (summary), "methods": (allMethods)} + } else { + set failedMethods = [] + if summary.methods.failed > 0 { + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + if caseObj.error '= "" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "error": (caseObj.error)}) + } + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + if methodObj.status = "failed" { + if methodObj.error '= "" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "error": (methodObj.error)}) + } + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + do failedMethods.%Push({"suite": (suiteObj.name), "case": (caseObj.name), "method": (methodObj.name), "action": (assertObj.action), "description": (assertObj.description), "location": (assertObj.location)}) + } } } } } } } + set output = {"summary": (summary), "failures": (failedMethods)} } - set output = {"summary": (summary), "failures": (failedMethods)} } else { set output = {"summary": (summary)} } diff --git a/src/cls/IPM/Test/ToonOutput.cls b/src/cls/IPM/Test/ToonOutput.cls index b474699ed..974505500 100644 --- a/src/cls/IPM/Test/ToonOutput.cls +++ b/src/cls/IPM/Test/ToonOutput.cls @@ -34,21 +34,23 @@ ClassMethod OutputToDevice( write !," methods["_summary.methods.total_"]: "_summary.methods.passed_" passed, "_summary.methods.failed_" failed" write !," assertions["_summary.assertions.total_"]: "_summary.assertions.passed_" passed, "_summary.assertions.failed_" failed" - if showDetail && verbose { - set rowStream = ..BuildToonRows(tree, 0, .rowCount) - write ! - write !,"results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" - do rowStream.Rewind() - while 'rowStream.AtEnd { - write !,rowStream.ReadLine() - } - } elseif showDetail && (summary.methods.failed > 0) { - set failStream = ..BuildToonRows(tree, 1, .failCount) - write ! - write !,"failures["_failCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" - do failStream.Rewind() - while 'failStream.AtEnd { - write !,failStream.ReadLine() + if showDetail { + if verbose { + set rowStream = ..BuildToonRows(tree, 0, .rowCount) + write ! + write !,"results["_rowCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + do rowStream.Rewind() + while 'rowStream.AtEnd { + write !,rowStream.ReadLine() + } + } elseif summary.methods.failed > 0 { + set failStream = ..BuildToonRows(tree, 1, .failCount) + write ! + write !,"failures["_failCount_"]{suiteName,testcaseName,methodName,status,assertAction,assertCounter,assertDescription,assertLocation}:" + do failStream.Rewind() + while 'failStream.AtEnd { + write !,failStream.ReadLine() + } } } } catch ex { diff --git a/src/cls/IPM/Test/YamlOutput.cls b/src/cls/IPM/Test/YamlOutput.cls index 042d28ccd..5f5b773c6 100644 --- a/src/cls/IPM/Test/YamlOutput.cls +++ b/src/cls/IPM/Test/YamlOutput.cls @@ -32,21 +32,23 @@ ClassMethod OutputToDevice( write !," passed: "_summary.assertions.passed write !," failed: "_summary.assertions.failed - if showDetail && verbose { - write ! - write !,"results:" - set resultStream = ..BuildYamlRows(tree, 0) - do resultStream.Rewind() - while 'resultStream.AtEnd { - write !,resultStream.ReadLine() - } - } elseif showDetail && (summary.methods.failed > 0) { - write ! - write !,"failures:" - set failStream = ..BuildYamlRows(tree, 1) - do failStream.Rewind() - while 'failStream.AtEnd { - write !,failStream.ReadLine() + if showDetail { + if verbose { + write ! + write !,"results:" + set resultStream = ..BuildYamlRows(tree, 0) + do resultStream.Rewind() + while 'resultStream.AtEnd { + write !,resultStream.ReadLine() + } + } elseif summary.methods.failed > 0 { + write ! + write !,"failures:" + set failStream = ..BuildYamlRows(tree, 1) + do failStream.Rewind() + while 'failStream.AtEnd { + write !,failStream.ReadLine() + } } } } catch ex { From 950ed31f914ad72c819f81d675b953d4b68ef7ba Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Apr 2026 15:59:19 -0400 Subject: [PATCH 18/18] Use null device for suppression instead --- src/cls/IPM/Lifecycle/Base.cls | 14 ++---- src/cls/IPM/ResourceProcessor/Test.cls | 12 +---- src/cls/IPM/Test/Utils.cls | 4 +- src/cls/IPM/Utils/Module.cls | 12 ----- src/cls/IPM/Utils/OutputSuppressor.cls | 68 ++++++++------------------ 5 files changed, 28 insertions(+), 82 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 7071add8a..5a3a6a528 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -1307,7 +1307,6 @@ Method %Package(ByRef pParams) As %Status [ Abstract ] Method %Verify(ByRef pParams) As %Status { set tSC = $$$OK - set capturing = 0 try { new $namespace set tInitNS = $select($namespace="%SYS": "USER", 1: $namespace) @@ -1316,10 +1315,9 @@ Method %Verify(ByRef pParams) As %Status // Suppress module install/load noise in quiet mode; ended before resource processors // run so each processor can manage its own output (Test.cls still shows summary + failures). - // BeginSuppressOutput is device-level and survives the namespace switch below. + // Null device is process-scoped, so suppression survives the namespace switch below. if explicitQuiet { - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginSuppressOutput(.tVerifyCookie)) - set capturing = 1 + set suppressor = ##class(%IPM.Utils.OutputSuppressor).%New() } if '$get(pParams("Verify","InCurrentNamespace"),0) { @@ -1405,10 +1403,7 @@ Method %Verify(ByRef pParams) As %Status do ##class(%IPM.Utils.Module).LoadDependencies(..Module, ..PhaseList, .pParams) // End suppression before resource processors run — each handles its own output. - if capturing { - $$$ThrowOnError(##class(%IPM.Utils.Module).EndSuppressOutput(tVerifyCookie)) - set capturing = 0 - } + set suppressor = "" set orderedResourceList = ..Module.GetOrderedResourceList() set tKey = "" @@ -1423,9 +1418,6 @@ Method %Verify(ByRef pParams) As %Status } } } catch e { - if capturing { - do ##class(%IPM.Utils.Module).EndSuppressOutput(tVerifyCookie) - } set tSC = e.AsStatus() } quit tSC diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 37228258f..a146f4b5b 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -112,10 +112,8 @@ Method OnPhase( set tVerbose = $get(pParams("Verbose"), 0) set tFlags = $select(tVerbose:"/display=all",1:"/display=none") set explicitQuiet = ($data(pParams("Verbose")) && (pParams("Verbose") = 0)) - set capturing = 0 if explicitQuiet { - $$$ThrowOnError(##class(%IPM.Utils.Module).BeginSuppressOutput(.cookie)) - set capturing = 1 + set suppressor = ##class(%IPM.Utils.OutputSuppressor).%New() } // Ensure unit tests and related classes are loaded. @@ -227,10 +225,7 @@ Method OnPhase( set tSC = $classmethod(outputClass,"ToFile",outputFile) $$$ThrowOnError(tSC) } - if capturing { - $$$ThrowOnError(##class(%IPM.Utils.Module).EndSuppressOutput(cookie)) - set capturing = 0 - } + set suppressor = "" set outputFormat = $get(pParams("outputformat")) if outputFormat = "" { @@ -262,9 +257,6 @@ Method OnPhase( write ! } } catch e { - if capturing { - do ##class(%IPM.Utils.Module).EndSuppressOutput(cookie) - } set tSC = e.AsStatus() } if $data(tOldUnitTestRoot,^UnitTestRoot) // Restore ^UnitTestRoot diff --git a/src/cls/IPM/Test/Utils.cls b/src/cls/IPM/Test/Utils.cls index 5355dbef0..e332634ad 100644 --- a/src/cls/IPM/Test/Utils.cls +++ b/src/cls/IPM/Test/Utils.cls @@ -211,7 +211,9 @@ ClassMethod CloseConnectionsForNamespace(pNamespace As %String) As %Status } while tProcs.%Next(.tStatus) { set tProc = ##class(SYS.Process).%OpenId(tProcs.%Get("PID")) - set tStatus = $$$ADDSC(tStatus,tProc.Terminate()) + if $isobject(tProc) { + set tStatus = $$$ADDSC(tStatus,tProc.Terminate()) + } } } catch e { set tStatus = e.AsStatus() diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls index 705be75d5..d7cbd19f3 100644 --- a/src/cls/IPM/Utils/Module.cls +++ b/src/cls/IPM/Utils/Module.cls @@ -1865,18 +1865,6 @@ getPackage(itemlist,package) ; } -/// Suppresses all output written during the enclosed block (discards it). -/// Safe to call while a BeginCaptureOutput capture is already active. -ClassMethod BeginSuppressOutput(Output pCookie As %String) As %Status -{ - quit ##class(%IPM.Utils.OutputSuppressor).Begin(.pCookie) -} - -ClassMethod EndSuppressOutput(pCookie As %String) As %Status -{ - quit ##class(%IPM.Utils.OutputSuppressor).End(pCookie) -} - /// This method enables I/O redirection (see EndCaptureOutput for retrieval). pCookie has the previous I/O redirection info. ClassMethod BeginCaptureOutput(Output pCookie As %String) As %Status [ ProcedureBlock = 0 ] { diff --git a/src/cls/IPM/Utils/OutputSuppressor.cls b/src/cls/IPM/Utils/OutputSuppressor.cls index f3ca13524..fd21d0672 100644 --- a/src/cls/IPM/Utils/OutputSuppressor.cls +++ b/src/cls/IPM/Utils/OutputSuppressor.cls @@ -1,60 +1,32 @@ -/// I/O redirect routine that discards all writes. -/// Modeled after CaptureOutput, but instead of capturing output to a variable, it simply discards it. -/// Used as the mnemonic device routine for BeginSuppressOutput / EndSuppressOutput. -/// Unlike BeginCaptureOutput, suppress does not use ^||%capture, so it is safe -/// to call while a capture is already active. -Class %IPM.Utils.OutputSuppressor +/// Suppresses all output by switching to the null device. +/// Instantiate via %New() to begin suppression; output is restored automatically +/// when the instance goes out of scope (or on error via %OnClose). +/// Safe to nest inside or outside BeginCaptureOutput. +Class %IPM.Utils.OutputSuppressor Extends %RegisteredObject { -ClassMethod Begin(Output cookie As %String) As %Status [ ProcedureBlock = 0 ] -{ - new sc,ex +Property PreviousDevice As %String [ Private ]; - #dim sc As %Status = $$$OK - #dim ex As %Exception.AbstractException +Property NullDevice As %String [ Private ]; +Method %OnNew() As %Status +{ + set ..PreviousDevice = $io + set ..NullDevice = ##class(%Library.Device).GetNullDevice() try { - if $zutil(82,12) { - set cookie=$zutil(96,12) - } else { - set cookie="" - } - - use $io::("^"_$zname) - - do $zutil(82,12,1) - - } catch (ex) { - set sc=ex.AsStatus() + open ..NullDevice + use ..NullDevice + } catch ex { + return ex.AsStatus() } - quit sc - -rstr(sz,to) [rt] public { - new rt set vr="rt" - set rd=$zutil(82,12,0) - set:$data(sz) vr=vr_"#"_sz set:$data(to) vr=vr_":"_to - read @vr - do:$data(to) $zutil(96,4,$test) - do $zutil(82,12,rd) - quit rt - } -wchr(s) public { } -wff() public { } -wnl() public { } -wstr(s) public { } -wtab(s) public { } -write(s) public { } + return $$$OK } -ClassMethod End(cookie As %String) As %Status +Method %OnClose() As %Status { - if cookie '= "" { - use $io::("^"_cookie) - } else { - do $zutil(82,12,0) - use $io::("") - } - quit $$$OK + close ..NullDevice + use ..PreviousDevice + return $$$OK } }