From fc6922bc3f3530255c113b8612bd29eee5c8b6c9 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Thu, 30 Jan 2025 12:01:49 -0600 Subject: [PATCH 01/49] SimDB / v3 integration --- .gitmodules | 3 + sparta/.gitignore | 2 +- sparta/CMakeLists.txt | 16 +- sparta/cmake/FindSparta.cmake | 17 +- sparta/cmake/SimdbTestingMacros.cmake | 63 - sparta/cmake/sparta-config.cmake | 2 +- sparta/doc/sparta_docs/framework_devel.txt | 3 - sparta/example/CoreModel/CMakeLists.txt | 34 +- sparta/example/CoreModel/src/BIU.hpp | 1 - sparta/example/CoreModel/src/ExampleInst.hpp | 50 + .../CoreModel/src/ExampleSimulation.cpp | 559 +----- .../CoreModel/src/ExampleSimulation.hpp | 28 - sparta/example/CoreModel/src/Execute.cpp | 7 +- sparta/example/CoreModel/src/Execute.hpp | 17 +- sparta/example/CoreModel/src/Fetch.cpp | 3 +- sparta/example/CoreModel/src/Fetch.hpp | 4 - sparta/example/CoreModel/src/LSU.cpp | 4 +- sparta/example/CoreModel/src/LSU.hpp | 305 +-- .../CoreModel/src/LoadStoreInstInfo.hpp | 228 +++ sparta/example/CoreModel/src/MSS.hpp | 1 - .../example/CoreModel/src/MemAccessInfo.hpp | 205 ++ sparta/example/CoreModel/src/Rename.cpp | 20 - sparta/example/CoreModel/src/Rename.hpp | 1 - sparta/example/CoreModel/src/main.cpp | 9 +- .../communication/CMakeLists.txt | 2 +- .../DynamicModelPipeline/CMakeLists.txt | 26 +- .../src/ExampleSimulation.cpp | 361 ---- .../src/ExampleSimulation.hpp | 8 - .../sparta_support/PythonInterpreter.cpp | 46 - .../sparta_support/PythonInterpreter.hpp | 9 - .../python/sparta_support/module_sparta.cpp | 672 ------- .../python/sparta_support/module_sparta.hpp | 58 - sparta/simdb | 1 + sparta/simdb/CMakeLists.txt | 24 - sparta/simdb/cmake/simdb-config.cmake | 13 - sparta/simdb/include/simdb/Constraints.hpp | 46 - sparta/simdb/include/simdb/DbConnProxy.hpp | 242 --- sparta/simdb/include/simdb/Errors.hpp | 107 - sparta/simdb/include/simdb/ObjectFactory.hpp | 35 - sparta/simdb/include/simdb/ObjectManager.hpp | 635 ------ sparta/simdb/include/simdb/ObjectRef.hpp | 207 -- sparta/simdb/include/simdb/TableProxy.hpp | 79 - sparta/simdb/include/simdb/TableRef.hpp | 637 ------ .../include/simdb/async/AsyncTaskEval.hpp | 630 ------ .../include/simdb/async/ConcurrentQueue.hpp | 58 - .../simdb/include/simdb/async/TimerThread.hpp | 246 --- .../include/simdb/impl/hdf5/DataTypeUtils.hpp | 228 --- .../include/simdb/impl/hdf5/HDF5ConnProxy.hpp | 223 --- .../include/simdb/impl/hdf5/Resources.hpp | 188 -- .../include/simdb/impl/sqlite/Errors.hpp | 21 - .../simdb/impl/sqlite/SQLiteConnProxy.hpp | 166 -- .../include/simdb/impl/sqlite/Schema.hpp | 48 - .../simdb/impl/sqlite/TransactionUtils.hpp | 112 -- .../simdb/schema/ColumnMetaStructs.hpp | 299 --- .../include/simdb/schema/ColumnTypedefs.hpp | 42 - .../include/simdb/schema/ColumnValue.hpp | 325 --- .../simdb/schema/ColumnValueContainer.hpp | 302 --- .../include/simdb/schema/DatabaseRoot.hpp | 650 ------ .../include/simdb/schema/DatabaseTypedefs.hpp | 11 - .../simdb/schema/GeneralMetaStructs.hpp | 80 - .../include/simdb/schema/ObjectProxy.hpp | 99 - sparta/simdb/include/simdb/schema/Schema.hpp | 1061 ---------- .../include/simdb/schema/TableSummaries.hpp | 83 - .../include/simdb/schema/TableTypedefs.hpp | 92 - .../simdb/include/simdb/utils/BlobHelpers.hpp | 84 - .../simdb/include/simdb/utils/CompatUtils.hpp | 19 - .../simdb/include/simdb/utils/MathUtils.hpp | 62 - .../simdb/include/simdb/utils/ObjectQuery.hpp | 807 -------- .../simdb/include/simdb/utils/StringUtils.hpp | 206 -- .../include/simdb/utils/Stringifiers.hpp | 103 - sparta/simdb/include/simdb/utils/uuids.hpp | 46 - sparta/simdb/include/simdb_fwd.hpp | 42 - sparta/simdb/src/HDF5Connection.cpp | 1099 ---------- sparta/simdb/src/ObjectManager.cpp | 868 -------- sparta/simdb/src/ObjectRef.cpp | 397 ---- sparta/simdb/src/SQLiteConnection.cpp | 1065 ---------- sparta/simdb/src/TableRef.cpp | 398 ---- sparta/simdb/src/simdb.cpp | 18 - sparta/simdb/test/CMakeLists.txt | 44 - sparta/simdb/test/Colors.hpp | 159 -- sparta/simdb/test/CoreDatabase/CMakeLists.txt | 8 - .../test/CoreDatabase/CoreDatabase_test.cpp | 769 ------- .../CoreDatabase/test_dbs/placeholder.txt | 4 - sparta/simdb/test/HDF5Database/CMakeLists.txt | 15 - .../test/HDF5Database/HDF5Database_test.cpp | 625 ------ .../HDF5Database/test_dbs/placeholder.txt | 4 - .../simdb/test/SQLiteDatabase/CMakeLists.txt | 9 - .../SQLiteDatabase/SQLiteDatabase_test.cpp | 1761 ----------------- sparta/simdb/test/SQLiteDatabase/sample.db | Bin 278528 -> 0 bytes .../SQLiteDatabase/test_dbs/placeholder.txt | 4 - sparta/simdb/test/SharedDB/CMakeLists.txt | 9 - sparta/simdb/test/SharedDB/SharedDB_test.cpp | 359 ---- .../test/SharedDB/test_dbs/placeholder.txt | 4 - sparta/simdb/test/SimDBTester.hpp | 862 -------- sparta/simdb/test/Thread/CMakeLists.txt | 9 - .../test/Thread/StandaloneCpp1/CMakeLists.txt | 7 - .../StandaloneCpp1/StandaloneThread_test1.cpp | 72 - sparta/simdb/test/Thread/Thread_test.cpp | 243 --- sparta/simdb/test/Utils/CMakeLists.txt | 7 - sparta/simdb/test/Utils/Utils_test.cpp | 68 - sparta/sparta/app/AppTriggers.hpp | 106 +- sparta/sparta/app/CommandLineSimulator.hpp | 12 - sparta/sparta/app/ReportDescriptor.hpp | 120 +- sparta/sparta/app/Simulation.hpp | 94 - sparta/sparta/app/SimulationConfiguration.hpp | 172 -- sparta/sparta/app/SimulationInfo.hpp | 63 +- .../sparta/async/AsyncNonTimeseriesReport.hpp | 247 --- sparta/sparta/async/AsyncTimeseriesReport.hpp | 743 ------- sparta/sparta/collection/Collectable.hpp | 894 --------- .../sparta/collection/CollectableTreeNode.hpp | 221 +-- sparta/sparta/collection/CollectionPoints.hpp | 236 +++ sparta/sparta/collection/Collector.hpp | 52 - .../sparta/collection/DelayedCollectable.hpp | 183 -- .../sparta/collection/IterableCollector.hpp | 352 ---- .../sparta/collection/PipelineCollector.hpp | 706 ++----- .../sparta/control/TemporaryRunController.hpp | 12 - sparta/sparta/kernel/Scheduler.hpp | 4 + sparta/sparta/pairs/SpartaKeyPairs.hpp | 4 +- sparta/sparta/pipeViewer/ClockFileWriter.hpp | 132 -- .../sparta/pipeViewer/InformationWriter.hpp | 79 - .../sparta/pipeViewer/LocationFileWriter.hpp | 191 -- sparta/sparta/pipeViewer/Outputter.hpp | 262 --- .../pipeViewer/transaction_structures.hpp | 149 -- sparta/sparta/ports/DataPort.hpp | 17 +- sparta/sparta/ports/SyncPort.hpp | 34 +- sparta/sparta/report/DatabaseInterface.hpp | 387 ---- sparta/sparta/report/Report.hpp | 194 +- .../report/db/DatabaseContextCounter.hpp | 155 -- sparta/sparta/report/db/ReportHeader.hpp | 116 -- .../sparta/report/db/ReportNodeHierarchy.hpp | 484 ----- sparta/sparta/report/db/ReportTimeseries.hpp | 167 -- sparta/sparta/report/db/ReportVerifier.hpp | 163 -- sparta/sparta/report/db/Schema.hpp | 33 - sparta/sparta/report/db/SimInfoSerializer.hpp | 182 -- .../sparta/report/db/SingleUpdateReport.hpp | 79 - .../sparta/report/db/StatInstRowIterator.hpp | 202 -- .../sparta/report/db/StatInstValueLookup.hpp | 160 -- sparta/sparta/report/db/format/toCSV.hpp | 144 -- sparta/sparta/report/format/JSON_detail.hpp | 78 +- .../sparta/resources/AgedArrayCollector.hpp | 71 - sparta/sparta/resources/Array.hpp | 22 +- sparta/sparta/resources/Buffer.hpp | 14 +- sparta/sparta/resources/CircularBuffer.hpp | 9 +- sparta/sparta/resources/Pipe.hpp | 17 +- sparta/sparta/resources/Pipeline.hpp | 1 - sparta/sparta/resources/Queue.hpp | 11 +- sparta/sparta/simulation/Clock.hpp | 24 - sparta/sparta/simulation/Resource.hpp | 1 - .../sparta/statistics/StatisticInstance.hpp | 89 +- .../sparta/statistics/db/SINodeHierarchy.hpp | 62 - .../sparta/statistics/db/SIValuesBuffer.hpp | 348 ---- sparta/sparta/utils/MathUtils.hpp | 52 + sparta/sparta/utils/MetaStructs.hpp | 13 + sparta/sparta/utils/SpartaSharedPointer.hpp | 33 + sparta/sparta/utils/SpartaTester.hpp | 4 +- sparta/src/AsyncNonTimeseriesReport.cpp | 53 - sparta/src/AsyncTimeseriesReport.cpp | 57 - sparta/src/Clock.cpp | 45 - sparta/src/CommandLineSimulator.cpp | 177 +- sparta/src/DatabaseContextCounter.cpp | 318 --- sparta/src/DatabaseSchema.cpp | 642 ------ sparta/src/ExpressionGrammar.cpp | 1 - sparta/src/ExpressionTrigger.cpp | 71 +- sparta/src/JsonFormatter.cpp | 109 - sparta/src/Report.cpp | 945 --------- sparta/src/ReportDescriptor.cpp | 311 +-- sparta/src/ReportHeader.cpp | 195 -- sparta/src/ReportRepository.cpp | 177 -- sparta/src/ReportTimeseries.cpp | 943 --------- sparta/src/ReportVerifier.cpp | 582 ------ sparta/src/SINodeHierarchy.cpp | 192 -- sparta/src/Simulation.cpp | 508 +---- sparta/src/SimulationInfo.cpp | 80 - sparta/src/SingleUpdateReport.cpp | 154 -- sparta/src/StatisticInstance.cpp | 228 +-- sparta/src/TemporaryRunController.cpp | 12 +- sparta/test/Array/Array_test.cpp | 37 +- sparta/test/Buffer/Buffer_test.cpp | 20 +- sparta/test/CMakeLists.txt | 4 - .../CircularBuffer/CircularBuffer_test.cpp | 4 - sparta/test/Collection/CMakeLists.txt | 7 - sparta/test/Collection/Collection_test.cpp | 72 - .../CommandLineSimulator_test.cpp | 100 - sparta/test/PairCollector/CMakeLists.txt | 3 - .../test/PairCollector/Collectable_test.cpp | 579 ------ sparta/test/Pipe/Pipe_test.cpp | 20 +- sparta/test/Pipeline/Pipeline_test.cpp | 33 +- sparta/test/Port/Port_test.cpp | 3 - sparta/test/Queue/Queue_test.cpp | 20 +- sparta/test/ReportVerifier/CMakeLists.txt | 5 - sparta/test/ReportVerifier/ReportVerifier.cpp | 159 -- sparta/test/SimDB/CMakeLists.txt | 7 - sparta/test/SimDB/SimDB_test.cpp | 675 ------- sparta/test/SimDB/sample.db | Bin 1572864 -> 0 bytes sparta/test/SyncPort/SyncPort_test.cpp | 20 +- .../TransactionDatabaseAPI_main.cpp | 2 - 196 files changed, 1292 insertions(+), 34688 deletions(-) create mode 100644 .gitmodules delete mode 100644 sparta/cmake/SimdbTestingMacros.cmake create mode 100644 sparta/example/CoreModel/src/LoadStoreInstInfo.hpp create mode 100644 sparta/example/CoreModel/src/MemAccessInfo.hpp create mode 160000 sparta/simdb delete mode 100644 sparta/simdb/CMakeLists.txt delete mode 100644 sparta/simdb/cmake/simdb-config.cmake delete mode 100644 sparta/simdb/include/simdb/Constraints.hpp delete mode 100644 sparta/simdb/include/simdb/DbConnProxy.hpp delete mode 100644 sparta/simdb/include/simdb/Errors.hpp delete mode 100644 sparta/simdb/include/simdb/ObjectFactory.hpp delete mode 100644 sparta/simdb/include/simdb/ObjectManager.hpp delete mode 100644 sparta/simdb/include/simdb/ObjectRef.hpp delete mode 100644 sparta/simdb/include/simdb/TableProxy.hpp delete mode 100644 sparta/simdb/include/simdb/TableRef.hpp delete mode 100644 sparta/simdb/include/simdb/async/AsyncTaskEval.hpp delete mode 100644 sparta/simdb/include/simdb/async/ConcurrentQueue.hpp delete mode 100644 sparta/simdb/include/simdb/async/TimerThread.hpp delete mode 100644 sparta/simdb/include/simdb/impl/hdf5/DataTypeUtils.hpp delete mode 100644 sparta/simdb/include/simdb/impl/hdf5/HDF5ConnProxy.hpp delete mode 100644 sparta/simdb/include/simdb/impl/hdf5/Resources.hpp delete mode 100644 sparta/simdb/include/simdb/impl/sqlite/Errors.hpp delete mode 100644 sparta/simdb/include/simdb/impl/sqlite/SQLiteConnProxy.hpp delete mode 100644 sparta/simdb/include/simdb/impl/sqlite/Schema.hpp delete mode 100644 sparta/simdb/include/simdb/impl/sqlite/TransactionUtils.hpp delete mode 100644 sparta/simdb/include/simdb/schema/ColumnMetaStructs.hpp delete mode 100644 sparta/simdb/include/simdb/schema/ColumnTypedefs.hpp delete mode 100644 sparta/simdb/include/simdb/schema/ColumnValue.hpp delete mode 100644 sparta/simdb/include/simdb/schema/ColumnValueContainer.hpp delete mode 100644 sparta/simdb/include/simdb/schema/DatabaseRoot.hpp delete mode 100644 sparta/simdb/include/simdb/schema/DatabaseTypedefs.hpp delete mode 100644 sparta/simdb/include/simdb/schema/GeneralMetaStructs.hpp delete mode 100644 sparta/simdb/include/simdb/schema/ObjectProxy.hpp delete mode 100644 sparta/simdb/include/simdb/schema/Schema.hpp delete mode 100644 sparta/simdb/include/simdb/schema/TableSummaries.hpp delete mode 100644 sparta/simdb/include/simdb/schema/TableTypedefs.hpp delete mode 100644 sparta/simdb/include/simdb/utils/BlobHelpers.hpp delete mode 100644 sparta/simdb/include/simdb/utils/CompatUtils.hpp delete mode 100644 sparta/simdb/include/simdb/utils/MathUtils.hpp delete mode 100644 sparta/simdb/include/simdb/utils/ObjectQuery.hpp delete mode 100644 sparta/simdb/include/simdb/utils/StringUtils.hpp delete mode 100644 sparta/simdb/include/simdb/utils/Stringifiers.hpp delete mode 100644 sparta/simdb/include/simdb/utils/uuids.hpp delete mode 100644 sparta/simdb/include/simdb_fwd.hpp delete mode 100644 sparta/simdb/src/HDF5Connection.cpp delete mode 100644 sparta/simdb/src/ObjectManager.cpp delete mode 100644 sparta/simdb/src/ObjectRef.cpp delete mode 100644 sparta/simdb/src/SQLiteConnection.cpp delete mode 100644 sparta/simdb/src/TableRef.cpp delete mode 100644 sparta/simdb/src/simdb.cpp delete mode 100644 sparta/simdb/test/CMakeLists.txt delete mode 100644 sparta/simdb/test/Colors.hpp delete mode 100644 sparta/simdb/test/CoreDatabase/CMakeLists.txt delete mode 100644 sparta/simdb/test/CoreDatabase/CoreDatabase_test.cpp delete mode 100644 sparta/simdb/test/CoreDatabase/test_dbs/placeholder.txt delete mode 100644 sparta/simdb/test/HDF5Database/CMakeLists.txt delete mode 100644 sparta/simdb/test/HDF5Database/HDF5Database_test.cpp delete mode 100644 sparta/simdb/test/HDF5Database/test_dbs/placeholder.txt delete mode 100644 sparta/simdb/test/SQLiteDatabase/CMakeLists.txt delete mode 100644 sparta/simdb/test/SQLiteDatabase/SQLiteDatabase_test.cpp delete mode 100644 sparta/simdb/test/SQLiteDatabase/sample.db delete mode 100644 sparta/simdb/test/SQLiteDatabase/test_dbs/placeholder.txt delete mode 100644 sparta/simdb/test/SharedDB/CMakeLists.txt delete mode 100644 sparta/simdb/test/SharedDB/SharedDB_test.cpp delete mode 100644 sparta/simdb/test/SharedDB/test_dbs/placeholder.txt delete mode 100644 sparta/simdb/test/SimDBTester.hpp delete mode 100644 sparta/simdb/test/Thread/CMakeLists.txt delete mode 100644 sparta/simdb/test/Thread/StandaloneCpp1/CMakeLists.txt delete mode 100644 sparta/simdb/test/Thread/StandaloneCpp1/StandaloneThread_test1.cpp delete mode 100644 sparta/simdb/test/Thread/Thread_test.cpp delete mode 100644 sparta/simdb/test/Utils/CMakeLists.txt delete mode 100644 sparta/simdb/test/Utils/Utils_test.cpp delete mode 100644 sparta/sparta/async/AsyncNonTimeseriesReport.hpp delete mode 100644 sparta/sparta/async/AsyncTimeseriesReport.hpp delete mode 100644 sparta/sparta/collection/Collectable.hpp create mode 100644 sparta/sparta/collection/CollectionPoints.hpp delete mode 100644 sparta/sparta/collection/Collector.hpp delete mode 100644 sparta/sparta/collection/DelayedCollectable.hpp delete mode 100644 sparta/sparta/collection/IterableCollector.hpp delete mode 100644 sparta/sparta/pipeViewer/ClockFileWriter.hpp delete mode 100644 sparta/sparta/pipeViewer/InformationWriter.hpp delete mode 100644 sparta/sparta/pipeViewer/LocationFileWriter.hpp delete mode 100644 sparta/sparta/pipeViewer/Outputter.hpp delete mode 100644 sparta/sparta/pipeViewer/transaction_structures.hpp delete mode 100644 sparta/sparta/report/DatabaseInterface.hpp delete mode 100644 sparta/sparta/report/db/DatabaseContextCounter.hpp delete mode 100644 sparta/sparta/report/db/ReportHeader.hpp delete mode 100644 sparta/sparta/report/db/ReportNodeHierarchy.hpp delete mode 100644 sparta/sparta/report/db/ReportTimeseries.hpp delete mode 100644 sparta/sparta/report/db/ReportVerifier.hpp delete mode 100644 sparta/sparta/report/db/Schema.hpp delete mode 100644 sparta/sparta/report/db/SimInfoSerializer.hpp delete mode 100644 sparta/sparta/report/db/SingleUpdateReport.hpp delete mode 100644 sparta/sparta/report/db/StatInstRowIterator.hpp delete mode 100644 sparta/sparta/report/db/StatInstValueLookup.hpp delete mode 100644 sparta/sparta/report/db/format/toCSV.hpp delete mode 100644 sparta/sparta/resources/AgedArrayCollector.hpp delete mode 100644 sparta/sparta/statistics/db/SINodeHierarchy.hpp delete mode 100644 sparta/sparta/statistics/db/SIValuesBuffer.hpp delete mode 100644 sparta/src/AsyncNonTimeseriesReport.cpp delete mode 100644 sparta/src/AsyncTimeseriesReport.cpp delete mode 100644 sparta/src/DatabaseContextCounter.cpp delete mode 100644 sparta/src/DatabaseSchema.cpp delete mode 100644 sparta/src/ReportHeader.cpp delete mode 100644 sparta/src/ReportTimeseries.cpp delete mode 100644 sparta/src/ReportVerifier.cpp delete mode 100644 sparta/src/SINodeHierarchy.cpp delete mode 100644 sparta/src/SingleUpdateReport.cpp delete mode 100644 sparta/test/Collection/CMakeLists.txt delete mode 100644 sparta/test/Collection/Collection_test.cpp delete mode 100644 sparta/test/PairCollector/CMakeLists.txt delete mode 100644 sparta/test/PairCollector/Collectable_test.cpp delete mode 100644 sparta/test/ReportVerifier/CMakeLists.txt delete mode 100644 sparta/test/ReportVerifier/ReportVerifier.cpp delete mode 100644 sparta/test/SimDB/CMakeLists.txt delete mode 100644 sparta/test/SimDB/SimDB_test.cpp delete mode 100644 sparta/test/SimDB/sample.db diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..7c58c0f42c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sparta/simdb"] + path = sparta/simdb + url = git@github.com:colbynyce-mips/simdb.git diff --git a/sparta/.gitignore b/sparta/.gitignore index 0f83e90cef..09ca92f5d8 100644 --- a/sparta/.gitignore +++ b/sparta/.gitignore @@ -12,4 +12,4 @@ build* cmake-build-* *~ compile_commands.json -.vscode \ No newline at end of file +.vscode diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index cf02d09cf3..ebd42f7698 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -13,8 +13,6 @@ include (${SPARTA_BASE}/cmake/sparta-config.cmake) # Add the source for libsparta.a list (APPEND SourceCppFiles src/ArgosOutputter.cpp - src/AsyncTimeseriesReport.cpp - src/AsyncNonTimeseriesReport.cpp src/Backtrace.cpp src/BaseFormatter.cpp src/Clock.cpp @@ -26,8 +24,6 @@ list (APPEND SourceCppFiles src/CounterBase.cpp src/CsvFormatter.cpp src/DAG.cpp - src/DatabaseContextCounter.cpp - src/DatabaseSchema.cpp src/Destination.cpp src/EdgeFactory.cpp src/EventNode.cpp @@ -45,10 +41,7 @@ list (APPEND SourceCppFiles src/RegisterSet.cpp src/Report.cpp src/ReportDescriptor.cpp - src/ReportHeader.cpp src/ReportRepository.cpp - src/ReportTimeseries.cpp - src/ReportVerifier.cpp src/Resource.cpp src/SpartaException.cpp src/RootTreeNode.cpp @@ -58,8 +51,6 @@ list (APPEND SourceCppFiles src/Simulation.cpp src/SimulationConfiguration.cpp src/SimulationInfo.cpp - src/SingleUpdateReport.cpp - src/SINodeHierarchy.cpp src/StatisticDef.cpp src/StatisticInstance.cpp src/StatisticsArchives.cpp @@ -160,7 +151,7 @@ if(DEFINED SPARTA_CXX_FLAGS_DEBUG AND SPARTA_CXX_FLAGS_DEBUG) set(CMAKE_CXX_FLAGS_DEBUG "${SPARTA_CXX_FLAGS_DEBUG}") message(STATUS "Using Sparta custom debug flags: ${CMAKE_CXX_FLAGS_DEBUG}") else() - set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3") + set(CMAKE_CXX_FLAGS_DEBUG "-U_FORTIFY_SOURCE -O0 -g3") message(STATUS "Using Sparta default debug flags: ${CMAKE_CXX_FLAGS_DEBUG}") endif() # @@ -216,14 +207,11 @@ add_library (sparta ${SourceCppFiles}) # Add local includes target_include_directories (sparta PUBLIC "./") -target_include_directories (sparta PUBLIC "simdb/include") +target_include_directories (sparta PUBLIC "./simdb/include") set (SPARTA_STATIC_LIBS ${PROJECT_BINARY_DIR}/libsparta.a) set (SPARTA_CMAKE_MACRO_PATH ${SPARTA_BASE}/cmake) -# Build the SimDB library -add_subdirectory (simdb) - # # Testing, examples, and tools # diff --git a/sparta/cmake/FindSparta.cmake b/sparta/cmake/FindSparta.cmake index 3967a8b071..9035f58e0f 100644 --- a/sparta/cmake/FindSparta.cmake +++ b/sparta/cmake/FindSparta.cmake @@ -2,7 +2,6 @@ include(FindPackageHandleStandardArgs) if(NOT SPARTA_FOUND) find_package(Boost REQUIRED COMPONENTS timer filesystem serialization program_options) - find_package(HDF5 REQUIRED COMPONENTS CXX) find_package(SQLite3 REQUIRED) find_package(ZLIB REQUIRED) find_package(yaml-cpp REQUIRED) @@ -20,14 +19,7 @@ if(NOT SPARTA_FOUND) HINTS ENV SPARTA_INSTALL_HOME PATH_SUFFIXES include) - find_path(SIMDB_INCLUDE_DIRS simdb/ObjectManager.hpp - HINTS ${SPARTA_INCLUDE_DIR} ${SPARTA_SEARCH_DIR} - HINTS ENV CPATH - HINTS ENV SPARTA_INSTALL_HOME - PATH_SUFFIXES include) - list(APPEND SPARTA_INCLUDE_DIRS ${SIMDB_INCLUDE_DIRS}) - - set(SPARTA_SEARCH_COMPOMPONENTS simdb sparta) + set(SPARTA_SEARCH_COMPOMPONENTS sparta) foreach(_comp ${SPARTA_SEARCH_COMPOMPONENTS}) # Search for the libraries find_library(SPARTA_${_comp}_LIBRARY ${_comp} @@ -62,8 +54,6 @@ if(NOT SPARTA_FOUND) if(NOT CMAKE_VERSION VERSION_LESS 3.0 AND SPARTA_FOUND) add_library(SPARTA::libsparta STATIC IMPORTED) set_property(TARGET SPARTA::libsparta PROPERTY IMPORTED_LOCATION "${SPARTA_sparta_LIBRARY}") - add_library(SPARTA::libsimdb STATIC IMPORTED) - set_property(TARGET SPARTA::libsimdb PROPERTY IMPORTED_LOCATION "${SPARTA_simdb_LIBRARY}") add_library(SPARTA::sparta INTERFACE IMPORTED) # Workaround as per https://github.com/jbeder/yaml-cpp/issues/774#issuecomment-927357017 @@ -74,9 +64,9 @@ if(NOT SPARTA_FOUND) get_target_property(YAML_CPP_INCLUDE_DIR yaml-cpp::yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) endif () set_property(TARGET SPARTA::sparta - PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPARTA_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${HDF5_CXX_INCLUDE_DIRS} ${RAPIDJSON_INCLUDE_DIR} ${RapidJSON_INCLUDE_DIR} ${YAML_CPP_INCLUDE_DIR}) + PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPARTA_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${RAPIDJSON_INCLUDE_DIR} ${RapidJSON_INCLUDE_DIR} ${YAML_CPP_INCLUDE_DIR}) set_property(TARGET SPARTA::sparta - PROPERTY INTERFACE_LINK_LIBRARIES SPARTA::libsparta SPARTA::libsimdb HDF5::HDF5 SQLite::SQLite3 + PROPERTY INTERFACE_LINK_LIBRARIES SPARTA::libsparta SQLite::SQLite3 Boost::filesystem Boost::serialization Boost::timer Boost::program_options ZLIB::ZLIB yaml-cpp Threads::Threads) @@ -87,7 +77,6 @@ if(NOT SPARTA_FOUND) set_property(TARGET SPARTA::sparta PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_17) include(${CMAKE_CURRENT_LIST_DIR}/SpartaTestingMacros.cmake) - include(${CMAKE_CURRENT_LIST_DIR}/SimdbTestingMacros.cmake) set(SPARTA_FOUND TRUE) endif() diff --git a/sparta/cmake/SimdbTestingMacros.cmake b/sparta/cmake/SimdbTestingMacros.cmake deleted file mode 100644 index c23a6a7475..0000000000 --- a/sparta/cmake/SimdbTestingMacros.cmake +++ /dev/null @@ -1,63 +0,0 @@ -# -# Testing macros as well as setting up Valgrind options -# - -# MACROS for adding to the targets. Use these to add your tests. - -# simdb_regress enforces that your binary gets built as part of the -# regression commands. -macro (simdb_regress target) - add_dependencies(simdb_regress ${target} ) - add_dependencies(simdb_regress_valgrind ${target}) -endmacro(simdb_regress) - -# A function to add a simdb test with various options -function(simdb_fully_named_test name target run_valgrind) - add_test (NAME ${name} COMMAND $ ${ARGN}) - simdb_regress(${target}) - # Only add a valgrind test if desired. - # The older 2.6 version of cmake has issues with this. - # will ignore them for now. - if (VALGRIND_REGRESS_ENABLED) - if (run_valgrind) - add_test (NAME valgrind_${name} COMMAND valgrind - ${VALGRIND_OPTS} $ ${ARGN}) - set_tests_properties(valgrind_${name} PROPERTIES LABELS ${VALGRIND_TEST_LABEL}) - endif() - endif() - target_link_libraries (${target} ${SimDB_LIBS}) - target_include_directories (${target} PUBLIC "${SPARTA_BASE}") -endfunction (simdb_fully_named_test) - -# Tell simdb to run the following target with the following name. -macro(simdb_named_test name target) - simdb_fully_named_test(${name} ${target} TRUE ${ARGN}) -endmacro(simdb_named_test) - -# Run the test without a valgrind test. -# This should only be used for special tests, and not an excuse to avoid -# fixing memory issues! -macro (simdb_named_test_no_valgrind name target) - simdb_fully_named_test(${name} ${target} FALSE ${ARGN}) -endmacro (simdb_named_test_no_valgrind) - -# Just add the executable to the testing using defaults. -macro (simdb_test target) - simdb_named_test(${target} ${target} ${ARGN}) -endmacro (simdb_test) - -# Define a macro for copying required files to be along side the build -# files. This is useful for golden outputs in simdb's tests that need -# to be copied to the build directory. -macro (simdb_copy build_target cp_file) - add_custom_command(TARGET ${build_target} PRE_BUILD - COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/${cp_file} ${CMAKE_CURRENT_BINARY_DIR}/) -endmacro(simdb_copy) - -# Define a macro for recursively copying required files to be along -# side the build files. This is useful for golden outputs in simdb's -# tests that need to be copied to the build directory. -macro (simdb_recursive_copy build_target cp_file) - add_custom_command(TARGET ${build_target} PRE_BUILD - COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/${cp_file} ${CMAKE_CURRENT_BINARY_DIR}/) -endmacro(simdb_recursive_copy) diff --git a/sparta/cmake/sparta-config.cmake b/sparta/cmake/sparta-config.cmake index 653129639d..e2b234d283 100644 --- a/sparta/cmake/sparta-config.cmake +++ b/sparta/cmake/sparta-config.cmake @@ -81,7 +81,7 @@ message (STATUS "Using HDF5 ${HDF5_VERSION}") # Populate the Sparta_LIBS variable with the required libraries for # basic Sparta linking -set (Sparta_LIBS sparta simdb HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread +set (Sparta_LIBS sparta HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread Boost::date_time Boost::iostreams Boost::serialization Boost::timer Boost::program_options) # On Linux we need to link against rt as well diff --git a/sparta/doc/sparta_docs/framework_devel.txt b/sparta/doc/sparta_docs/framework_devel.txt index 0b2fc3c5bf..12a7e7f745 100644 --- a/sparta/doc/sparta_docs/framework_devel.txt +++ b/sparta/doc/sparta_docs/framework_devel.txt @@ -33,9 +33,6 @@ # Example regression make example_regress - - # SimDB regression - make simdb_regress \endcode \section doxygen_convention Doxygen Convention diff --git a/sparta/example/CoreModel/CMakeLists.txt b/sparta/example/CoreModel/CMakeLists.txt index d501a336f8..da5c1feb55 100644 --- a/sparta/example/CoreModel/CMakeLists.txt +++ b/sparta/example/CoreModel/CMakeLists.txt @@ -59,8 +59,8 @@ sparta_regress(sparta_core_example) sparta_named_test(sparta_core_example_20000cycs sparta_core_example -r 200000) sparta_named_test(sparta_core_example_fetch_max_ipc sparta_core_example -i 1M -p top.cpu.core0.fetch.params.fetch_max_ipc true) sparta_named_test(sparta_core_example_pipeout sparta_core_example -i 1000 -z pipeout -K layouts/cpu_layout.alf) -sparta_named_test(sparta_core_example_pipeout_icount sparta_core_example -i 1000 --debug-on-icount 100 -z pipeout -K layouts/cpu_layout.alf) -sparta_named_test(sparta_core_example_pipeout_cycle sparta_core_example -i 1000 --debug-on 100 -z pipeout -K layouts/cpu_layout.alf) +sparta_named_test(sparta_core_example_pipeout_icount sparta_core_example -i 1000 --debug-on-icount 100 -z pipeout_icount -K layouts/cpu_layout.alf) +sparta_named_test(sparta_core_example_pipeout_cycle sparta_core_example -i 1000 --debug-on 100 -z pipeout_cycle -K layouts/cpu_layout.alf) # Using a standard YAML causes this test to fail. Need to investigate Issue #4 # sparta_named_test(sparta_core_example_preload sparta_core_example -i 1000 -p top.cpu.core0.preloader.params.preload_file sample_preload.yaml) sparta_named_test(sparta_core_example_multicore_yaml_opts_smoke_test sparta_core_example -i 10k --report top multicore_report_opts_basic.yaml out.txt --num-cores 2) @@ -83,11 +83,11 @@ sparta_named_test(sparta_core_example_simultaneous_report_yamls sparta_core_exam sparta_named_test(sparta_core_example_simulation_control sparta_core_example -i 10k --control ctrl.yaml) sparta_named_test(sparta_core_example_simulation_control_separate_files sparta_core_example -i 10k --control ctrl_builtin.yaml --control ctrl_custom.yaml) sparta_named_test(sparta_core_example_notif_start_trigger_with_update_count sparta_core_example -i 10k --report notif_start_trigger_with_update_count.yaml) -sparta_named_test(sparta_core_example_toggle_triggers sparta_core_example -i 100k --report toggle_expression_triggers.yaml --feature simdb 0) -sparta_named_test(sparta_core_example_parameterized_toggle_triggers sparta_core_example -i 10k --report toggle_parameterized_expression_triggers.yaml --parameter top.cpu.core0.params.foo 1 --feature simdb 0) -sparta_named_test(sparta_core_example_skipped_update_csv_rows sparta_core_example -i 50k --report toggle_trigger_skipping_csv_rows.yaml --feature simdb 0) -sparta_named_test(sparta_core_example_toggle_and_start_triggers_together sparta_core_example -i 50k --report toggle_and_start_triggers_together.yaml --feature simdb 0) -sparta_named_test(sparta_core_example_toggle_trigger_enabled_and_disabled_at_same_time sparta_core_example -i 50k --report toggle_trigger_enabled_disabled_same_time.yaml --feature simdb 0) +sparta_named_test(sparta_core_example_toggle_triggers sparta_core_example -i 100k --report toggle_expression_triggers.yaml) +sparta_named_test(sparta_core_example_parameterized_toggle_triggers sparta_core_example -i 10k --report toggle_parameterized_expression_triggers.yaml --parameter top.cpu.core0.params.foo 1) +sparta_named_test(sparta_core_example_skipped_update_csv_rows sparta_core_example -i 50k --report toggle_trigger_skipping_csv_rows.yaml) +sparta_named_test(sparta_core_example_toggle_and_start_triggers_together sparta_core_example -i 50k --report toggle_and_start_triggers_together.yaml) +sparta_named_test(sparta_core_example_toggle_trigger_enabled_and_disabled_at_same_time sparta_core_example -i 50k --report toggle_trigger_enabled_disabled_same_time.yaml) sparta_named_test(sparta_core_example_bogus_nans_in_timeseries_reports sparta_core_example -i 50k --report tagged_notif_based_start_trigger.yaml) sparta_named_test(sparta_core_example_all_descriptor_patterns sparta_core_example -i 50k --report all_descriptor_patterns.yaml --num-cores 2) sparta_named_test(sparta_core_example_retired_inst_path sparta_core_example -i 10k --report all_descriptor_patterns.yaml --retired-inst-counter-path rename.stats.rename_uop_queue_utilization_count0 --num-cores 2) @@ -96,7 +96,7 @@ sparta_named_test(sparta_core_example_two_reports_json_full_format sparta_core_e sparta_named_test(sparta_core_example_two_reports_json_detail_format sparta_core_example -i 10k --report multireports_json_detail.yaml) sparta_named_test(sparta_core_example_two_reports_json_reduced_format sparta_core_example -i 10k --report multireports_json_reduced.yaml) sparta_named_test(sparta_core_example_tree_node_extensions_written_to_final_config sparta_core_example -i 10k --arch-search-dir . --arch extensions_in_arch_file.yaml --config-file extensions_in_config_file.yaml --extension-file tree_node_extensions.yaml --write-final-config final_cfg_with_extensions.yaml) -sparta_named_test(sparta_core_example_update_reports_on_demand sparta_core_example -i 10k --report update_report_on_demand.yaml --feature simdb 0) +sparta_named_test(sparta_core_example_update_reports_on_demand sparta_core_example -i 10k --report update_report_on_demand.yaml) sparta_named_test(sparta_core_example_context_counter_trigger_expressions sparta_core_example -i 10k --report context_counter_report_triggers.yaml -p top.cpu.core0.params.foo 8k) sparta_named_test(sparta_core_example_weighted_context_counter_trigger_expressions sparta_core_example -i 10k --report weighted_context_counter_report_triggers.yaml -p top.cpu.core0.params.foo 8k --config-file context_weights.yaml) if ($ENV{MEMORY_PROFILING_ENABLED}) @@ -135,25 +135,7 @@ sparta_named_test(sparta_core_example_arch_with_extensions sparta_core_example - # The old behavior of Sparta allowed extensions to be optional by default. Now they are an error #sparta_named_test(sparta_core_example_indiv_tree_node_extensions sparta_core_example -i 10k -p top.cpu.core0.lsu.extension.dog.language_ woof --extension-file tree_node_extensions.yaml -p top.cpu.core0.lsu.extension.cat.language_ meow --write-final-config final.yaml) sparta_named_test(sparta_core_example_indiv_tree_node_extensions sparta_core_example -i 10k --extension-file tree_node_extensions.yaml -p top.cpu.core0.lsu.extension.cat.language_ meow --write-final-config final.yaml) -sparta_named_test(sparta_core_example_sqldb_timeseries sparta_core_example -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_min_blob_size.yaml) -sparta_named_test(sparta_core_example_sqldb_timeseries_opts1 sparta_core_example -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts1.yaml) -sparta_named_test(sparta_core_example_sqldb_timeseries_opts2 sparta_core_example -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts2.yaml) -sparta_named_test(sparta_core_example_sqldb_timeseries_opts3 sparta_core_example -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts3.yaml) -sparta_named_test(sparta_core_example_sqldb_timeseries_opts4 sparta_core_example -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts4.yaml) -sparta_named_test(sparta_core_example_simdb_non_timeseries sparta_core_example -i 10k --report multireport_r1.yaml --feature simdb 1) -#sparta_named_test(sparta_core_example_all_report_formats_no_verif sparta_core_example -i 20k --report all_report_formats.yaml --feature simdb 1) -sparta_named_test(sparta_core_example_expected_dest_file_location sparta_core_example -i 20k --report all_report_formats.yaml --report-verif-output-dir @ --feature simdb 1 --feature simdb-verify 1) -sparta_named_test(sparta_core_example_csv_headers_in_db sparta_core_example -i 100k --report csv_headers_in_db.yaml --feature simdb 1 --feature simdb-verify 1) #sparta_named_test(sparta_core_example_custom_histogram_stats sparta_core_example -i 100k --report multireport_r3.yaml) -sparta_named_test(sparta_core_example_simdb_configure_opts1 sparta_core_example -i 10k --report all_update_triggers.yaml --simdb-dir .) -sparta_named_test(sparta_core_example_simdb_configure_opts2 sparta_core_example -i 10k --report all_update_triggers.yaml --simdb-dir myfolder) -sparta_named_test(sparta_core_example_simdb_configure_opts3 sparta_core_example -i 10k --report all_update_triggers.yaml --simdb-dir my/sub/folder) -sparta_named_test(sparta_core_example_simdb_configure_opts4 sparta_core_example -i 10k --report all_update_triggers.yaml --simdb-dir myfolder --simdb-enabled-components dbaccess_opts1.yaml) -sparta_named_test_no_valgrind(sparta_core_example_simdb_configure_opts5_wildcard1 sparta_core_example -i 10k --simdb-enabled-components dbaccess_opts2.yaml --feature wildcard-components 1) -sparta_named_test_no_valgrind(sparta_core_example_simdb_configure_opts6_wildcard2 sparta_core_example -i 10k --simdb-enabled-components dbaccess_opts3.yaml) -sparta_named_test_no_valgrind(sparta_core_example_simdb_perf_async_task_controller sparta_core_example -i 50k --feature simdb-perf-async-ctrl 1) -sparta_named_test(sparta_core_example_collect_legacy_reports sparta_core_example -i 10k --report all_json_formats.yaml --collect-legacy-reports collection/dir) -sparta_named_test(sparta_core_example_collect_legacy_reports_specific_format sparta_core_example -i 10k --report all_json_formats.yaml --collect-legacy-reports collection/dir json json_reduced) sparta_named_test(sparta_core_example_arch_report_simple sparta_core_example -i 10k --arch simple --arch-search-dir . --report top arch_report.yaml 1) sparta_named_test(sparta_core_example_arch_report_default sparta_core_example -i 10k --report top arch_report.yaml 1) diff --git a/sparta/example/CoreModel/src/BIU.hpp b/sparta/example/CoreModel/src/BIU.hpp index 49ebb26678..fb71226f07 100644 --- a/sparta/example/CoreModel/src/BIU.hpp +++ b/sparta/example/CoreModel/src/BIU.hpp @@ -9,7 +9,6 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" -#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "CoreTypes.hpp" diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 2d97ecf90f..2e0c031238 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -10,6 +10,7 @@ #include "sparta/simulation/State.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" +#include "simdb/collection/Structs.hpp" #include #include @@ -250,3 +251,52 @@ namespace core_example SPARTA_ADDPAIR("vaddr", &ExampleInst::getVAdr, std::ios::hex)) }; } + +namespace simdb +{ + +template <> +inline void defineEnumMap(std::string& enum_name, std::map& map) +{ + using TargetUnit = core_example::ExampleInst::TargetUnit; + + enum_name = "TargetUnit"; + map["ALU0"] = static_cast(TargetUnit::ALU0); + map["ALU1"] = static_cast(TargetUnit::ALU1); + map["FPU"] = static_cast(TargetUnit::FPU); + map["BR"] = static_cast(TargetUnit::BR); + map["LSU"] = static_cast(TargetUnit::LSU); + map["ROB"] = static_cast(TargetUnit::ROB); +} + +template <> +inline void defineStructSchema(StructSchema& schema) +{ + using TargetUnit = core_example::ExampleInst::TargetUnit; + + schema.setStructName("ExampleInst"); + schema.addField("DID"); + schema.addField("uid"); + schema.addField("mnemonic"); + schema.addBoolField("complete"); + schema.addField("unit"); + schema.addField("latency"); + schema.addHexField("raddr"); + schema.addHexField("vaddr"); + schema.setAutoColorizeColumn("DID"); +} + +template <> +inline void writeStructFields(const core_example::ExampleInst* inst, StructFieldSerializer* serializer) +{ + serializer->writeField(inst->getUniqueID()); + serializer->writeField(inst->getUniqueID()); + serializer->writeField(inst->getMnemonic()); + serializer->writeField(inst->getCompletedStatus()); + serializer->writeField(inst->getUnit()); + serializer->writeField(inst->getExecuteTime()); + serializer->writeField(inst->getRAdr()); + serializer->writeField(inst->getVAdr()); +} + +} // namespace simdb diff --git a/sparta/example/CoreModel/src/ExampleSimulation.cpp b/sparta/example/CoreModel/src/ExampleSimulation.cpp index 0bd3c4425e..fb94b91598 100644 --- a/sparta/example/CoreModel/src/ExampleSimulation.cpp +++ b/sparta/example/CoreModel/src/ExampleSimulation.cpp @@ -18,14 +18,6 @@ #include "sparta/statistics/Histogram.hpp" #include "sparta/statistics/HistogramFunctionManager.hpp" #include "sparta/utils/SpartaTester.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/TableProxy.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" -#include "simdb/utils/uuids.hpp" -#include "simdb/utils/ObjectQuery.hpp" #include "Fetch.hpp" #include "Decode.hpp" @@ -42,270 +34,6 @@ #include "BIU.hpp" #include "MSS.hpp" -namespace { - - // Struct for writing and verifying SQLite records. - // See buildSchemaA() below. - struct TestSQLiteSchemaA { - struct Numbers { - double First; - double Second; - }; - Numbers numbers; - - struct Metadata { - std::string Name; - double Value; - }; - Metadata metadata; - - static TestSQLiteSchemaA createRandom() { - TestSQLiteSchemaA s; - s.numbers.First = rand() / 1000 * 3.14; - s.numbers.Second = rand() / 1000 * 3.14; - s.metadata.Name = simdb::generateUUID(); - s.metadata.Value = rand() / 1000 * 3.14; - return s; - } - }; - - // Another struct for writing and verifying SQLite - // records. See buildSchemaB() below. - struct TestSQLiteSchemaB { - struct Strings { - std::string First; - std::string Second; - }; - Strings strings; - - struct Metadata { - std::string Name; - std::string Value; - }; - Metadata metadata; - - static TestSQLiteSchemaB createRandom() { - TestSQLiteSchemaB s; - s.strings.First = simdb::generateUUID(); - s.strings.Second = simdb::generateUUID(); - s.metadata.Name = simdb::generateUUID(); - s.metadata.Value = simdb::generateUUID(); - return s; - } - }; - - // Struct for writing and verifying HDF5 records - struct TestHDF5SchemaC { - double x; - double y; - uint16_t z; - - static TestHDF5SchemaC createRandom() { - TestHDF5SchemaC s; - s.x = rand() / 1000 * 3.14; - s.y = rand() / 1000 * 3.14; - s.z = rand(); - return s; - } - }; -} - -namespace sparta_simdb { - - // Helper class which creates random SQLite / HDF5 - // structs for SimDB writes, and stores the structs - // in memory too. The data will be read back from - // the database at the end of simulation, and the - // values retrieved from file will be compared with - // the values that were stored in memory. - class DatabaseTester { - public: - DatabaseTester() = default; - ~DatabaseTester() = default; - - const TestSQLiteSchemaA & createAndStoreRecordForSQLiteSchemaA() { - if (records_schemaA_.size() < 100) { - indices_schemaA_.emplace_back(records_schemaA_.size()); - records_schemaA_.emplace_back(TestSQLiteSchemaA::createRandom()); - return records_schemaA_.back(); - } else { - indices_schemaA_.emplace_back(rand() % records_schemaA_.size()); - return records_schemaA_[indices_schemaA_.back()]; - } - } - - const TestSQLiteSchemaB & createAndStoreRecordForSQLiteSchemaB() { - if (records_schemaB_.size() < 100) { - indices_schemaB_.emplace_back(records_schemaB_.size()); - records_schemaB_.emplace_back(TestSQLiteSchemaB::createRandom()); - return records_schemaB_.back(); - } else { - indices_schemaB_.emplace_back(rand() % records_schemaB_.size()); - return records_schemaB_[indices_schemaB_.back()]; - } - } - - const TestHDF5SchemaC & createAndStoreRecordForHDF5SchemaC() { - records_schemaC_.emplace_back(TestHDF5SchemaC::createRandom()); - return records_schemaC_.back(); - } - - const std::vector & getWrittenRecordsForSchemaA() const { - return records_schemaA_; - } - - const std::vector & getWrittenRecordsForSchemaB() const { - return records_schemaB_; - } - - const std::vector & getWrittenRecordsForSchemaC() const { - return records_schemaC_; - } - - void verifyRecords(const std::string & db_file) const { - simdb::ObjectManager obj_mgr("."); - if (!obj_mgr.connectToExistingDatabase(db_file)) { - return; - } - - auto numeric_db = GET_DB_FROM_CURRENT_SIMULATION(NumericMeta); - if (numeric_db) { - auto values_query = - numeric_db->createObjectQueryForTable("Numbers"); - - if (values_query) { - double first = 0, second = 0; - values_query->writeResultIterationsTo( - "First", &first, "Second", &second); - - if (values_query->countMatches() != indices_schemaA_.size()) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - - auto result_iter = values_query->executeQuery(); - size_t record_idx = 0; - while (result_iter->getNext()) { - const auto & expected = records_schemaA_[indices_schemaA_[record_idx]]; - if (first != expected.numbers.First) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - if (second != expected.numbers.Second) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - ++record_idx; - } - } - - auto meta_query = - numeric_db->createObjectQueryForTable("Metadata"); - if (meta_query) { - std::string name; - double value = 0; - meta_query->writeResultIterationsTo("Name", &name, "Value", &value); - - if (meta_query->countMatches() != indices_schemaA_.size()) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - - auto result_iter = meta_query->executeQuery(); - size_t record_idx = 0; - while (result_iter->getNext()) { - const auto & expected = records_schemaA_[indices_schemaA_[record_idx]]; - if (name != expected.metadata.Name) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - if (value != expected.metadata.Value) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - ++record_idx; - } - } - } - } - - private: - std::vector records_schemaA_; - std::vector records_schemaB_; - std::vector records_schemaC_; - std::vector indices_schemaA_; - std::vector indices_schemaB_; - std::vector indices_schemaC_; - }; -} - -namespace { - - // Schema builder to test two simdb::ObjectManager's - // bound to the same database file, separated in that - // same file by their respective application name. - // A third schema builder is for another ObjectManager, - // though it will be used to write records to an HDF5 - // database, and therefore will be in its own file. - // SimDB's worker thread should be able to keep them - // separated into two groups: one group for the two - // SQLite database connections, and one group only - // serving the one HDF5 connection. - // - // Note that the two schema builders below have some - // overlap in their table definitions: schemaA and - // schemaB have some of the same table names, but - // these tables have different column configurations. - // This should not be a problem for ObjectManager - // since it will use its unique application name - // with the table names we give it to create a - // unique schema inside the shared file, separated - // from other applications tied to the same file. - // The specific way in which the schemas are kept - // separate in the file is not our concern; the - // DbConnProxy subclasses take care of those - // specifics. - void buildSchemaA(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addColumn("First", dt::double_t) - .addColumn("Second", dt::double_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::double_t); - } - - void buildSchemaB(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Strings") - .addColumn("First", dt::string_t) - .addColumn("Second", dt::string_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::string_t); - } - - void buildSchemaC(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addField("x", dt::double_t, FOFFSET(TestHDF5SchemaC,x)) - .addField("y", dt::double_t, FOFFSET(TestHDF5SchemaC,y)) - .addField("z", dt::uint16_t, FOFFSET(TestHDF5SchemaC,z)); - } - - simdb::DbConnProxy * createSQLiteProxy() - { - return new simdb::SQLiteConnProxy; - } - - simdb::DbConnProxy * createHDF5Proxy() - { - return new simdb::HDF5ConnProxy; - } -} - namespace sparta { // Example parameter set used to reproduce write-final-config @@ -336,19 +64,6 @@ namespace sparta { TreeNode(parent, "baz_node", "BazGroup", 0, desc) { baz_.reset(new IntParameterSet(this)); - checkDbAccess(); - } - - void checkDbAccess(const bool stop_checking = false) { - if (stop_checking_db_access_) { - return; - } - if (auto dbconn = GET_DB_FOR_COMPONENT(Stats, this)) { - //Run a simple query against the database just to verify - //the connection is open and accepting requests - (void) dbconn->findObject("ObjectManagersInDatabase", 1); - stop_checking_db_access_ = stop_checking; - } } void readParams() { @@ -364,7 +79,6 @@ namespace sparta { private: std::unique_ptr baz_; - bool stop_checking_db_access_ = false; }; } @@ -448,15 +162,6 @@ double calculateAverageOfInternalCounters( return agg / counters.size(); } -void tryAccessSimDB() -{ - if (auto dbconn = GET_DB_FROM_CURRENT_SIMULATION(Stats)) { - //Run a simple query against the database just to verify - //the connection is open and accepting requests - (void) dbconn->findObject("ObjectManagersInDatabase", 1); - } -} - ExampleSimulator::ExampleSimulator(const std::string& topology, sparta::Scheduler & scheduler, uint32_t num_cores, @@ -466,8 +171,7 @@ ExampleSimulator::ExampleSimulator(const std::string& topology, cpu_topology_(topology), num_cores_(num_cores), instruction_limit_(instruction_limit), - show_factories_(show_factories), - simdb_tester_(std::make_shared()) + show_factories_(show_factories) { // Set up the CPU Resource Factory to be available through ResourceTreeNode getResourceSet()->addResourceFactory(); @@ -488,20 +192,6 @@ ExampleSimulator::ExampleSimulator(const std::string& topology, // definition YAML files. sparta::trigger::ContextCounterTrigger::registerContextCounterCalcFunction( "avg", &calculateAverageOfInternalCounters); - - //SQLite namespaces: NumericMeta & StringMeta - REGISTER_SIMDB_NAMESPACE(NumericMeta, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(NumericMeta, buildSchemaA); - - REGISTER_SIMDB_NAMESPACE(StringMeta, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(StringMeta, buildSchemaB); - - //HDF5 namespace: NumericVals - REGISTER_SIMDB_NAMESPACE(NumericVals, HDF5); - REGISTER_SIMDB_SCHEMA_BUILDER(NumericVals, buildSchemaC); - - //Proxy factory registration - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(HDF5, createHDF5Proxy); } void ExampleSimulator::registerStatCalculationFcns_() @@ -529,17 +219,6 @@ ExampleSimulator::~ExampleSimulator() getRoot()->DEREGISTER_FOR_NOTIFICATION( onTriggered_, std::string, "sparta_expression_trigger_fired"); } - - if (simdb_perf_async_ctrl_enabled_) { - std::set simdb_files; - if (auto dbconn = GET_DB_FOR_COMPONENT(NumericMeta, this)) { - simdb_files.insert(dbconn->getDatabaseFile()); - } - - for (const auto & db_file : simdb_files) { - simdb_tester_->verifyRecords(db_file); - } - } } //! Get the resource factory needed to build and bind the tree @@ -685,55 +364,6 @@ void ExampleSimulator::buildTree_() void ExampleSimulator::configureTree_() { - //Context-aware SimDB access - std::pair sqlite_db_files; - if (auto dbconn = GET_DB_FOR_COMPONENT(NumericMeta, this)) { - const TestSQLiteSchemaA data = simdb_tester_-> - createAndStoreRecordForSQLiteSchemaA(); - - const double first = data.numbers.First; - const double second = data.numbers.Second; - dbconn->getTable("Numbers")->createObjectWithArgs( - "First", first, "Second", second); - - const std::string meta_name = data.metadata.Name; - const double meta_value = data.metadata.Value; - dbconn->getTable("Metadata")->createObjectWithArgs( - "Name", meta_name, "Value", meta_value); - - sqlite_db_files.first = dbconn->getDatabaseFile(); - - //Verification of the two records we just made above - //will occur at the end of the simulation. - } - - if (auto dbconn = GET_DB_FOR_COMPONENT(StringMeta, this)) { - const TestSQLiteSchemaB data = simdb_tester_-> - createAndStoreRecordForSQLiteSchemaB(); - - const std::string first = data.strings.First; - const std::string second = data.strings.Second; - dbconn->getTable("Strings")->createObjectWithArgs( - "First", first, "Second", second); - - const std::string meta_name = data.metadata.Name; - const std::string meta_value = data.metadata.Value; - dbconn->getTable("Metadata")->createObjectWithArgs( - "Name", meta_name, "Value", meta_value); - - sqlite_db_files.second = dbconn->getDatabaseFile(); - - //Verification of the two records we just made above - //will occur at the end of the simulation. - } - - //Both of the ObjectManager's used above should have put the - //created records into the same file. - sparta_assert(sqlite_db_files.first == sqlite_db_files.second); - - //Context-unaware SimDB access - tryAccessSimDB(); - validateTreeNodeExtensions_(); // In TREE_CONFIGURING phase @@ -775,9 +405,6 @@ void ExampleSimulator::configureTree_() getRoot()->REGISTER_FOR_NOTIFICATION( onTriggered_, std::string, "sparta_expression_trigger_fired"); on_triggered_notifier_registered_ = true; - - simdb_perf_async_ctrl_enabled_ = sparta::IsFeatureValueEnabled( - getFeatureConfiguration(), "simdb-perf-async-ctrl") > 0; } void ExampleSimulator::bindTree_() @@ -803,19 +430,6 @@ void ExampleSimulator::bindTree_() "1 ns", getRoot())); - lazy_table_create_trigger_.reset(new sparta::trigger::ExpressionTrigger( - "DelayedTableCreate", - CREATE_SPARTA_HANDLER(ExampleSimulator, addToStatsSchema_), - "top.cpu.core0.rob.stats.total_number_retired >= 12000", - getRoot()->getSearchScope(), - nullptr)); - - if (auto db_root = GET_DB_FROM_CURRENT_SIMULATION(Stats)) { - lazy_table_proxy_ = db_root->getConditionalTable("Lazy"); - sparta_assert(lazy_table_proxy_ != nullptr); - sparta_assert(lazy_table_proxy_->getTable() == nullptr); - } - static const uint32_t warmup_multiplier = 1000; auto gen_expression = [](const uint32_t core_idx) { std::ostringstream oss; @@ -862,132 +476,6 @@ void ExampleSimulator::postRandomNumber_() const size_t random = rand() % 25; testing_notification_source_->postNotification(random); random_number_trigger_->reschedule(); - - if (dispatch_baz_) { - dispatch_baz_->checkDbAccess(true); - } - - if (!simdb_perf_async_ctrl_enabled_) { - return; - } - - using ObjectDatabase = simdb::ObjectManager::ObjectDatabase; - - // In the SimDB-related code below, note that GET_DB_FOR_COMPONENT is - // returning a unique_ptr, not a shared_ptr. - // - // The ability to request database connections and get unique_ptr's - // back is important because it demonstrates that different parts - // of the simulator can write data into the same database, into their - // own namespace's schema, sharing the same worker thread (which is - // just implementation detail, but it's important for performance and - // scalability) with no coordination required between the simulator - // components / call sites. - // - // Also note that we have a mixture of DB writes going on here. There - // are two separate physical database files: one is SQLite, and the - // other is HDF5. The SQLite file has two namespaces in it, named - // NumericMeta and StringMeta; the HDF5 file just has one namespace - // in it called NumericVals. These namespaces, their database formats, - // and the namespace schema definition was registered with SimDB from - // the ExampleSimulator's constructor earlier on. - - if (auto obj_db = GET_DB_FOR_COMPONENT(NumericMeta, this)) { - // Helper class which writes a data record on the worker thread - class TestWriter : public simdb::WorkerTask - { - public: - TestWriter(ObjectDatabase * obj_db, - sparta_simdb::DatabaseTester * db_tester) : - obj_db_(obj_db), - simdb_tester_(db_tester) - {} - - void completeTask() override { - const TestSQLiteSchemaA data = simdb_tester_-> - createAndStoreRecordForSQLiteSchemaA(); - - obj_db_->getTable("Numbers")->createObjectWithArgs( - "First", data.numbers.First, - "Second", data.numbers.Second); - - obj_db_->getTable("Metadata")->createObjectWithArgs( - "Name", data.metadata.Name, - "Value", data.metadata.Value); - } - - private: - ObjectDatabase * obj_db_ = nullptr; - sparta_simdb::DatabaseTester * simdb_tester_ = nullptr; - }; - - std::unique_ptr task(new TestWriter( - obj_db, simdb_tester_.get())); - obj_db->getTaskQueue()->addWorkerTask(std::move(task)); - } - - if (auto obj_db = GET_DB_FOR_COMPONENT(StringMeta, this)) { - // Helper class which writes a data record on the worker thread - class TestWriter : public simdb::WorkerTask - { - public: - TestWriter(ObjectDatabase * obj_db, - sparta_simdb::DatabaseTester * db_tester) : - obj_db_(obj_db), - simdb_tester_(db_tester) - {} - - void completeTask() override { - const TestSQLiteSchemaB data = simdb_tester_-> - createAndStoreRecordForSQLiteSchemaB(); - - obj_db_->getTable("Strings")->createObjectWithArgs( - "First", data.strings.First, - "Second", data.strings.Second); - - obj_db_->getTable("Metadata")->createObjectWithArgs( - "Name", data.metadata.Name, - "Value", data.metadata.Value); - } - - private: - ObjectDatabase * obj_db_ = nullptr; - sparta_simdb::DatabaseTester * simdb_tester_ = nullptr; - }; - - std::unique_ptr task(new TestWriter( - obj_db, simdb_tester_.get())); - obj_db->getTaskQueue()->addWorkerTask(std::move(task)); - } - - if (auto obj_db = GET_DB_FOR_COMPONENT(NumericVals, this)) { - // Helper class which writes a data record on the worker thread - class TestWriter : public simdb::WorkerTask - { - public: - TestWriter(ObjectDatabase * obj_db, - sparta_simdb::DatabaseTester * db_tester) : - obj_db_(obj_db), - simdb_tester_(db_tester) - {} - - void completeTask() override { - const TestHDF5SchemaC data = simdb_tester_-> - createAndStoreRecordForHDF5SchemaC(); - - obj_db_->getTable("Numbers")->createObjectWithVals( - data.x, data.y, data.z); - } - - private: - ObjectDatabase * obj_db_ = nullptr; - sparta_simdb::DatabaseTester * simdb_tester_ = nullptr; - }; - - std::unique_ptr task(new TestWriter( - obj_db, simdb_tester_.get())); - obj_db->getTaskQueue()->addWorkerTask(std::move(task)); - } } void ExampleSimulator::postToToggleTrigger_() @@ -1020,51 +508,6 @@ void ExampleSimulator::postToToggleTrigger_() toggle_notif_trigger_->reschedule(); } -void ExampleSimulator::addToStatsSchema_() -{ - if (auto db_root = getDatabaseRoot()) { - if (auto db_namespace = db_root->getNamespace("Stats")) { - db_namespace->addToSchema([&](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("Lazy") - .addColumn("Foo", dt::string_t) - .addColumn("Bar", dt::int32_t); - }); - - lazy_table_create_trigger_.reset(new sparta::trigger::ExpressionTrigger( - "DelayedTableCreate", - CREATE_SPARTA_HANDLER(ExampleSimulator, addToLazySchemaTable_), - "top.cpu.core0.rob.stats.total_number_retired >= 40000", - getRoot()->getSearchScope(), - nullptr)); - } - } -} - -void ExampleSimulator::addToLazySchemaTable_() -{ - if (lazy_table_proxy_->isWritable()) { - const std::string foo = "hello_world"; - const int bar = 45; - - auto recordA = lazy_table_proxy_->getTable()->createObjectWithArgs( - "Foo", foo, "Bar", bar); - - auto db_root = GET_DB_FROM_CURRENT_SIMULATION(Stats); - sparta_assert(db_root != nullptr); - - auto recordB = db_root->getTable("Lazy")->createObjectWithArgs( - "Foo", foo, "Bar", bar); - - sparta_assert(recordA->getPropertyString("Foo") == - recordB->getPropertyString("Foo")); - - sparta_assert(recordA->getPropertyInt32("Bar") == - recordB->getPropertyInt32("Bar")); - } -} - void ExampleSimulator::onTriggered_(const std::string & msg) { std::cout << " [trigger] " << msg << std::endl; diff --git a/sparta/example/CoreModel/src/ExampleSimulation.hpp b/sparta/example/CoreModel/src/ExampleSimulation.hpp index 679953df93..4f77c5e820 100644 --- a/sparta/example/CoreModel/src/ExampleSimulation.hpp +++ b/sparta/example/CoreModel/src/ExampleSimulation.hpp @@ -10,9 +10,6 @@ namespace sparta { class Baz; class ParameterSet; } -namespace sparta_simdb { - class DatabaseTester; -} namespace sparta { namespace trigger { class ExpressionTrigger; } } namespace core_example{ class CPUFactory; } @@ -115,16 +112,6 @@ class ExampleSimulator : public sparta::app::Simulation std::unique_ptr toggle_notif_trigger_; void postToToggleTrigger_(); - /*! - * \brief Trigger which adds a table to the 'Stats' database namespace - * during simulation, and a TableProxy object referring to that table - * which is cached before the table actually exists in the schema. - */ - sparta::trigger::ExpiringExpressionTrigger lazy_table_create_trigger_; - simdb::TableProxy * lazy_table_proxy_ = nullptr; - void addToStatsSchema_(); - void addToLazySchemaTable_(); - /*! * \brief Notification source and dedicated warmup listeners used to mimic * legacy report start events. @@ -155,19 +142,4 @@ class ExampleSimulator : public sparta::app::Simulation * \brief Optional flag to print registered factories to console */ bool show_factories_; - - /* \brief Flag which enables SimDB-related code to run for - * interactive performance benchmarks / comparison. False - * by default so that we don't impact unit testing / smoke - * testing times for all regression test runs. - */ - bool simdb_perf_async_ctrl_enabled_ = false; - - /*! - * \brief Tester class which holds onto data structures that - * are randomly generated and written to SimDB during the - * simulation, verifying the contents at the end of the - * simulation for accuracy. Defined in this class' CPP file. - */ - std::shared_ptr simdb_tester_; }; diff --git a/sparta/example/CoreModel/src/Execute.cpp b/sparta/example/CoreModel/src/Execute.cpp index ac512d43f5..64f24576c6 100644 --- a/sparta/example/CoreModel/src/Execute.cpp +++ b/sparta/example/CoreModel/src/Execute.cpp @@ -13,8 +13,7 @@ namespace core_example ignore_inst_execute_time_(p->ignore_inst_execute_time), execute_time_(p->execute_time), scheduler_size_(p->scheduler_size), - in_order_issue_(p->in_order_issue), - collected_inst_(node, node->getName()) + in_order_issue_(p->in_order_issue) { in_execute_inst_. registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Execute, getInstsFromDispatch_, @@ -85,7 +84,7 @@ namespace core_example ex_inst.setStatus(ExampleInst::Status::SCHEDULED); const uint32_t exe_time = ignore_inst_execute_time_ ? execute_time_ : ex_inst.getExecuteTime(); - collected_inst_.collectWithDuration(ex_inst, exe_time); + // TODO cnyce collected_inst_.collectWithDuration(ex_inst, exe_time); if(SPARTA_EXPECT_FALSE(info_logger_)) { info_logger_ << "Executing: " << ex_inst << " for " << exe_time + getClock()->currentCycle(); @@ -153,7 +152,7 @@ namespace core_example if(complete_inst_.getNumOutstandingEvents() == 0) { unit_busy_ = false; - collected_inst_.closeRecord(); + // TODO cnyce collected_inst_.closeRecord(); } } diff --git a/sparta/example/CoreModel/src/Execute.hpp b/sparta/example/CoreModel/src/Execute.hpp index c27334f2b6..4f3ab7e6bf 100644 --- a/sparta/example/CoreModel/src/Execute.hpp +++ b/sparta/example/CoreModel/src/Execute.hpp @@ -20,8 +20,8 @@ #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/ports/Port.hpp" -#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" #include "CoreTypes.hpp" #include "FlushManager.hpp" @@ -82,9 +82,15 @@ namespace core_example const uint32_t execute_time_; const uint32_t scheduler_size_; const bool in_order_issue_; - sparta::collection::IterableCollector> - ready_queue_collector_ {getContainer(), "scheduler_queue", - &ready_queue_, scheduler_size_}; + + using IterableCollectorType = sparta::collection::IterableCollector>; + + // Collection + IterableCollectorType ready_queue_collector_{ + getContainer(), "scheduler_queue", &ready_queue_, scheduler_size_}; + + sparta::collection::Collectable collected_inst_{ + getContainer(), "collected_inst", nullptr}; // Events used to issue and complete the instruction sparta::UniqueEvent<> issue_inst_{&unit_event_set_, getName() + "_issue_inst", @@ -93,9 +99,6 @@ namespace core_example &unit_event_set_, getName() + "_complete_inst", CREATE_SPARTA_HANDLER_WITH_DATA(Execute, completeInst_, ExampleInstPtr)}; - // A pipeline collector - sparta::collection::Collectable collected_inst_; - // Counter sparta::Counter total_insts_issued_{ getStatisticSet(), "total_insts_issued", diff --git a/sparta/example/CoreModel/src/Fetch.cpp b/sparta/example/CoreModel/src/Fetch.cpp index 393303df12..f1b910bbbf 100644 --- a/sparta/example/CoreModel/src/Fetch.cpp +++ b/sparta/example/CoreModel/src/Fetch.cpp @@ -104,8 +104,7 @@ namespace core_example Fetch::Fetch(sparta::TreeNode * node, const FetchParameterSet * p) : sparta::Unit(node), - num_insts_to_fetch_(p->num_to_fetch), - next_pc_(node, "next_pc", &vaddr_) + num_insts_to_fetch_(p->num_to_fetch) { in_fetch_queue_credits_. registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Fetch, receiveFetchQueueCredits_, uint32_t)); diff --git a/sparta/example/CoreModel/src/Fetch.hpp b/sparta/example/CoreModel/src/Fetch.hpp index 9df40489cf..4393a7bb56 100644 --- a/sparta/example/CoreModel/src/Fetch.hpp +++ b/sparta/example/CoreModel/src/Fetch.hpp @@ -11,7 +11,6 @@ #include #include "sparta/ports/DataPort.hpp" #include "sparta/events/SingleCycleUniqueEvent.hpp" -#include "sparta/collection/Collectable.hpp" #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/TreeNode.hpp" #include "sparta/simulation/ParameterSet.hpp" @@ -102,9 +101,6 @@ namespace core_example // instructions or a perfect IPC set std::unique_ptr> fetch_inst_event_; - // A pipeline collector - sparta::collection::Collectable next_pc_; - //////////////////////////////////////////////////////////////////////////////// // Callbacks diff --git a/sparta/example/CoreModel/src/LSU.cpp b/sparta/example/CoreModel/src/LSU.cpp index aa7e0f3081..a2e485f2b3 100644 --- a/sparta/example/CoreModel/src/LSU.cpp +++ b/sparta/example/CoreModel/src/LSU.cpp @@ -19,7 +19,7 @@ namespace core_example tlb_always_hit_(p->tlb_always_hit), dl1_always_hit_(p->dl1_always_hit), - + cache_busy_collectable_(getContainer(), "dcache_busy", &cache_busy_), issue_latency_(p->issue_latency), mmu_latency_(p->mmu_latency), cache_latency_(p->cache_latency), @@ -573,7 +573,7 @@ namespace core_example } // Arbitrate instruction issue from ldst_inst_queue - const LSU::LoadStoreInstInfoPtr & LSU::arbitrateInstIssue_() + const LoadStoreInstInfoPtr & LSU::arbitrateInstIssue_() { sparta_assert(ldst_inst_queue_.size() > 0, "Arbitration fails: issue is empty!"); diff --git a/sparta/example/CoreModel/src/LSU.hpp b/sparta/example/CoreModel/src/LSU.hpp index cf3a64a481..da5ec9ab06 100644 --- a/sparta/example/CoreModel/src/LSU.hpp +++ b/sparta/example/CoreModel/src/LSU.hpp @@ -9,7 +9,6 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" -#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "sparta/resources/Pipeline.hpp" #include "sparta/resources/Buffer.hpp" @@ -24,6 +23,8 @@ #include "FlushManager.hpp" #include "SimpleTLB.hpp" #include "SimpleDL1.hpp" +#include "MemAccessInfo.hpp" +#include "LoadStoreInstInfo.hpp" namespace core_example { @@ -87,12 +88,9 @@ namespace core_example // Type Name/Alias Declaration //////////////////////////////////////////////////////////////////////////////// + // allocator for this object type + sparta::SpartaSharedPointerAllocator memory_access_allocator; - class LoadStoreInstInfo; - class MemoryAccessInfo; - - using LoadStoreInstInfoPtr = sparta::SpartaSharedPointer; - using MemoryAccessInfoPtr = sparta::SpartaSharedPointer; using FlushCriteria = FlushManager::FlushingCriteria; enum class PipelineStage @@ -103,235 +101,8 @@ namespace core_example NUM_STAGES }; - // Forward declaration of the Pair Definition class is must as we are friending it. - class MemoryAccessInfoPairDef; - // Keep record of memory access information in LSU - class MemoryAccessInfo { - public: - - // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself - using SpartaPairDefinitionType = MemoryAccessInfoPairDef; - - enum class MMUState : std::uint32_t { - NO_ACCESS = 0, - __FIRST = NO_ACCESS, - MISS, - HIT, - NUM_STATES, - __LAST = NUM_STATES - }; - - enum class CacheState : std::uint64_t { - NO_ACCESS = 0, - __FIRST = NO_ACCESS, - MISS, - HIT, - NUM_STATES, - __LAST = NUM_STATES - }; - - MemoryAccessInfo() = delete; - - MemoryAccessInfo(const ExampleInstPtr & inst_ptr) : - ldst_inst_ptr_(inst_ptr), - phyAddrIsReady_(false), - mmu_access_state_(MMUState::NO_ACCESS), - - // Construct the State object here - cache_access_state_(CacheState::NO_ACCESS) {} - - virtual ~MemoryAccessInfo() {} - - // This ExampleInst pointer will act as our portal to the ExampleInst class - // and we will use this pointer to query values from functions of ExampleInst class - const ExampleInstPtr & getInstPtr() const { return ldst_inst_ptr_; } - - // This is a function which will be added in the SPARTA_ADDPAIRs API. - uint64_t getInstUniqueID() const { - const ExampleInstPtr &inst_ptr = getInstPtr(); - - return inst_ptr == nullptr ? 0 : inst_ptr->getUniqueID(); - } - - void setPhyAddrStatus(bool isReady) { phyAddrIsReady_ = isReady; } - bool getPhyAddrStatus() const { return phyAddrIsReady_; } - - const MMUState & getMMUState() const { - return mmu_access_state_.getEnumValue(); - } - - void setMMUState(const MMUState & state) { - mmu_access_state_.setValue(state); - } - - const CacheState & getCacheState() const { - return cache_access_state_.getEnumValue(); - } - void setCacheState(const CacheState & state) { - cache_access_state_.setValue(state); - } - - // This is a function which will be added in the addArgs API. - bool getPhyAddrIsReady() const{ - return phyAddrIsReady_; - } - - - private: - // load/store instruction pointer - ExampleInstPtr ldst_inst_ptr_; - - // Indicate MMU address translation status - bool phyAddrIsReady_; - - // MMU access status - sparta::State mmu_access_state_; - - // Cache access status - sparta::State cache_access_state_; - - }; // class MemoryAccessInfo - - // allocator for this object type - sparta::SpartaSharedPointerAllocator memory_access_allocator; - - /*! - * \class MemoryAccessInfoPairDef - * \brief Pair Definition class of the Memory Access Information that flows through the example/CoreModel - */ - - // This is the definition of the PairDefinition class of MemoryAccessInfo. - // This PairDefinition class could be named anything but it needs to inherit - // publicly from sparta::PairDefinition templatized on the actual class MemoryAcccessInfo. - class MemoryAccessInfoPairDef : public sparta::PairDefinition{ - public: - - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - MemoryAccessInfoPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(MemoryAccessInfo); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &MemoryAccessInfo::getInstUniqueID), - SPARTA_ADDPAIR("valid", &MemoryAccessInfo::getPhyAddrIsReady), - SPARTA_ADDPAIR("mmu", &MemoryAccessInfo::getMMUState), - SPARTA_ADDPAIR("cache", &MemoryAccessInfo::getCacheState), - SPARTA_FLATTEN( &MemoryAccessInfo::getInstPtr)) - }; - - // Forward declaration of the Pair Definition class is must as we are friending it. - class LoadStoreInstInfoPairDef; - // Keep record of instruction issue information - class LoadStoreInstInfo - { - public: - // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself - using SpartaPairDefinitionType = LoadStoreInstInfoPairDef; - enum class IssuePriority : std::uint16_t - { - HIGHEST = 0, - __FIRST = HIGHEST, - CACHE_RELOAD, // Receive mss ack, waiting for cache re-access - CACHE_PENDING, // Wait for another outstanding miss finish - MMU_RELOAD, // Receive for mss ack, waiting for mmu re-access - MMU_PENDING, // Wait for another outstanding miss finish - NEW_DISP, // Wait for new issue - LOWEST, - NUM_OF_PRIORITIES, - __LAST = NUM_OF_PRIORITIES - }; - - enum class IssueState : std::uint32_t - { - READY = 0, // Ready to be issued - __FIRST = READY, - ISSUED, // On the flight somewhere inside Load/Store Pipe - NOT_READY, // Not ready to be issued - NUM_STATES, - __LAST = NUM_STATES - }; - - LoadStoreInstInfo() = delete; - LoadStoreInstInfo(const MemoryAccessInfoPtr & info_ptr) : - mem_access_info_ptr_(info_ptr), - rank_(IssuePriority::LOWEST), - state_(IssueState::NOT_READY) {} - - // This ExampleInst pointer will act as one of the two portals to the ExampleInst class - // and we will use this pointer to query values from functions of ExampleInst class - const ExampleInstPtr & getInstPtr() const { - return mem_access_info_ptr_->getInstPtr(); - } - - // This MemoryAccessInfo pointer will act as one of the two portals to the MemoryAccesInfo class - // and we will use this pointer to query values from functions of MemoryAccessInfo class - const MemoryAccessInfoPtr & getMemoryAccessInfoPtr() const { - return mem_access_info_ptr_; - } - - // This is a function which will be added in the SPARTA_ADDPAIRs API. - uint64_t getInstUniqueID() const { - const MemoryAccessInfoPtr &mem_access_info_ptr = getMemoryAccessInfoPtr(); - - return mem_access_info_ptr == nullptr ? 0 : mem_access_info_ptr->getInstUniqueID(); - } - - void setPriority(const IssuePriority & rank) { - rank_.setValue(rank); - } - - const IssuePriority & getPriority() const { - return rank_.getEnumValue(); - } - - void setState(const IssueState & state) { - state_.setValue(state); - } - - const IssueState & getState() const { - return state_.getEnumValue(); - } - - - bool isReady() const { return (getState() == IssueState::READY); } - - bool winArb(const LoadStoreInstInfoPtr & that) const - { - if (that == nullptr) { - return true; - } - - return (static_cast(this->getPriority()) - < static_cast(that->getPriority())); - } - - private: - MemoryAccessInfoPtr mem_access_info_ptr_; - sparta::State rank_; - sparta::State state_; - - }; // class LoadStoreInstInfo - sparta::SpartaSharedPointerAllocator load_store_info_allocator; - /*! - * \class LoadStoreInstInfoPairDef - * \brief Pair Definition class of the load store instruction that flows through the example/CoreModel - */ - // This is the definition of the PairDefinition class of LoadStoreInstInfo. - // This PairDefinition class could be named anything but it needs to inherit - // publicly from sparta::PairDefinition templatized on the actual class LoadStoreInstInfo. - class LoadStoreInstInfoPairDef : public sparta::PairDefinition{ - public: - - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - LoadStoreInstInfoPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(LoadStoreInstInfo); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &LoadStoreInstInfo::getInstUniqueID), - SPARTA_ADDPAIR("rank", &LoadStoreInstInfo::getPriority), - SPARTA_ADDPAIR("state", &LoadStoreInstInfo::getState), - SPARTA_FLATTEN( &LoadStoreInstInfo::getMemoryAccessInfoPtr)) - }; - void setTLB(SimpleTLB& tlb) { tlb_cache_ = &tlb; @@ -394,8 +165,8 @@ namespace core_example // Keep track of the instruction that causes current outstanding cache miss ExampleInstPtr cache_pending_inst_ptr_ = nullptr; - sparta::collection::Collectable cache_busy_collectable_{ - getContainer(), "dcache_busy", &cache_busy_}; + // Collection + sparta::collection::Collectable cache_busy_collectable_; // NOTE: // Depending on which kind of cache (e.g. blocking vs. non-blocking) is being used @@ -526,86 +297,50 @@ namespace core_example void flushLSPipeline_(const Comp &); }; - inline std::ostream & operator<<(std::ostream & os, - const core_example::LSU::MemoryAccessInfo::MMUState & mmu_access_state){ - switch(mmu_access_state){ - case LSU::MemoryAccessInfo::MMUState::NO_ACCESS: - os << "no_access"; - break; - case LSU::MemoryAccessInfo::MMUState::MISS: - os << "miss"; - break; - case LSU::MemoryAccessInfo::MMUState::HIT: - os << "hit"; - break; - case LSU::MemoryAccessInfo::MMUState::NUM_STATES: - throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); - } - return os; - } - - inline std::ostream & operator<<(std::ostream & os, - const core_example::LSU::MemoryAccessInfo::CacheState & cache_access_state){ - switch(cache_access_state){ - case LSU::MemoryAccessInfo::CacheState::NO_ACCESS: - os << "no_access"; - break; - case LSU::MemoryAccessInfo::CacheState::MISS: - os << "miss"; - break; - case LSU::MemoryAccessInfo::CacheState::HIT: - os << "hit"; - break; - case LSU::MemoryAccessInfo::CacheState::NUM_STATES: - throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); - } - return os; - } - inline std::ostream& operator<<(std::ostream& os, - const core_example::LSU::LoadStoreInstInfo::IssuePriority& rank){ + const core_example::LoadStoreInstInfo::IssuePriority& rank){ switch(rank){ - case LSU::LoadStoreInstInfo::IssuePriority::HIGHEST: + case LoadStoreInstInfo::IssuePriority::HIGHEST: os << "(highest)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::CACHE_RELOAD: + case LoadStoreInstInfo::IssuePriority::CACHE_RELOAD: os << "($_reload)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::CACHE_PENDING: + case LoadStoreInstInfo::IssuePriority::CACHE_PENDING: os << "($_pending)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::MMU_RELOAD: + case LoadStoreInstInfo::IssuePriority::MMU_RELOAD: os << "(mmu_reload)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::MMU_PENDING: + case LoadStoreInstInfo::IssuePriority::MMU_PENDING: os << "(mmu_pending)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::NEW_DISP: + case LoadStoreInstInfo::IssuePriority::NEW_DISP: os << "(new_disp)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::LOWEST: + case LoadStoreInstInfo::IssuePriority::LOWEST: os << "(lowest)"; break; - case LSU::LoadStoreInstInfo::IssuePriority::NUM_OF_PRIORITIES: + case LoadStoreInstInfo::IssuePriority::NUM_OF_PRIORITIES: throw sparta::SpartaException("NUM_OF_PRIORITIES cannot be a valid enum state."); } return os; } inline std::ostream& operator<<(std::ostream& os, - const core_example::LSU::LoadStoreInstInfo::IssueState& state){ + const core_example::LoadStoreInstInfo::IssueState& state){ // Print instruction issue state switch(state){ - case LSU::LoadStoreInstInfo::IssueState::READY: + case LoadStoreInstInfo::IssueState::READY: os << "(ready)"; break; - case LSU::LoadStoreInstInfo::IssueState::ISSUED: + case LoadStoreInstInfo::IssueState::ISSUED: os << "(issued)"; break; - case LSU::LoadStoreInstInfo::IssueState::NOT_READY: + case LoadStoreInstInfo::IssueState::NOT_READY: os << "(not_ready)"; break; - case LSU::LoadStoreInstInfo::IssueState::NUM_STATES: + case LoadStoreInstInfo::IssueState::NUM_STATES: throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); } return os; diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp new file mode 100644 index 0000000000..112944f2d0 --- /dev/null +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -0,0 +1,228 @@ +#pragma once + +#include "MemAccessInfo.hpp" + +namespace core_example +{ + class LoadStoreInstInfo; + using LoadStoreInstInfoPtr = sparta::SpartaSharedPointer; + + // Forward declaration of the Pair Definition class is must as we are friending it. + class LoadStoreInstInfoPairDef; + // Keep record of instruction issue information + class LoadStoreInstInfo + { + public: + // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself + using SpartaPairDefinitionType = LoadStoreInstInfoPairDef; + enum class IssuePriority : std::uint16_t + { + HIGHEST = 0, + __FIRST = HIGHEST, + CACHE_RELOAD, // Receive mss ack, waiting for cache re-access + CACHE_PENDING, // Wait for another outstanding miss finish + MMU_RELOAD, // Receive for mss ack, waiting for mmu re-access + MMU_PENDING, // Wait for another outstanding miss finish + NEW_DISP, // Wait for new issue + LOWEST, + NUM_OF_PRIORITIES, + __LAST = NUM_OF_PRIORITIES + }; + + enum class IssueState : std::uint32_t + { + READY = 0, // Ready to be issued + __FIRST = READY, + ISSUED, // On the flight somewhere inside Load/Store Pipe + NOT_READY, // Not ready to be issued + NUM_STATES, + __LAST = NUM_STATES + }; + + LoadStoreInstInfo() = delete; + LoadStoreInstInfo(const MemoryAccessInfoPtr & info_ptr) : + mem_access_info_ptr_(info_ptr), + rank_(IssuePriority::LOWEST), + state_(IssueState::NOT_READY) {} + + // This ExampleInst pointer will act as one of the two portals to the ExampleInst class + // and we will use this pointer to query values from functions of ExampleInst class + const ExampleInstPtr & getInstPtr() const { + return mem_access_info_ptr_->getInstPtr(); + } + + // This MemoryAccessInfo pointer will act as one of the two portals to the MemoryAccesInfo class + // and we will use this pointer to query values from functions of MemoryAccessInfo class + const MemoryAccessInfoPtr & getMemoryAccessInfoPtr() const { + return mem_access_info_ptr_; + } + + // This is a function which will be added in the SPARTA_ADDPAIRs API. + uint64_t getInstUniqueID() const { + const MemoryAccessInfoPtr &mem_access_info_ptr = getMemoryAccessInfoPtr(); + + return mem_access_info_ptr == nullptr ? 0 : mem_access_info_ptr->getInstUniqueID(); + } + + void setPriority(const IssuePriority & rank) { + rank_.setValue(rank); + } + + const IssuePriority & getPriority() const { + return rank_.getEnumValue(); + } + + void setState(const IssueState & state) { + state_.setValue(state); + } + + const IssueState & getState() const { + return state_.getEnumValue(); + } + + + bool isReady() const { return (getState() == IssueState::READY); } + + bool winArb(const LoadStoreInstInfoPtr & that) const + { + if (that == nullptr) { + return true; + } + + return (static_cast(this->getPriority()) + < static_cast(that->getPriority())); + } + + private: + MemoryAccessInfoPtr mem_access_info_ptr_; + sparta::State rank_; + sparta::State state_; + + }; // class LoadStoreInstInfo + + /*! + * \class LoadStoreInstInfoPairDef + * \brief Pair Definition class of the load store instruction that flows through the example/CoreModel + */ + // This is the definition of the PairDefinition class of LoadStoreInstInfo. + // This PairDefinition class could be named anything but it needs to inherit + // publicly from sparta::PairDefinition templatized on the actual class LoadStoreInstInfo. + class LoadStoreInstInfoPairDef : public sparta::PairDefinition{ + public: + + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + LoadStoreInstInfoPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(LoadStoreInstInfo); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &LoadStoreInstInfo::getInstUniqueID), + SPARTA_ADDPAIR("rank", &LoadStoreInstInfo::getPriority), + SPARTA_ADDPAIR("state", &LoadStoreInstInfo::getState), + SPARTA_FLATTEN( &LoadStoreInstInfo::getMemoryAccessInfoPtr)) + }; +} // namespace core_example + +namespace simdb +{ + +template <> +inline void defineEnumMap(std::string& enum_name, std::map& map) +{ + using IssuePriority = core_example::LoadStoreInstInfo::IssuePriority; + + enum_name = "IssuePriority"; + map["HIGHEST"] = static_cast(IssuePriority::HIGHEST); + map["CACHE_RELOAD"] = static_cast(IssuePriority::CACHE_RELOAD); + map["CACHE_PENDING"] = static_cast(IssuePriority::CACHE_PENDING); + map["MMU_RELOAD"] = static_cast(IssuePriority::MMU_RELOAD); + map["MMU_PENDING"] = static_cast(IssuePriority::MMU_PENDING); + map["NEW_DISP"] = static_cast(IssuePriority::NEW_DISP); + map["LOWEST"] = static_cast(IssuePriority::LOWEST); +} + +template <> +inline void defineEnumMap(std::string& enum_name, std::map& map) +{ + using IssueState = core_example::LoadStoreInstInfo::IssueState; + + enum_name = "IssueState"; + map["READY"] = static_cast(IssueState::READY); + map["ISSUED"] = static_cast(IssueState::ISSUED); + map["NOT_READY"] = static_cast(IssueState::NOT_READY); +} + +} // namespace simdb + +inline std::ostream& operator<<(std::ostream& os, + const core_example::LoadStoreInstInfo::IssuePriority& rank){ + switch(rank){ + case core_example::LoadStoreInstInfo::IssuePriority::HIGHEST: + os << "(highest)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::CACHE_RELOAD: + os << "($_reload)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::CACHE_PENDING: + os << "($_pending)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::MMU_RELOAD: + os << "(mmu_reload)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::MMU_PENDING: + os << "(mmu_pending)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::NEW_DISP: + os << "(new_disp)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::LOWEST: + os << "(lowest)"; + break; + case core_example::LoadStoreInstInfo::IssuePriority::NUM_OF_PRIORITIES: + throw sparta::SpartaException("NUM_OF_PRIORITIES cannot be a valid enum state."); + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, + const core_example::LoadStoreInstInfo::IssueState& state){ + // Print instruction issue state + switch(state){ + case core_example::LoadStoreInstInfo::IssueState::READY: + os << "(ready)"; + break; + case core_example::LoadStoreInstInfo::IssueState::ISSUED: + os << "(issued)"; + break; + case core_example::LoadStoreInstInfo::IssueState::NOT_READY: + os << "(not_ready)"; + break; + case core_example::LoadStoreInstInfo::IssueState::NUM_STATES: + throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); + } + return os; +} + +namespace simdb +{ + +template <> +inline void defineStructSchema(StructSchema& schema) +{ + schema.setStructName("LSInstInfo"); + schema.addField("DID"); + schema.addField("rank"); + schema.addField("state"); + schema.setAutoColorizeColumn("DID"); + //schema.addFlattenField("mem_access_info_ptr"); +} + +template <> +inline void writeStructFields( + const core_example::LoadStoreInstInfo* inst, + StructFieldSerializer* serializer) +{ + serializer->writeField(inst->getInstUniqueID()); + serializer->writeField(inst->getPriority()); + serializer->writeField(inst->getState()); +} + +} // namespace simdb diff --git a/sparta/example/CoreModel/src/MSS.hpp b/sparta/example/CoreModel/src/MSS.hpp index 9e8f6b5fc7..f84242084f 100644 --- a/sparta/example/CoreModel/src/MSS.hpp +++ b/sparta/example/CoreModel/src/MSS.hpp @@ -9,7 +9,6 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" -#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "sparta/ports/SyncPort.hpp" #include "sparta/resources/Pipe.hpp" diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp new file mode 100644 index 0000000000..bf0aa0edfc --- /dev/null +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -0,0 +1,205 @@ +#pragma once + +#include "ExampleInst.hpp" +#include "simdb/collection/Structs.hpp" + +namespace core_example +{ + class MemoryAccessInfo; + using MemoryAccessInfoPtr = sparta::SpartaSharedPointer; + + // Forward declaration of the Pair Definition class is must as we are friending it. + class MemoryAccessInfoPairDef; + // Keep record of memory access information in LSU + class MemoryAccessInfo { + public: + + // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself + using SpartaPairDefinitionType = MemoryAccessInfoPairDef; + + enum class MMUState : std::uint32_t { + NO_ACCESS = 0, + __FIRST = NO_ACCESS, + MISS, + HIT, + NUM_STATES, + __LAST = NUM_STATES + }; + + enum class CacheState : std::uint64_t { + NO_ACCESS = 0, + __FIRST = NO_ACCESS, + MISS, + HIT, + NUM_STATES, + __LAST = NUM_STATES + }; + + MemoryAccessInfo() = delete; + + MemoryAccessInfo(const ExampleInstPtr & inst_ptr) : + ldst_inst_ptr_(inst_ptr), + phyAddrIsReady_(false), + mmu_access_state_(MMUState::NO_ACCESS), + + // Construct the State object here + cache_access_state_(CacheState::NO_ACCESS) {} + + virtual ~MemoryAccessInfo() {} + + // This ExampleInst pointer will act as our portal to the ExampleInst class + // and we will use this pointer to query values from functions of ExampleInst class + const ExampleInstPtr & getInstPtr() const { return ldst_inst_ptr_; } + + // This is a function which will be added in the SPARTA_ADDPAIRs API. + uint64_t getInstUniqueID() const { + const ExampleInstPtr &inst_ptr = getInstPtr(); + + return inst_ptr == nullptr ? 0 : inst_ptr->getUniqueID(); + } + + void setPhyAddrStatus(bool isReady) { phyAddrIsReady_ = isReady; } + bool getPhyAddrStatus() const { return phyAddrIsReady_; } + + const MMUState & getMMUState() const { + return mmu_access_state_.getEnumValue(); + } + + void setMMUState(const MMUState & state) { + mmu_access_state_.setValue(state); + } + + const CacheState & getCacheState() const { + return cache_access_state_.getEnumValue(); + } + void setCacheState(const CacheState & state) { + cache_access_state_.setValue(state); + } + + // This is a function which will be added in the addArgs API. + bool getPhyAddrIsReady() const{ + return phyAddrIsReady_; + } + + private: + // load/store instruction pointer + ExampleInstPtr ldst_inst_ptr_; + + // Indicate MMU address translation status + bool phyAddrIsReady_; + + // MMU access status + sparta::State mmu_access_state_; + + // Cache access status + sparta::State cache_access_state_; + + }; // class MemoryAccessInfo + + /*! + * \class MemoryAccessInfoPairDef + * \brief Pair Definition class of the Memory Access Information that flows through the example/CoreModel + */ + + // This is the definition of the PairDefinition class of MemoryAccessInfo. + // This PairDefinition class could be named anything but it needs to inherit + // publicly from sparta::PairDefinition templatized on the actual class MemoryAcccessInfo. + class MemoryAccessInfoPairDef : public sparta::PairDefinition{ + public: + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + MemoryAccessInfoPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(MemoryAccessInfo); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &MemoryAccessInfo::getInstUniqueID), + SPARTA_ADDPAIR("valid", &MemoryAccessInfo::getPhyAddrIsReady), + SPARTA_ADDPAIR("mmu", &MemoryAccessInfo::getMMUState), + SPARTA_ADDPAIR("cache", &MemoryAccessInfo::getCacheState), + SPARTA_FLATTEN( &MemoryAccessInfo::getInstPtr)) + }; + + inline std::ostream & operator<<(std::ostream & os, + const core_example::MemoryAccessInfo::MMUState & mmu_access_state){ + switch(mmu_access_state){ + case MemoryAccessInfo::MMUState::NO_ACCESS: + os << "no_access"; + break; + case MemoryAccessInfo::MMUState::MISS: + os << "miss"; + break; + case MemoryAccessInfo::MMUState::HIT: + os << "hit"; + break; + case MemoryAccessInfo::MMUState::NUM_STATES: + throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); + } + return os; + } + + inline std::ostream & operator<<(std::ostream & os, + const core_example::MemoryAccessInfo::CacheState & cache_access_state){ + switch(cache_access_state){ + case MemoryAccessInfo::CacheState::NO_ACCESS: + os << "no_access"; + break; + case MemoryAccessInfo::CacheState::MISS: + os << "miss"; + break; + case MemoryAccessInfo::CacheState::HIT: + os << "hit"; + break; + case MemoryAccessInfo::CacheState::NUM_STATES: + throw sparta::SpartaException("NUM_STATES cannot be a valid enum state."); + } + return os; + } + +} // namespace core_example + +namespace simdb +{ + +template <> +inline void defineEnumMap(std::string& enum_name, std::map& map) +{ + using MMUState = core_example::MemoryAccessInfo::MMUState; + + enum_name = "MMUState"; + map["NoAccess"] = static_cast(MMUState::NO_ACCESS); + map["Miss"] = static_cast(MMUState::MISS); + map["Hit"] = static_cast(MMUState::HIT); +} + +template <> +inline void defineEnumMap(std::string& enum_name, std::map& map) +{ + using CacheState = core_example::MemoryAccessInfo::CacheState; + + enum_name = "CacheState"; + map["NoAccess"] = static_cast(CacheState::NO_ACCESS); + map["Miss"] = static_cast(CacheState::MISS); + map["Hit"] = static_cast(CacheState::HIT); +} + +template <> +inline void defineStructSchema(StructSchema& schema) +{ + schema.setStructName("MemoryAccessInfo"); + schema.addField("DID"); + schema.addBoolField("valid"); + schema.addField("mmu"); + schema.addField("cache"); + schema.setAutoColorizeColumn("DID"); +} + +template <> +inline void writeStructFields( + const core_example::MemoryAccessInfo* inst, + StructFieldSerializer* serializer) +{ + serializer->writeField(inst->getInstUniqueID()); + serializer->writeField(inst->getPhyAddrIsReady()); + serializer->writeField(inst->getMMUState()); + serializer->writeField(inst->getCacheState()); +} + +} // namespace simdb diff --git a/sparta/example/CoreModel/src/Rename.cpp b/sparta/example/CoreModel/src/Rename.cpp index e68c30a144..c028ec2711 100644 --- a/sparta/example/CoreModel/src/Rename.cpp +++ b/sparta/example/CoreModel/src/Rename.cpp @@ -9,9 +9,6 @@ #include "Rename.hpp" #include "sparta/events/StartupEvent.hpp" #include "sparta/app/FeatureConfiguration.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" namespace core_example { @@ -62,23 +59,6 @@ namespace core_example } out_uop_queue_credits_.send(uop_queue_.size()); uop_queue_.clear(); - - if (!stop_checking_db_access_) { - if (auto container = getContainer()) { - if (auto sim = container->getSimulation()) { - if (sparta::IsFeatureValueEnabled(sim->getFeatureConfiguration(), - "wildcard-components")) - { - if (auto dbconn = GET_DB_FOR_COMPONENT(Stats, container)) { - //Run a simple query against the database just to verify - //the connection is open and accepting requests - (void) dbconn->findObject("ObjectManagersInDatabase", 1); - } - } - } - } - stop_checking_db_access_ = true; - } } void Rename::decodedInstructions_(const InstGroup & insts) diff --git a/sparta/example/CoreModel/src/Rename.hpp b/sparta/example/CoreModel/src/Rename.hpp index 4935845d7b..39aac64450 100644 --- a/sparta/example/CoreModel/src/Rename.hpp +++ b/sparta/example/CoreModel/src/Rename.hpp @@ -67,7 +67,6 @@ namespace core_example const uint32_t num_to_rename_per_cycle_; uint32_t credits_dispatch_ = 0; - bool stop_checking_db_access_ = false; //! Send initial credits void sendInitialCredits_(); diff --git a/sparta/example/CoreModel/src/main.cpp b/sparta/example/CoreModel/src/main.cpp index 56bc856b81..9e12adfbbe 100644 --- a/sparta/example/CoreModel/src/main.cpp +++ b/sparta/example/CoreModel/src/main.cpp @@ -9,6 +9,7 @@ #include "sparta/app/CommandLineSimulator.hpp" #include "sparta/app/MultiDetailOptions.hpp" #include "sparta/sparta.hpp" +#include "sparta/utils/SpartaTester.hpp" // User-friendly usage that correspond with sparta::app::CommandLineSimulator // options @@ -91,9 +92,11 @@ int main(int argc, char **argv) cls.postProcess(&sim); - }catch(...){ - // Could still handle or log the exception here - throw; + }catch(const std::exception& ex){ + TEST_INIT + EXPECT_EQUAL(std::string(ex.what()), ""); + REPORT_ERROR; + return ERROR_CODE; } return 0; } diff --git a/sparta/example/Documentation/communication/CMakeLists.txt b/sparta/example/Documentation/communication/CMakeLists.txt index d5a218fa08..775adb41d3 100644 --- a/sparta/example/Documentation/communication/CMakeLists.txt +++ b/sparta/example/Documentation/communication/CMakeLists.txt @@ -5,7 +5,7 @@ project(DOCUMENTATION_MODEL) # with different defines macro (example_comm_build name build_defines) add_executable(${name} main.cpp) - target_link_libraries(${name} sparta ${Sparta_LIBS} -L${PROJECT_BINARY_DIR}/../../.. -L${PROJECT_BINARY_DIR}/../../../simdb/) + target_link_libraries(${name} sparta ${Sparta_LIBS} -L${PROJECT_BINARY_DIR}/../../..) set_target_properties(${name} PROPERTIES COMPILE_DEFINITIONS "${build_defines}") sparta_test(${name}) endmacro(example_comm_build) diff --git a/sparta/example/DynamicModelPipeline/CMakeLists.txt b/sparta/example/DynamicModelPipeline/CMakeLists.txt index 12d7f891e1..3ccb7405f6 100644 --- a/sparta/example/DynamicModelPipeline/CMakeLists.txt +++ b/sparta/example/DynamicModelPipeline/CMakeLists.txt @@ -58,11 +58,11 @@ sparta_named_test(dynamic_model_pipeline_simultaneous_report_yamls dynamic_model sparta_named_test(dynamic_model_pipeline_simulation_control dynamic_model_pipeline -i 10k --control ctrl.yaml) sparta_named_test(dynamic_model_pipeline_simulation_control_separate_files dynamic_model_pipeline -i 10k --control ctrl_builtin.yaml --control ctrl_custom.yaml) sparta_named_test(dynamic_model_pipeline_notif_start_trigger_with_update_count dynamic_model_pipeline -i 10k --report notif_start_trigger_with_update_count.yaml) -sparta_named_test(dynamic_model_pipeline_toggle_triggers dynamic_model_pipeline -i 100k --report toggle_expression_triggers.yaml --feature simdb 0) -sparta_named_test(dynamic_model_pipeline_parameterized_toggle_triggers dynamic_model_pipeline -i 10k --report toggle_parameterized_expression_triggers.yaml --parameter top.core0.params.foo 1 --feature simdb 0) -sparta_named_test(dynamic_model_pipeline_skipped_update_csv_rows dynamic_model_pipeline -i 50k --report toggle_trigger_skipping_csv_rows.yaml --feature simdb 0) -sparta_named_test(dynamic_model_pipeline_toggle_and_start_triggers_together dynamic_model_pipeline -i 50k --report toggle_and_start_triggers_together.yaml --feature simdb 0) -sparta_named_test(dynamic_model_pipeline_toggle_trigger_enabled_and_disabled_at_same_time dynamic_model_pipeline -i 50k --report toggle_trigger_enabled_disabled_same_time.yaml --feature simdb 0) +sparta_named_test(dynamic_model_pipeline_toggle_triggers dynamic_model_pipeline -i 100k --report toggle_expression_triggers.yaml) +sparta_named_test(dynamic_model_pipeline_parameterized_toggle_triggers dynamic_model_pipeline -i 10k --report toggle_parameterized_expression_triggers.yaml --parameter top.core0.params.foo 1) +sparta_named_test(dynamic_model_pipeline_skipped_update_csv_rows dynamic_model_pipeline -i 50k --report toggle_trigger_skipping_csv_rows.yaml) +sparta_named_test(dynamic_model_pipeline_toggle_and_start_triggers_together dynamic_model_pipeline -i 50k --report toggle_and_start_triggers_together.yaml) +sparta_named_test(dynamic_model_pipeline_toggle_trigger_enabled_and_disabled_at_same_time dynamic_model_pipeline -i 50k --report toggle_trigger_enabled_disabled_same_time.yaml) sparta_named_test(dynamic_model_pipeline_bogus_nans_in_timeseries_reports dynamic_model_pipeline -i 50k --report tagged_notif_based_start_trigger.yaml) sparta_named_test(dynamic_model_pipeline_all_descriptor_patterns dynamic_model_pipeline -i 50k --report all_descriptor_patterns.yaml --num-cores 2) sparta_named_test(dynamic_model_pipeline_retired_inst_path dynamic_model_pipeline -i 10k --report all_descriptor_patterns.yaml --retired-inst-counter-path rename.stats.rename_uop_queue_utilization_count0 --num-cores 2) @@ -71,7 +71,7 @@ sparta_named_test(dynamic_model_pipeline_two_reports_json_full_format dynamic_mo sparta_named_test(dynamic_model_pipeline_two_reports_json_detail_format dynamic_model_pipeline -i 10k --report multireports_json_detail.yaml) sparta_named_test(dynamic_model_pipeline_two_reports_json_reduced_format dynamic_model_pipeline -i 10k --report multireports_json_reduced.yaml) sparta_named_test(dynamic_model_pipeline_tree_node_extensions_written_to_final_config dynamic_model_pipeline -i 10k --arch-search-dir . --arch extensions_in_arch_file.yaml --config-file extensions_in_config_file.yaml --extension-file tree_node_extensions.yaml --write-final-config final_cfg_with_extensions.yaml) -sparta_named_test(dynamic_model_pipeline_update_reports_on_demand dynamic_model_pipeline -i 10k --report update_report_on_demand.yaml --feature simdb 0) +sparta_named_test(dynamic_model_pipeline_update_reports_on_demand dynamic_model_pipeline -i 10k --report update_report_on_demand.yaml) sparta_named_test(dynamic_model_pipeline_context_counter_trigger_expressions dynamic_model_pipeline -i 10k --report context_counter_report_triggers.yaml -p top.core0.params.foo 8k) sparta_named_test(dynamic_model_pipeline_weighted_context_counter_trigger_expressions dynamic_model_pipeline -i 10k --report weighted_context_counter_report_triggers.yaml -p top.core0.params.foo 8k --config-file context_weights.yaml) if ($ENV{MEMORY_PROFILING_ENABLED}) @@ -110,17 +110,3 @@ sparta_named_test(dynamic_model_pipeline_arch_with_extensions dynamic_model_pipe # Extensions supplied that are not bound are now errors #sparta_named_test(dynamic_model_pipeline_indiv_tree_node_extensions dynamic_model_pipeline -i 10k -p top.core0.lsu.extension.dog.language_ woof --extension-file tree_node_extensions.yaml -p top.core0.lsu.extension.cat.language_ meow --write-final-config final.yaml) sparta_named_test(dynamic_model_pipeline_indiv_tree_node_extensions dynamic_model_pipeline -i 10k --extension-file tree_node_extensions.yaml -p top.core0.lsu.extension.cat.language_ meow --write-final-config final.yaml) -sparta_named_test(dynamic_model_pipeline_sqldb_timeseries dynamic_model_pipeline -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_min_blob_size.yaml) -sparta_named_test(dynamic_model_pipeline_sqldb_timeseries_opts1 dynamic_model_pipeline -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts1.yaml) -sparta_named_test(dynamic_model_pipeline_sqldb_timeseries_opts2 dynamic_model_pipeline -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts2.yaml) -sparta_named_test(dynamic_model_pipeline_sqldb_timeseries_opts3 dynamic_model_pipeline -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts3.yaml) -sparta_named_test(dynamic_model_pipeline_sqldb_timeseries_opts4 dynamic_model_pipeline -i 100k --report all_update_triggers.yaml --feature simdb 1 simdb_si_opts4.yaml) -sparta_named_test(dynamic_model_pipeline_simdb_non_timeseries dynamic_model_pipeline -i 10k --report multireport_r1.yaml --feature simdb 1) -#sparta_named_test(dynamic_model_pipeline_all_report_formats_no_verif dynamic_model_pipeline -i 20k --report all_report_formats.yaml --feature simdb 1) -sparta_named_test(dynamic_model_pipeline_expected_dest_file_location dynamic_model_pipeline -i 20k --report all_report_formats.yaml --report-verif-output-dir @ --feature simdb 1 --feature simdb-verify 1) -sparta_named_test(dynamic_model_pipeline_csv_headers_in_db dynamic_model_pipeline -i 100k --report csv_headers_in_db.yaml --feature simdb 1 --feature simdb-verify 1) -sparta_named_test(dynamic_model_pipeline_simdb_configure_opts1 dynamic_model_pipeline -i 10k --report all_update_triggers.yaml --simdb-dir .) -sparta_named_test(dynamic_model_pipeline_simdb_configure_opts2 dynamic_model_pipeline -i 10k --report all_update_triggers.yaml --simdb-dir myfolder) -sparta_named_test(dynamic_model_pipeline_simdb_configure_opts3 dynamic_model_pipeline -i 10k --report all_update_triggers.yaml --simdb-dir my/sub/folder) -sparta_named_test(dynamic_model_pipeline_simdb_configure_opts4 dynamic_model_pipeline -i 10k --report all_update_triggers.yaml --simdb-dir myfolder --simdb-enabled-components dbaccess_opts1.yaml) -sparta_named_test(dynamic_model_pipeline_simdb_perf_async_task_controller dynamic_model_pipeline -i 50k --feature simdb-perf-async-ctrl 1) diff --git a/sparta/example/DynamicModelPipeline/src/ExampleSimulation.cpp b/sparta/example/DynamicModelPipeline/src/ExampleSimulation.cpp index a833b40724..887773a549 100644 --- a/sparta/example/DynamicModelPipeline/src/ExampleSimulation.cpp +++ b/sparta/example/DynamicModelPipeline/src/ExampleSimulation.cpp @@ -13,13 +13,6 @@ #include "sparta/simulation/TreeNodeExtensions.hpp" #include "sparta/trigger/ContextCounterTrigger.hpp" #include "sparta/utils/StringUtils.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" -#include "simdb/utils/uuids.hpp" -#include "simdb/utils/ObjectQuery.hpp" #include "Fetch.hpp" #include "Decode.hpp" @@ -35,269 +28,6 @@ #include "BIU.hpp" #include "MSS.hpp" -namespace { - - // Struct for writing and verifying SQLite records. - // See buildSchemaA() below. - struct TestSQLiteSchemaA { - struct Numbers { - double First; - double Second; - }; - Numbers numbers; - - struct Metadata { - std::string Name; - double Value; - }; - Metadata metadata; - - static TestSQLiteSchemaA createRandom() { - TestSQLiteSchemaA s; - s.numbers.First = rand() / 1000 * 3.14; - s.numbers.Second = rand() / 1000 * 3.14; - s.metadata.Name = simdb::generateUUID(); - s.metadata.Value = rand() / 1000 * 3.14; - return s; - } - }; - - // Another struct for writing and verifying SQLite - // records. See buildSchemaB() below. - struct TestSQLiteSchemaB { - struct Strings { - std::string First; - std::string Second; - }; - Strings strings; - - struct Metadata { - std::string Name; - std::string Value; - }; - Metadata metadata; - - static TestSQLiteSchemaB createRandom() { - TestSQLiteSchemaB s; - s.strings.First = simdb::generateUUID(); - s.strings.Second = simdb::generateUUID(); - s.metadata.Name = simdb::generateUUID(); - s.metadata.Value = simdb::generateUUID(); - return s; - } - }; - - // Struct for writing and verifying HDF5 records - struct TestHDF5SchemaC { - double x; - double y; - uint16_t z; - - static TestHDF5SchemaC createRandom() { - TestHDF5SchemaC s; - s.x = rand() / 1000 * 3.14; - s.y = rand() / 1000 * 3.14; - s.z = rand(); - return s; - } - }; - - // Helper class which creates random SQLite / HDF5 - // structs for SimDB writes, and stores the structs - // in memory too. The data will be read back from - // the database at the end of simulation, and the - // values retrieved from file will be compared with - // the values that were stored in memory. - class DatabaseTester { - public: - static DatabaseTester & getTester() { - static DatabaseTester tester; - return tester; - } - - ~DatabaseTester() = default; - - const TestSQLiteSchemaA & createAndStoreRecordForSQLiteSchemaA() { - if (records_schemaA_.size() < 100) { - indices_schemaA_.emplace_back(records_schemaA_.size()); - records_schemaA_.emplace_back(TestSQLiteSchemaA::createRandom()); - return records_schemaA_.back(); - } else { - indices_schemaA_.emplace_back(rand() % records_schemaA_.size()); - return records_schemaA_[indices_schemaA_.back()]; - } - } - - const TestSQLiteSchemaB & createAndStoreRecordForSQLiteSchemaB() { - if (records_schemaB_.size() < 100) { - indices_schemaB_.emplace_back(records_schemaB_.size()); - records_schemaB_.emplace_back(TestSQLiteSchemaB::createRandom()); - return records_schemaB_.back(); - } else { - indices_schemaB_.emplace_back(rand() % records_schemaB_.size()); - return records_schemaB_[indices_schemaB_.back()]; - } - } - - const TestHDF5SchemaC & createAndStoreRecordForHDF5SchemaC() { - records_schemaC_.emplace_back(TestHDF5SchemaC::createRandom()); - return records_schemaC_.back(); - } - - const std::vector & getWrittenRecordsForSchemaA() const { - return records_schemaA_; - } - - const std::vector & getWrittenRecordsForSchemaB() const { - return records_schemaB_; - } - - const std::vector & getWrittenRecordsForSchemaC() const { - return records_schemaC_; - } - - void verifyRecords(const std::string & db_file) const { - simdb::ObjectManager obj_mgr("."); - if (!obj_mgr.connectToExistingDatabase(db_file)) { - return; - } - - auto numeric_db = GET_DB_FROM_CURRENT_SIMULATION(NumericMeta); - if (numeric_db) { - auto values_query = - numeric_db->createObjectQueryForTable("Numbers"); - - if (values_query) { - double first, second; - values_query->writeResultIterationsTo( - "First", &first, "Second", &second); - - if (values_query->countMatches() != indices_schemaA_.size()) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - - auto result_iter = values_query->executeQuery(); - size_t record_idx = 0; - while (result_iter->getNext()) { - const auto & expected = records_schemaA_[indices_schemaA_[record_idx]]; - if (first != expected.numbers.First) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - if (second != expected.numbers.Second) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - ++record_idx; - } - } - - auto meta_query = - numeric_db->createObjectQueryForTable("Metadata"); - if (meta_query) { - std::string name; - double value; - meta_query->writeResultIterationsTo("Name", &name, "Value", &value); - - if (meta_query->countMatches() != indices_schemaA_.size()) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - - auto result_iter = meta_query->executeQuery(); - size_t record_idx = 0; - while (result_iter->getNext()) { - const auto & expected = records_schemaA_[indices_schemaA_[record_idx]]; - if (name != expected.metadata.Name) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - if (value != expected.metadata.Value) { - throw sparta::SpartaException("Could not verify SimDB records"); - } - ++record_idx; - } - } - } - } - - private: - DatabaseTester() = default; - std::vector records_schemaA_; - std::vector records_schemaB_; - std::vector records_schemaC_; - std::vector indices_schemaA_; - std::vector indices_schemaB_; - std::vector indices_schemaC_; - }; - - // Schema builder to test two simdb::ObjectManager's - // bound to the same database file, separated in that - // same file by their respective application name. - // A third schema builder is for another ObjectManager, - // though it will be used to write records to an HDF5 - // database, and therefore will be in its own file. - // SimDB's worker thread should be able to keep them - // separated into two groups: one group for the two - // SQLite database connections, and one group only - // serving the one HDF5 connection. - // - // Note that the two schema builders below have some - // overlap in their table definitions: schemaA and - // schemaB have some of the same table names, but - // these tables have different column configurations. - // This should not be a problem for ObjectManager - // since it will use its unique application name - // with the table names we give it to create a - // unique schema inside the shared file, separated - // from other applications tied to the same file. - // The specific way in which the schemas are kept - // separate in the file is not our concern; the - // DbConnProxy subclasses take care of those - // specifics. - void buildSchemaA(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addColumn("First", dt::double_t) - .addColumn("Second", dt::double_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::double_t); - } - - void buildSchemaB(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Strings") - .addColumn("First", dt::string_t) - .addColumn("Second", dt::string_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::string_t); - } - - void buildSchemaC(simdb::Schema & schema) - { - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addField("x", dt::double_t, FOFFSET(TestHDF5SchemaC,x)) - .addField("y", dt::double_t, FOFFSET(TestHDF5SchemaC,y)) - .addField("z", dt::uint16_t, FOFFSET(TestHDF5SchemaC,z)); - } - - simdb::DbConnProxy * createSQLiteProxy() - { - return new simdb::SQLiteConnProxy; - } - - simdb::DbConnProxy * createHDF5Proxy() - { - return new simdb::HDF5ConnProxy; - } -} - namespace sparta { // Example parameter set used to reproduce write-final-config @@ -329,11 +59,6 @@ namespace sparta { TreeNode(parent, "baz_node", "BazGroup", 0, desc) { baz_.reset(new IntParameterSet(this)); - if (auto dbconn = GET_DB_FOR_COMPONENT(Stats, this)) { - //Run a simple query against the database just to verify - //the connection is open and accepting requests - (void) dbconn->findObject("ObjectManagersInDatabase", 1); - } } void readParams() { @@ -427,15 +152,6 @@ double calculateAverageOfInternalCounters( return agg / counters.size(); } -void tryAccessSimDB() -{ - if (auto dbconn = GET_DB_FROM_CURRENT_SIMULATION(Stats)) { - //Run a simple query against the database just to verify - //the connection is open and accepting requests - (void) dbconn->findObject("ObjectManagersInDatabase", 1); - } -} - ExampleSimulator::ExampleSimulator(sparta::Scheduler & scheduler, uint32_t num_cores, uint64_t instruction_limit, @@ -484,20 +200,6 @@ ExampleSimulator::ExampleSimulator(sparta::Scheduler & scheduler, // definition YAML files. sparta::trigger::ContextCounterTrigger::registerContextCounterCalcFunction( "avg", &calculateAverageOfInternalCounters); - - //SQLite namespaces: NumericMeta & StringMeta - REGISTER_SIMDB_NAMESPACE(NumericMeta, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(NumericMeta, buildSchemaA); - - REGISTER_SIMDB_NAMESPACE(StringMeta, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(StringMeta, buildSchemaB); - - //HDF5 namespace: NumericVals - REGISTER_SIMDB_NAMESPACE(NumericVals, HDF5); - REGISTER_SIMDB_SCHEMA_BUILDER(NumericVals, buildSchemaC); - - //Proxy factory registration - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(HDF5, createHDF5Proxy); } @@ -508,17 +210,6 @@ ExampleSimulator::~ExampleSimulator() getRoot()->DEREGISTER_FOR_NOTIFICATION( onTriggered_, std::string, "sparta_expression_trigger_fired"); } - - if (simdb_perf_async_ctrl_enabled_) { - std::set simdb_files; - if (auto dbconn = GET_DB_FOR_COMPONENT(NumericMeta, this)) { - simdb_files.insert(dbconn->getDatabaseFile()); - } - - for (const auto & db_file : simdb_files) { - DatabaseTester::getTester().verifyRecords(db_file); - } - } } void ExampleSimulator::buildTree_() @@ -749,55 +440,6 @@ void ExampleSimulator::buildTree_() void ExampleSimulator::configureTree_() { - //Context-aware SimDB access - std::pair sqlite_db_files; - if (auto dbconn = GET_DB_FOR_COMPONENT(NumericMeta, this)) { - const TestSQLiteSchemaA data = DatabaseTester::getTester(). - createAndStoreRecordForSQLiteSchemaA(); - - const double first = data.numbers.First; - const double second = data.numbers.Second; - dbconn->getTable("Numbers")->createObjectWithArgs( - "First", first, "Second", second); - - const std::string meta_name = data.metadata.Name; - const double meta_value = data.metadata.Value; - dbconn->getTable("Metadata")->createObjectWithArgs( - "Name", meta_name, "Value", meta_value); - - sqlite_db_files.first = dbconn->getDatabaseFile(); - - //Verification of the two records we just made above - //will occur at the end of the simulation. - } - - if (auto dbconn = GET_DB_FOR_COMPONENT(StringMeta, this)) { - const TestSQLiteSchemaB data = DatabaseTester::getTester(). - createAndStoreRecordForSQLiteSchemaB(); - - const std::string first = data.strings.First; - const std::string second = data.strings.Second; - dbconn->getTable("Strings")->createObjectWithArgs( - "First", first, "Second", second); - - const std::string meta_name = data.metadata.Name; - const std::string meta_value = data.metadata.Value; - dbconn->getTable("Metadata")->createObjectWithArgs( - "Name", meta_name, "Value", meta_value); - - sqlite_db_files.second = dbconn->getDatabaseFile(); - - //Verification of the two records we just made above - //will occur at the end of the simulation. - } - - //Both of the ObjectManager's used above should have put the - //created records into the same file. - sparta_assert(sqlite_db_files.first == sqlite_db_files.second); - - //Context-unaware SimDB access - tryAccessSimDB(); - validateTreeNodeExtensions_(); // In TREE_CONFIGURING phase @@ -838,9 +480,6 @@ void ExampleSimulator::configureTree_() getRoot()->REGISTER_FOR_NOTIFICATION( onTriggered_, std::string, "sparta_expression_trigger_fired"); on_triggered_notifier_registered_ = true; - - simdb_perf_async_ctrl_enabled_ = sparta::IsFeatureValueEnabled( - getFeatureConfiguration(), "simdb-perf-async-ctrl") > 0; } void ExampleSimulator::bindTree_() diff --git a/sparta/example/DynamicModelPipeline/src/ExampleSimulation.hpp b/sparta/example/DynamicModelPipeline/src/ExampleSimulation.hpp index d59463e47e..5c880690d4 100644 --- a/sparta/example/DynamicModelPipeline/src/ExampleSimulation.hpp +++ b/sparta/example/DynamicModelPipeline/src/ExampleSimulation.hpp @@ -121,13 +121,5 @@ class ExampleSimulator : public sparta::app::Simulation * \brief If present, test tree node extensions */ void validateTreeNodeExtensions_(); - - /*! - * \brief Flag which enables SimDB-related code to run for - * interactive performance benchmarks / comparison. False - * by default so that we don't impact unit testing / smoke - * testing times for all regression test runs. - */ - bool simdb_perf_async_ctrl_enabled_ = false; }; diff --git a/sparta/python/sparta_support/PythonInterpreter.cpp b/sparta/python/sparta_support/PythonInterpreter.cpp index 2ee8f7153c..fe1200d81d 100644 --- a/sparta/python/sparta_support/PythonInterpreter.cpp +++ b/sparta/python/sparta_support/PythonInterpreter.cpp @@ -21,7 +21,6 @@ #include "sparta/app/ReportDescriptor.hpp" #include "sparta/statistics/dispatch/archives/StatisticsArchives.hpp" #include "sparta/statistics/dispatch/streams/StreamController.hpp" -#include "simdb/async/AsyncTaskEval.hpp" static const std::thread::id main_thread_id = std::this_thread::get_id(); @@ -377,51 +376,6 @@ void PythonInterpreter::publishStatisticsStreams(statistics::StatisticsStreams * published_obj_names_[streams] = "stream_config"; } -void PythonInterpreter::publishSimulationDatabase(simdb::ObjectManager * sim_db) -{ - sparta_assert(sim_db, "Cannot publish null simdb::ObjectManager " - "object to Python environment"); - - sparta_assert(Py_IsInitialized(), - "Attempted to publish simulation database object when Python " - "was not initialized"); - - auto global_ns = getGlobalNS(); - - try { - global_ns["sim_db"] = WrapperCache().wrap(sim_db); - std::cout << "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n" - << "* Simulation Database: \n" - << "* * * You can now access any timeseries data produced by the simulator's \n" - "* * * statistics reporting engine using the 'sim_db' object. \n" - << "* * * " << std::endl; - } catch (bp::error_already_set&) { - PyErr_Print(); - } - - published_obj_names_[sim_db] = "sim_db"; -} - -void PythonInterpreter::publishDatabaseController(simdb::AsyncTaskEval * db_queue) -{ - sparta_assert(db_queue, "Cannot publish null simdb::AsyncTaskEval " - "object to Python environment"); - - sparta_assert(Py_IsInitialized(), - "Attempted to publish database controller object when Python " - "was not initialized"); - - auto global_ns = getGlobalNS(); - - try { - global_ns["__db_queue"] = WrapperCache().wrap(db_queue); - } catch (bp::error_already_set&) { - PyErr_Print(); - } - - published_obj_names_[db_queue] = "__db_queue"; -} - void PythonInterpreter::publishSimulator(app::Simulation * sim) { sparta_assert(sim, "Cannot publish null app::Simulator object to Python environment"); sparta_assert(Py_IsInitialized(), diff --git a/sparta/python/sparta_support/PythonInterpreter.hpp b/sparta/python/sparta_support/PythonInterpreter.hpp index dea3ca3e42..939bbcf958 100644 --- a/sparta/python/sparta_support/PythonInterpreter.hpp +++ b/sparta/python/sparta_support/PythonInterpreter.hpp @@ -9,7 +9,6 @@ #pragma once #include "sparta/sparta.hpp" // For macro definitions -#include "simdb_fwd.hpp" #ifndef _GNU_SOURCE #define _GNU_SOURCE @@ -43,10 +42,6 @@ namespace statistics { class StatisticsStreams; } -namespace async { - class AsynchronousTaskEval; -} - namespace python { /*! @@ -148,10 +143,6 @@ namespace python { void publishStatisticsStreams(statistics::StatisticsStreams * streams); - void publishSimulationDatabase(simdb::ObjectManager * sim_db); - - void publishDatabaseController(simdb::AsyncTaskEval * db_queue); - void publishSimulator(app::Simulation * sim); void publishTree(RootTreeNode * n); diff --git a/sparta/python/sparta_support/module_sparta.cpp b/sparta/python/sparta_support/module_sparta.cpp index 2b104ea026..859f853fee 100644 --- a/sparta/python/sparta_support/module_sparta.cpp +++ b/sparta/python/sparta_support/module_sparta.cpp @@ -33,13 +33,6 @@ #include "python/sparta_support/PythonInterpreter.hpp" #include "python/sparta_support/facade/ReportDescriptor.hpp" #include "sparta/statistics/dispatch/archives/ReportStatisticsArchive.hpp" -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/report/db/format/toCSV.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/async/AsyncTaskEval.hpp" #include #include @@ -648,60 +641,6 @@ sparta::statistics::StatisticsArchives * StatisticsArchives__import( return iter->second.get(); } -// In-memory cache of timeseries wrappers. These are used both during -// simulation and outside of a simulation, and are put in global scope. -// After simulation, there is no clear owner of a database wrapper, so -// we are making these objects global. -std::unordered_map> loaded_db_timeseries; - -// In-memory cache of SQLite database connections. These are used outside -// of a simulation to connect to an existing database. Without a simulator, -// ownership of database connections is vague at best, so we just put them -// in global scope. -std::unordered_map> db_connections; - -simdb::ObjectManager * ReportTimeseries__connectToDatabase( - const std::string & db_fullpath) -{ - //First check if this DB connection is already alive - auto iter = db_connections.find(db_fullpath); - if (iter != db_connections.end()) { - return iter->second.get(); - } - - //Let's check the database path/filename of the simulator's "sim_db" - //object that was published to the Python namespace, if there was one. - //Having multiple connections open for the same database file may cause - //performance issues. - bp::object main = bp::import("__main__"); - bp::object global_ns = main.attr("__dict__"); - - if (global_ns.contains(bp::str("sim_db"))) { - simdb::ObjectManager * sim_db = - bp::extract(global_ns["sim_db"]); - - sparta_assert(sim_db != nullptr); - if (sim_db->getDatabaseFile() == db_fullpath) { - return sim_db; - } - } - - //Instantiate and cache a new database connection - std::shared_ptr obj_mgr(new simdb::ObjectManager); - if (!obj_mgr->connectToExistingDatabase(db_fullpath)) { - //The ObjectManager rejected this database file for some reason - std::cout << "ERROR! This is not a valid database file: " - << db_fullpath << std::endl; - return nullptr; - } - - //Looks good. Cache it in memory and return it. - db_connections[db_fullpath] = obj_mgr; - return obj_mgr.get(); -} - bp::object StatisticsArchives__getattr__(sparta::statistics::StatisticsArchives * n, const std::string & attr) { @@ -1134,573 +1073,6 @@ bp::object StreamNode__streamTo(bp::tuple args, bp::dict kwargs) return bp::object(bp::borrowed(Py_None)); } -//! Recreate a sparta::Report and its sparta::StatisticInstance objects from -//! from the provided report node database ID, living in the provided -//! ObjectManager (SimDB). You may call this method for any report format -//! *except* csv and csv_cumulative: -//! -//! txt, text, html, htm, js_json, jjson, python, py, json, JSON, -//! json_reduced, JSON_reduced, json_detail, JSON_detail, gnuplot, -//! gplt, stats_mapping -//! -void LOCAL_SimulationDatabase__createReport( - const simdb::ObjectManager * sim_db, - const int report_db_id, - const std::string & filename, - const std::string & format, - const sparta::Scheduler * scheduler) -{ - if (!sparta::Report::createFormattedReportFromDatabase( - *sim_db, report_db_id, filename, format, scheduler)) - { - std::cout << "Unable to create report file '" - << filename << "' " << std::endl; - } -} - -enum class SimDBReportType { - AutoSummary, - Json, - JsonReduced, - JsonDetail, - JsJson, - Html, - Text, - PyDictionary, - GnuPlot, - StatsMapping -}; - -template -void SimulationDatabase__createReport( - simdb::ObjectManager * sim_db, - const int report_db_id, - const std::string & filename, - const sparta::Scheduler * scheduler) -{ - using rt = SimDBReportType; - - switch (type) { - case rt::AutoSummary: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "auto", scheduler); - break; - } - case rt::Json: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "json", scheduler); - break; - } - case rt::JsonReduced: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "json_reduced", scheduler); - break; - } - case rt::JsonDetail: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "json_detail", scheduler); - break; - } - case rt::JsJson: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "js_json", scheduler); - break; - } - case rt::Html: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "html", scheduler); - break; - } - case rt::Text: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "txt", scheduler); - break; - } - case rt::PyDictionary: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "python", scheduler); - break; - } - case rt::GnuPlot: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "gnuplot", scheduler); - break; - } - case rt::StatsMapping: { - LOCAL_SimulationDatabase__createReport( - sim_db, report_db_id, filename, "stats_mapping", scheduler); - break; - } - } -} - -//! At the end of SPARTA simulations, the database report contents -//! may be checked for accuracy against a known correct/expected -//! report. For debugging purposes, any discrepancies encountered -//! during this post-simulation verification step may be put into -//! the database. This method prints the verification summaries. -void LOCAL_SimulationDatabase__printVerifFailureSummary( - const simdb::ObjectManager & sim_db, - const simdb::DatabaseID report_verif_result_id, - const simdb::DatabaseID sim_info_id) -{ - std::string failure_summary; - simdb::ObjectQuery summary_query(sim_db, "ReportVerificationFailureSummaries"); - - summary_query.addConstraints( - "ReportVerificationResultID", - simdb::constraints::equal, - report_verif_result_id); - - summary_query.writeResultIterationsTo("FailureSummary", &failure_summary); - auto result_iter = summary_query.executeQuery(); - if (result_iter->getNext()) { - std::cout << failure_summary << "\n"; - } - - simdb::ObjectQuery sim_info_query(sim_db, "SimInfo"); - sim_info_query.addConstraints("Id", simdb::constraints::equal, sim_info_id); - - std::string sinfo_name, sinfo_cmdline, sinfo_workingdir, sinfo_simversion; - std::string sinfo_exe, sinfo_spartaversion, sinfo_repro, sinfo_other; - - sim_info_query.writeResultIterationsTo( - "Name", &sinfo_name, - "Cmdline", &sinfo_cmdline, - "WorkingDir", &sinfo_workingdir, - "Exe", &sinfo_exe, - "SimulatorVersion", &sinfo_simversion, - "SpartaVersion", &sinfo_spartaversion, - "Repro", &sinfo_repro, - "Other", &sinfo_other); - - auto change_unset_to_hyphen = [](std::string & str) { - if (str == "unset") { - str = "-"; - } - }; - change_unset_to_hyphen(sinfo_name); - change_unset_to_hyphen(sinfo_cmdline); - change_unset_to_hyphen(sinfo_workingdir); - change_unset_to_hyphen(sinfo_exe); - change_unset_to_hyphen(sinfo_simversion); - change_unset_to_hyphen(sinfo_spartaversion); - change_unset_to_hyphen(sinfo_repro); - change_unset_to_hyphen(sinfo_other); - - result_iter = sim_info_query.executeQuery(); - if (result_iter->getNext()) { - std::cout << " Name: " << sinfo_name << "\n"; - std::cout << " Cmdline: " << sinfo_cmdline << "\n"; - std::cout << " WorkingDir: " << sinfo_workingdir << "\n"; - std::cout << " Exe: " << sinfo_exe << "\n"; - std::cout << " SimulatorVersion: " << sinfo_simversion << "\n"; - std::cout << " SpartaVersion: " << sinfo_spartaversion << "\n"; - std::cout << " Repro: " << sinfo_repro << "\n"; - std::cout << " Other: " << sinfo_other << "\n\n"; - std::cout << std::endl; - } - std::cout << std::endl; -} - -//! Utility to print out a high-level pass/fail summary for all -//! report verification checks run on the given database. -void SimulationDatabase__printVerificationSummary( - simdb::ObjectManager * sim_db, - const bool verbose) -{ - sim_db->safeTransaction([&]() { - bool summary_header_printed = false; - auto print_summary_header = [&summary_header_printed, sim_db]() { - if (!summary_header_printed) { - std::cout << "- - - - - - - - Report Verification Summary - - - - - - - " << std::endl; - std::cout << " (" << sim_db->getDatabaseFile() << ")\n" << std::endl; - summary_header_printed = true; - } - }; - if (verbose) { - print_summary_header(); - } - if (verbose) { - std::cout << "PASSED:\n"; - simdb::ObjectQuery passed_query(*sim_db, "ReportVerificationResults"); - passed_query.addConstraints("Passed", simdb::constraints::equal, 1); - - std::string dest_file; - int is_timeseries; - passed_query.writeResultIterationsTo( - "DestFile", &dest_file, "IsTimeseries", &is_timeseries); - - auto result_iter = passed_query.executeQuery(); - std::set passed_timeseries, passed_non_timeseries; - while (result_iter->getNext()) { - if (is_timeseries) { - passed_timeseries.insert(dest_file); - } else { - passed_non_timeseries.insert(dest_file); - } - } - - std::cout << " Timeseries...\n"; - if (passed_timeseries.empty()) { - std::cout << " (none)\n"; - } else { - for (const auto & pass : passed_timeseries) { - std::cout << " " << pass << "\n"; - } - } - - std::cout << std::endl; - std::cout << " Non-timeseries...\n"; - if (passed_non_timeseries.empty()) { - std::cout << " (none)\n"; - } else { - for (const auto & pass : passed_non_timeseries) { - std::cout << " " << pass << "\n"; - } - } - std::cout << "\n" << std::endl; - } - - bool fail_header_printed = false; - auto print_failure_header = [&fail_header_printed]() { - if (!fail_header_printed) { - std::cout << "FAILED:\n"; - fail_header_printed = true; - } - }; - - simdb::ObjectQuery failed_query(*sim_db, "ReportVerificationResults"); - failed_query.addConstraints("Passed", simdb::constraints::equal, 0); - - if (verbose || failed_query.countMatches() > 0) { - print_summary_header(); - print_failure_header(); - } - - std::string dest_file; - int is_timeseries; - simdb::DatabaseID result_verif_id; - simdb::DatabaseID sim_info_id; - - failed_query.writeResultIterationsTo( - "Id", &result_verif_id, - "DestFile", &dest_file, - "IsTimeseries", &is_timeseries, - "SimInfoID", &sim_info_id); - - using ResultVerifAndSimInfoIDs = std::pair; - auto result_iter = failed_query.executeQuery(); - std::map failed_timeseries; - std::map failed_non_timeseries; - - while (result_iter->getNext()) { - if (is_timeseries) { - failed_timeseries[dest_file] = - std::make_pair(result_verif_id, sim_info_id); - } else { - failed_non_timeseries[dest_file] = - std::make_pair(result_verif_id, sim_info_id); - } - } - - if (!verbose && failed_timeseries.empty() && failed_non_timeseries.empty()) { - return; - } else { - print_failure_header(); - } - - if (verbose || !failed_timeseries.empty()) { - std::cout << " Timeseries...\n"; - if (failed_timeseries.empty()) { - std::cout << " (none)\n"; - } else { - for (const auto & fail : failed_timeseries) { - std::cout << " " << fail.first << "\n"; - LOCAL_SimulationDatabase__printVerifFailureSummary( - *sim_db, fail.second.first, fail.second.second); - std::cout << " + + + + + + + + + + + + + + + + + + +\n"; - } - } - } - - if (verbose || !failed_non_timeseries.empty()) { - std::cout << " Non-timeseries...\n"; - if (failed_non_timeseries.empty()) { - std::cout << " (none)\n"; - } else { - for (const auto & fail : failed_non_timeseries) { - std::cout << " " << fail.first << "\n"; - LOCAL_SimulationDatabase__printVerifFailureSummary( - *sim_db, fail.second.first, fail.second.second); - std::cout << " + + + + + + + + + + + + + + + + + + +\n"; - } - } - } - - std::cout << std::endl; - }); -} - -//! Utility to print out a high-level pass/fail summary for all -//! report verification checks run on all databases (*.db files) -//! found in the given directory. -void SimulationDatabase__printAllVerificationSummaries( - const std::string & simdb_dir, - const bool verbose) -{ - namespace sfs = std::filesystem; - - auto p = sfs::path(simdb_dir); - if (!sfs::exists(p) || !sfs::is_directory(p)) { - std::cout << "Not a valid directory: '" << simdb_dir << "'\n" << std::endl; - return; - } - - for (sfs::directory_iterator iter(p); iter != sfs::directory_iterator(); ++iter) { - if (sfs::is_regular_file(iter->status()) && iter->path().extension().string() == ".db") { - const std::string db_full_filename = iter->path().string(); - sfs::path db_path(db_full_filename); - const std::string db_filename = db_path.stem().string() + db_path.extension().string(); - simdb::ObjectManager sim_db(simdb_dir); - if (sim_db.connectToExistingDatabase(db_filename)) { - SimulationDatabase__printVerificationSummary(&sim_db, verbose); - } else { - std::cout << "Unable to open database file: '" << db_filename << "'\n"; - } - } - } -} - -//! Hidden utility which prints out a ".expected" and a -//! ".actual" report file that was saved / deep copied -//! into the database during the post-simulation report verification -//! check. -void SimulationDatabase__getVerificationFailureReportDiffs( - simdb::ObjectManager * sim_db, - const std::string & orig_dest_file) -{ - sim_db->safeTransaction([&]() { - simdb::ObjectQuery query(*sim_db, "ReportVerificationDeepCopyFiles"); - query.addConstraints("DestFile", simdb::constraints::equal, orig_dest_file); - - std::string expected, actual; - query.writeResultIterationsTo("Expected", &expected, "Actual", &actual); - - std::vector> deep_copy_files; - auto result_iter = query.executeQuery(); - size_t suffix_idx = 1; - while (result_iter->getNext()) { - const std::string suffix = (suffix_idx++ > 1) ? - ("." + boost::lexical_cast(suffix_idx)) : ""; - const std::string dest_file_expected = orig_dest_file + ".expected" + suffix; - const std::string dest_file_actual = orig_dest_file + ".actual" + suffix; - - std::ofstream fout_expected(dest_file_expected); - fout_expected << expected; - - std::ofstream fout_actual(dest_file_actual); - fout_actual << actual; - - deep_copy_files.emplace_back(dest_file_expected, dest_file_actual); - } - - if (!deep_copy_files.empty()) { - std::cout << "The following files can be diff'd for discrepancies:\n"; - for (const auto & diff_files : deep_copy_files) { - std::cout << diff_files.first << "\n" << diff_files.second << "\n"; - } - std::cout << std::endl; - } - }); -} - -//! Go through the given directory, and look into each "*.db" file we find -//! for any reports that failed the post-simulation verification check. -void SimulationDatabase__getVerificationFailuresInDir( - const std::string & simdb_dir) -{ - namespace sfs = std::filesystem; - auto p = sfs::path(simdb_dir); - if (!sfs::exists(p) || !sfs::is_directory(p)) { - std::cout << "Not a valid directory: '" << simdb_dir << "'\n" << std::endl; - return; - } - - std::map> db_subdirs_with_failures; - for (sfs::directory_iterator iter(p); iter != sfs::directory_iterator(); ++iter) { - if (sfs::is_regular_file(iter->status()) && iter->path().extension().string() == ".db") { - const std::string db_full_filename = iter->path().string(); - sfs::path db_path(db_full_filename); - const std::string db_filename = db_path.stem().string() + db_path.extension().string(); - simdb::ObjectManager sim_db(simdb_dir); - if (sim_db.connectToExistingDatabase(db_filename)) { - simdb::ObjectQuery query(sim_db, "ReportVerificationDeepCopyFiles"); - - std::string filename; - query.writeResultIterationsTo("DestFile", &filename); - - auto result_iter = query.executeQuery(); - while (result_iter->getNext()) { - SimulationDatabase__getVerificationFailureReportDiffs(&sim_db, filename); - db_subdirs_with_failures[db_filename].insert(filename); - } - } - } - } - - if (!db_subdirs_with_failures.empty()) { - std::cout << "The following database files had report " - << "verification failures:\n"; - for (const auto & subdir : db_subdirs_with_failures) { - std::cout << "\tDatabase file " << subdir.first << " had failures in:\n"; - for (const auto & dest_file : subdir.second) { - std::cout << "\t\t" << dest_file << "\n"; - } - std::cout << "\n"; - } - } else { - std::cout << "This directory contained no database files " - << "with report verification failures.\n"; - } - std::cout << std::endl; -} - -bp::object SimulationDatabase__getattr__(simdb::ObjectManager * sim_db, - const std::string & attr) -{ - using namespace sparta::statistics; - - bp::object o = WrapperCache().get_wrapper(sim_db); - - if (attr != "__members__" && hasattr(o, "__members__")) { - bp::object pymembers = o.attr("__members__"); - bp::object pyattr = bp::str(attr); - if (pymembers.contains(pyattr)) { - // Search through the available timeseries records for the one whose - // source report descriptor's dest_file matches this attribute, and if - // found, cache it in this object's dictionary - std::vector> timeseries_obj_refs; - sim_db->findObjects("Timeseries", {}, timeseries_obj_refs); - - for (auto & timeseries_obj_ref : timeseries_obj_refs) { - sparta_assert(timeseries_obj_ref != nullptr, - "Unexpected null timeseries returned from the database"); - auto ts = std::make_shared(std::move(timeseries_obj_ref)); - auto dest_file = ts->getHeader().getSourceReportDescDestFile(); - - auto slash = dest_file.find_last_of("/"); - if (slash < dest_file.size() - 1) { - dest_file = dest_file.substr(slash + 1); - } - - boost::replace_all(dest_file, ".", "_"); - if (attr == dest_file) { - //This is the timeseries report we're looking for - loaded_db_timeseries[dest_file] = ts; - - auto wrapped_timeseries = WrapperCache().wrap(ts.get()); - bp::object d = o.attr("__dict__"); - d[attr] = wrapped_timeseries; - return wrapped_timeseries; - } - } - } - } - - // Failed to get this attribute - PyErr_SetString(PyExc_AttributeError, (boost::format( - "There is no timeseries named '%s'") % attr).str().c_str()); - throw bp::error_already_set(); -} - -void ASYNC_SIM_ENGINE_SYNCHRONIZE() -{ - bp::object main = bp::import("__main__"); - bp::object global_ns = main.attr("__dict__"); - - if (global_ns.contains(bp::str("__db_queue"))) { - simdb::AsyncTaskEval * db_queue = - bp::extract(global_ns["__db_queue"]); - - sparta_assert(db_queue != nullptr); - db_queue->emitPreFlushNotification(); - db_queue->flushQueue(); - } -} - -bp::object LOCAL_ReportTimeseries__getPythonArrayFromSIValues( - const std::vector> & si_values) -{ - (void)si_values; - // if (si_values.empty()) { - // return makeEmptyArray(); - // } - - // //Helper that converts a std::vector into a boost::python::list - // auto cpp_vec_to_py_array = [](const std::vector & data) - // { - // Py_intptr_t shape[1] = { static_cast(data.size()) }; - // np::ndarray arr = np::zeros(1, shape, np::dtype::get_builtin()); - // std::copy(data.begin(), data.end(), reinterpret_cast(arr.get_data())); - // return arr; - // }; - - bp::list py_arr; - // for (const auto & blob : si_values) { - // py_arr.append(cpp_vec_to_py_array(blob)); - // } - return std::move(py_arr); -} - -bp::object ReportTimeseries__getValuesInTimeRange(sparta::db::ReportTimeseries * ts, - const uint64_t start_picoseconds, - const uint64_t end_picoseconds) -{ - ASYNC_SIM_ENGINE_SYNCHRONIZE(); - - std::vector> si_values; - ts->getStatisticInstValuesBetweenSimulatedPicoseconds( - start_picoseconds, end_picoseconds, si_values); - - return LOCAL_ReportTimeseries__getPythonArrayFromSIValues(si_values); -} - -bp::object ReportTimeseries__getValuesInClockRange(sparta::db::ReportTimeseries * ts, - const uint64_t start_cycle, - const uint64_t end_cycle) -{ - ASYNC_SIM_ENGINE_SYNCHRONIZE(); - - std::vector> si_values; - ts->getStatisticInstValuesBetweenRootClockCycles( - start_cycle, end_cycle, si_values); - - return LOCAL_ReportTimeseries__getPythonArrayFromSIValues(si_values); -} - -bp::object ReportTimeseries__getAllValues(sparta::db::ReportTimeseries * ts) -{ - ASYNC_SIM_ENGINE_SYNCHRONIZE(); - - std::vector> si_values; - static const auto start = std::numeric_limits::min(); - static const auto end = std::numeric_limits::max(); - ts->getStatisticInstValuesBetweenSimulatedPicoseconds(start, end, si_values); - - return LOCAL_ReportTimeseries__getPythonArrayFromSIValues(si_values); -} - -bp::object ReportTimeseries__toCSV(sparta::db::ReportTimeseries * ts, - const std::string & csv_filename) -{ - ASYNC_SIM_ENGINE_SYNCHRONIZE(); - - sparta::db::format::toCSV(ts, csv_filename); - - return bp::object(bp::borrowed(Py_None)); -} - /*! * \brief Define the "sparta" module and wrappers for all necessary components * @@ -2530,50 +1902,6 @@ BOOST_PYTHON_MODULE(sparta) .def("streamTo", raw_function(&StreamNode__streamTo, 2)) ; - class_("SimulationDatabase", no_init) - .def("__getattr__", &SimulationDatabase__getattr__) - .def("toAutoSummary", &SimulationDatabase__createReport) - .def("toJson", &SimulationDatabase__createReport) - .def("toJsonReduced", &SimulationDatabase__createReport) - .def("toJsonDetail", &SimulationDatabase__createReport) - .def("toJsJson", &SimulationDatabase__createReport) - .def("toHtml", &SimulationDatabase__createReport) - .def("toText", &SimulationDatabase__createReport) - .def("toDictionary", &SimulationDatabase__createReport) - .def("toGnuPlot", &SimulationDatabase__createReport) - .def("toStatsMapping", &SimulationDatabase__createReport) - ; - - class_("DatabaseQueue", no_init) - ; - - def("connectToDatabase", &ReportTimeseries__connectToDatabase, - return_value_policy>()) - ; - - def("__printFailedVerificationSummaries", &SimulationDatabase__printAllVerificationSummaries) - ; - - def("__getFailedVerificationFiles", &SimulationDatabase__getVerificationFailuresInDir) - ; - - class_("ReportTimeseries", no_init) - .def("getHeader", &sparta::db::ReportTimeseries::getHeader, - return_internal_reference<>()) - .def("getValuesInTimeRange", &ReportTimeseries__getValuesInTimeRange) - .def("getValuesInClockRange", &ReportTimeseries__getValuesInClockRange) - .def("getAllValues", &ReportTimeseries__getAllValues) - .def("toCSV", &ReportTimeseries__toCSV) - ; - - class_("ReportHeader", no_init) - .def("getReportName", &sparta::db::ReportHeader::getReportName) - .def("getReportStartTime", &sparta::db::ReportHeader::getReportStartTime) - .def("getReportEndTime", &sparta::db::ReportHeader::getReportEndTime) - .def("setStringMetadata", &sparta::db::ReportHeader::setStringMetadata) - .def("getStringMetadata", &sparta::db::ReportHeader::getStringMetadata) - ; - class_("Simulation", no_init) .add_property("root", // "top" object, typically make_function(Simulation_getRoot, diff --git a/sparta/python/sparta_support/module_sparta.hpp b/sparta/python/sparta_support/module_sparta.hpp index 92eb34b3e6..a82d94a090 100644 --- a/sparta/python/sparta_support/module_sparta.hpp +++ b/sparta/python/sparta_support/module_sparta.hpp @@ -27,10 +27,6 @@ #include "python/sparta_support/facade/ReportTriggers.hpp" #include "sparta/statistics/dispatch/archives/StatisticsArchives.hpp" #include "sparta/statistics/dispatch/streams/StatisticsStreams.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/ReportHeader.hpp" #include "sparta/dynamic_pipeline/GenericUnit.hpp" #include "sparta/dynamic_pipeline/GenericResourceFactory.hpp" #include "sparta/ports/Port.hpp" @@ -761,60 +757,6 @@ struct casting_wrapper } }; -/*! - * \brief Wrapper for the Simulation Database object - */ -template <> -struct casting_wrapper -{ - bp::object new_wrapper(const simdb::ObjectManager * sim_db) { - return bp::object(bp::ptr(sim_db)); - } - - void prepopulate(const simdb::ObjectManager * sim_db, bp::object & obj) const { - bp::list members; - if (!sim_db->getQualifiedTableName("Timeseries", "Stats").empty()) { - std::vector> timeseries_obj_refs; - sim_db->findObjects("Timeseries", {}, timeseries_obj_refs); - - for (auto & timeseries_obj_ref : timeseries_obj_refs) { - sparta_assert(timeseries_obj_ref != nullptr, - "Unexpected null timeseries returned from the database"); - sparta::db::ReportTimeseries ts(std::move(timeseries_obj_ref)); - std::string dest_file = ts.getHeader().getSourceReportDescDestFile(); - if (dest_file.empty()) { - throw sparta::SpartaException("Encountered a timeseries record in the ") - << "database that did not have its DestFile column value set. " - << "See database file '" << sim_db->getDatabaseFile() << "' to " - << "investigate (table=\"Timeseries\", rowid=" - << timeseries_obj_ref->getId() << ")."; - } - - //Python will not allow file separators in member names. - //In the case of CSV files written to a subdirectory like: - // - // foo/ - // bar/out.csv - // - //The dest_file could be something like: - // "example/CoreModel/AccuracyCheckedDBs/foo/bar/out.csv" - // - //Let's just take the file name, "out.csv", and use that - //to prepopulate the python object members. - auto slash = dest_file.find_last_of("/"); - if (slash < dest_file.size() - 1) { - dest_file = dest_file.substr(slash + 1); - } - - //Python will not allow dots in member names - boost::replace_all(dest_file, ".", "_"); - members.append(bp::str(dest_file)); - } - } - bp::setattr(obj, "__members__", members); - } -}; - void TreeNode___setattr__(sparta::TreeNode* n, const std::string& attr, bp::object val); class ResourceTreeNodeWrapper{ diff --git a/sparta/simdb b/sparta/simdb new file mode 160000 index 0000000000..cb3c59e0ac --- /dev/null +++ b/sparta/simdb @@ -0,0 +1 @@ +Subproject commit cb3c59e0ac26bb8346f67ccbb215897e650d4e94 diff --git a/sparta/simdb/CMakeLists.txt b/sparta/simdb/CMakeLists.txt deleted file mode 100644 index c599b5055f..0000000000 --- a/sparta/simdb/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -cmake_minimum_required(VERSION 3.19) -project(simdb CXX) - -set(SIMDB_BASE ${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${SIMDB_BASE}/include) -include(${SIMDB_BASE}/cmake/simdb-config.cmake) - -list(APPEND SimDB_CPP - src/HDF5Connection.cpp - src/ObjectManager.cpp - src/ObjectRef.cpp - src/SQLiteConnection.cpp - src/TableRef.cpp - src/simdb.cpp) - -add_library(simdb ${SimDB_CPP}) - -add_subdirectory(test EXCLUDE_FROM_ALL) - -install(TARGETS simdb - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - ) -install(DIRECTORY include/ DESTINATION include) diff --git a/sparta/simdb/cmake/simdb-config.cmake b/sparta/simdb/cmake/simdb-config.cmake deleted file mode 100644 index 6717f3b90c..0000000000 --- a/sparta/simdb/cmake/simdb-config.cmake +++ /dev/null @@ -1,13 +0,0 @@ -############################################################### -# RequiredLibraries # -############################################################### - -# Find SQLite3 -find_package(SQLite3 3.19 REQUIRED) -message("-- Using SQLite3 ${SQLite3_VERSION}") - -# Find HDF5. Need to enable C language for HDF5 testing -enable_language(C) -find_package(HDF5 1.10 REQUIRED) - -set(SimDB_LIBS simdb ${HDF5_LIBRARIES} sqlite3 z pthread) diff --git a/sparta/simdb/include/simdb/Constraints.hpp b/sparta/simdb/include/simdb/Constraints.hpp deleted file mode 100644 index 5a8f11b38f..0000000000 --- a/sparta/simdb/include/simdb/Constraints.hpp +++ /dev/null @@ -1,46 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Errors.hpp" - -#include -#include - -namespace simdb { - -//! Constraints enumeration used when building -//! SELECT, UPDATE, and DELETE statements. -enum class constraints : int8_t { - equal, // = - not_equal, // != - greater, // > - less, // < - greater_equal, // >= - less_equal, // <= - in_set, // IN - not_in_set, // NOT IN - INVALID -}; - -//! Stream operator for constraints enumeration. -inline std::ostream & operator<<(std::ostream & os, - const constraints constraint) -{ - switch (constraint) { - case constraints::equal: os << " = " ; break; - case constraints::not_equal: os << " != " ; break; - case constraints::greater: os << " > " ; break; - case constraints::less: os << " < " ; break; - case constraints::greater_equal: os << " >= " ; break; - case constraints::less_equal: os << " <= " ; break; - case constraints::in_set: os << " IN " ; break; - case constraints::not_in_set: os << " NOT IN " ; break; - case constraints::INVALID: - throw DBException("Cannot stringify constraints::INVALID"); - } - return os; -} - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/DbConnProxy.hpp b/sparta/simdb/include/simdb/DbConnProxy.hpp deleted file mode 100644 index 978f330b85..0000000000 --- a/sparta/simdb/include/simdb/DbConnProxy.hpp +++ /dev/null @@ -1,242 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/ColumnValue.hpp" -#include "simdb/ObjectFactory.hpp" -#include "simdb_fwd.hpp" - -#include -#include - -namespace simdb { - -/*! - * \brief Interface class for database connections. Subclasses - * are responsible for turning a Schema object into a fully - * realized database with an open connection (SQL, HDF5, etc.) - * and for issuing commands (INSERT, SELECT, ...) against that - * database. - */ -class DbConnProxy -{ -public: - virtual ~DbConnProxy() = default; - - //! Return a file extension to be used when creating database - //! files using auto-generated file names. - virtual const char * getDatabaseFileExtension() const = 0; - - //! Inspect and validate this schema before it gets instantiated - //! by an ObjectManager. - virtual void validateSchema(const Schema & schema) const = 0; - - //! Turn a Schema object into an actual database connection. - virtual void realizeSchema( - const Schema & schema, - const ObjectManager & obj_mgr) = 0; - - //! By default, ObjectManager will assume that the Table's - //! that were in the Schema object given to it at the start - //! of simulation are all in the physical database. Optionally - //! override this method to change how table names are figured - //! out. - virtual void getTableNames(std::unordered_set & table_names) { - (void) table_names; - } - - //! Try to open a connection to an existing database file. - virtual bool connectToExistingDatabase(const std::string & db_file) = 0; - - //! Get the full database filename being used. This includes - //! the database path, stem, and extension. Returns empty if - //! no connection is open. - virtual std::string getDatabaseFullFilename() const = 0; - - //! Is this connection still alive and well? - virtual bool isValid() const = 0; - - //! Tell ObjectManager whether or not your database should - //! have larger writes/reads enclosed inside calls to the - //! beginAtomicTransaction() / commitAtomicTransaction() - //! methods. - virtual bool supportsAtomicTransactions() const = 0; - - //! If supportsAtomicTransactions() returns true, calls to - //! this method will occur at the start of atomic writes/ - //! reads. - virtual void beginAtomicTransaction() const {} - - //! If supportsAtomicTransactions() returns true, calls to - //! this method will occur at the end of atomic writes/ - //! reads. - virtual void commitAtomicTransaction() const {} - - //! When a call into the TableRef class is made to delete one - //! or more records that match a certain set of constraints, - //! this method will end up getting invoked. In SQL, this - //! is equivalent to something like this: - //! - //! DELETE FROM Accounts WHERE PendingDelete=1 - //! - virtual void performDeletion( - const std::string & table_name, - const ColumnValues & where_clauses) const = 0; - - //! When a call into the TableRef class is made to update the - //! values of one or more records that match a certain set of - //! constraints, this method will end up getting invoked. In - //! SQL, this is equivalent to something like this: - //! - //! UPDATE Accounts SET PendingDelete=1 - //! WHERE Balance=0 AND LastUseDays>365 - //! - //! Returns the total number of updated records. - virtual size_t performUpdate( - const std::string & table_name, - const ColumnValues & col_values, - const ColumnValues & where_clauses) const = 0; - - //! \brief Some database implementations may have tables - //! whose columns are all held in contiguous memory / on disk - //! and those implementations may have a more performant - //! way to read the requested data. This is also for sub- - //! classes which do not support ObjectQuery / indexed - //! queries. - //! - //! \param table_name Name of the table containing data - //! we want to read - //! - //! \param prop_name Name of the specific property (column / - //! field) in that table being read - //! - //! \param db_id Unique database ID for the requested - //! record. Equivalent to rowid in SQL. - //! - //! \param dest_ptr Preallocated memory the raw bytes from - //! the database should be written into - //! - //! \param num_bytes Number of bytes of preallocated - //! memory the dest_ptr is pointing to - //! - //! \return Number of bytes read - virtual size_t readRawBytes( - const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - void * dest_ptr, - const size_t num_bytes) const - { - (void) table_name; - (void) prop_name; - (void) db_id; - (void) dest_ptr; - (void) num_bytes; - throw DBException("Not implemented"); - } - - //! Provide an object/record factory for the given table. - virtual AnySizeObjectFactory getObjectFactoryForTable( - const std::string & table_name) const = 0; - - //! Tables which only contain fixed-sized columns may - //! be able to implement a more performant object factory - //! than the returned callback from getObjectFactoryForTable() - //! - //! The ObjectManager knows which tables only contain - //! fixed-sized columns. For these tables, it will call - //! this getFixedSizeObjectFactoryForTable() method *once*, - //! and if this method was not overridden, ObjectManager - //! will fall back on calling getObjectFactoryForTable() - //! for all further calls to its getTable() method for - //! the requested table. - virtual FixedSizeObjectFactory getFixedSizeObjectFactoryForTable( - const std::string & table_name) const - { - (void) table_name; - return FixedSizeObjectFactory(); - } - -private: - //! This proxy is intended to be used directly with - //! the core SimDB classes. - friend class ObjectManager; - friend class ObjectRef; - friend class ObjectQuery; - - //! First-time database file open. - virtual std::string openDbFile_( - const std::string & db_dir, - const std::string & db_file, - const bool open_file) = 0; - - //! Create a prepared statement for the provided command. - //! The specific pointer type of the output void** is tied - //! to the database tech being used. This is intended to - //! be implementation detail, so this method is accessible - //! to friends only. - virtual void prepareStatement_( - const std::string & command, - void ** statement) const = 0; - - //! Leveraging the ObjectQuery utility for fast lookups - //! may not be an option for some SimDB implementations. - //! This utility is only available for SimDB's SQLite - //! implementation at the moment. - //! - //! Developer note: This method goes hand in hand with - //! the prepareStatement_() method above. Both of these - //! methods will ultimately be combined into a more - //! implementation-agnostic pure virtual public method - //! in the DbConnProxy class. Until at least one more - //! non-SQL implementation finds a performant way to - //! leverage single-constraint and multi-constraint - //! indexing like SQLite provides out of the box, it - //! is not very obvious what that public API should - //! look like. The ObjectQuery internals are hard - //! coded to work with SQLite only today. - virtual bool supportsObjectQuery_() const { - return false; - } - - //! For DbConnProxy subclasses that do not support - //! ObjectQuery, the ObjectManager::findObject() - //! method calls will reroute to this method. - //! The virtual hasObjectImpl_() method must be - //! overridden, even if that subclass override - //! has to return FALSE. - //! - //! The reason why hasObject_() and hasObjectImpl_() - //! are split up into two methods like this is that - //! ObjectManager needs to be sure that the subclass - //! actually attempted to find the requested record. - //! If hasObject_() was a virtual method that returned - //! false by default, ObjectManager would not know - //! if the attempt was even made (since FALSE is a - //! valid answer to the question, "do you have a - //! record with this ID in this table?") - bool hasObject_( - const std::string & table_name, - const DatabaseID db_id) const - { - assert(!supportsObjectQuery_()); - return hasObjectImpl_(table_name, db_id); - } - - //! For DbConnProxy subclasses that do not support - //! ObjectQuery, this method must be overridden to - //! allow ObjectManager::findObject() to find any - //! database object/record by its table name and - //! its database ID. - virtual bool hasObjectImpl_( - const std::string & table_name, - const DatabaseID db_id) const - { - (void) table_name; - (void) db_id; - throw DBException("Not implemented"); - } -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/Errors.hpp b/sparta/simdb/include/simdb/Errors.hpp deleted file mode 100644 index 26a4708e8f..0000000000 --- a/sparta/simdb/include/simdb/Errors.hpp +++ /dev/null @@ -1,107 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include - -namespace simdb { - -//! Used to construct and throw a standard C++ exception -class DBException : public std::exception -{ -public: - DBException() = default; - - //! Construct a DBException object - DBException(const std::string & reason) { - reason_ << reason; - } - - //! Copy construct a DBException object - DBException(const DBException & rhs) { - reason_ << rhs.reason_.str(); - } - - /// Destroy! - virtual ~DBException() noexcept override {} - - /** - * \brief Overload from std::exception - * \return Const char * of the exception reason - */ - virtual const char * what() const noexcept override { - reason_str_ = reason_.str(); - return reason_str_.c_str(); - } - - /** - * \brief Append additional information to the message. - */ - template - DBException & operator<<(const T & msg) { - reason_ << msg; - return *this; - } - -private: - // The reason/explanation for the exception - std::stringstream reason_; - - // Need to keep a local copy of the string formed in the - // string stream for the 'what' call - mutable std::string reason_str_; -}; - -//! General-purpose database exception used for interrupts -class DatabaseInterrupt : public DBException -{ -public: - const char * what() const noexcept override final { - exception_message_ << " [simdb] Database operation was interrupted"; - const std::string details = getExceptionDetails_(); - if (!details.empty()) { - exception_message_ << " (" << details << ")"; - } - reason_str_ = exception_message_.str(); - return reason_str_.c_str(); - } - -private: - //Subclasses should tack on some more information - //about the specific exception. - virtual std::string getExceptionDetails_() const = 0; - - mutable std::ostringstream exception_message_; - mutable std::string reason_str_; -}; - -//! Exception class for database access errors. This -//! is made into a separate class without providing -//! a different implementation for any of the base -//! class methods so that ObjectManager can catch -//! these access exceptions and keep retrying the -//! transactions. This supports atomic begin/commit -//! transactions for the databases that support -//! atomic transactions. -class DBAccessException : public DBException -{ -}; - -} // namespace simdb - -#define ADD_FILE_INFORMATION(ex, file, line) \ - ex << ": in file: '" << file << "', on line: " \ - << std::dec << line; - -#define simdb_throw(message) \ - { \ - std::stringstream msg; \ - msg << message; \ - simdb::DBException ex(std::string("abort: ") + msg.str()); \ - ADD_FILE_INFORMATION(ex, __FILE__, __LINE__); \ - throw ex; \ - } - diff --git a/sparta/simdb/include/simdb/ObjectFactory.hpp b/sparta/simdb/include/simdb/ObjectFactory.hpp deleted file mode 100644 index c46e0644cc..0000000000 --- a/sparta/simdb/include/simdb/ObjectFactory.hpp +++ /dev/null @@ -1,35 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/DatabaseTypedefs.hpp" -#include "simdb/schema/ColumnValue.hpp" -#include "simdb_fwd.hpp" - -#include -#include -#include - -namespace simdb { - -//! SimDB implementations can register "object factories" -//! for any of their tables. These routines perform table -//! inserts. The return value is the created record's ID. -//! This ID should be unique for the given table the new -//! record lives in. -using AnySizeObjectFactory = std::function< - DatabaseID(DbConnProxy * db_proxy, - const std::string & table_name, - const ColumnValues & obj_values)>; - -//! The fixed-size object factory is intended to be a -//! more performant factory implementation for tables -//! that only contain fixed-size columns, i.e. PODs. -using FixedSizeObjectFactory = std::function< - DatabaseID(DbConnProxy * db_proxy, - const std::string & table_name, - const void * raw_bytes_ptr, - const size_t num_raw_bytes)>; - -} - diff --git a/sparta/simdb/include/simdb/ObjectManager.hpp b/sparta/simdb/include/simdb/ObjectManager.hpp deleted file mode 100644 index b9c37aa847..0000000000 --- a/sparta/simdb/include/simdb/ObjectManager.hpp +++ /dev/null @@ -1,635 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/Schema.hpp" -#include "simdb/ObjectFactory.hpp" -#include "simdb/utils/StringUtils.hpp" -#include "simdb_fwd.hpp" - -#include -#include -#include -#include -#include -#include - -namespace simdb { - -//! Warnings utility that will log any messages for -//! you into memory, and write them to file when the -//! object goes out of scope. -class WarningLogger -{ -public: - explicit WarningLogger(const std::string & warn_filename) : - warn_filename_(warn_filename) - {} - - ~WarningLogger() - { - const std::string warnings = msgs_.str(); - if (!warnings.empty()) { - std::ofstream fout(warn_filename_); - fout << warnings; - } - } - - template - const WarningLogger & operator<<(const MessageT & msg) const - { - msgs_ << msg; - return *this; - } - - const WarningLogger & operator<<(const char * msg) const - { - msgs_ << msg; - return *this; - } - -private: - mutable std::ostringstream msgs_; - const std::string warn_filename_; -}; - -/*! - * \brief Database object manager. Used in order to create - * databases with a user-specified schema, and connect to - * existing databases that were previously created with - * another ObjectManager. - */ -class ObjectManager -{ -public: - //! Construct an ObjectManager. This does not open any - //! database connection or create any database just yet. - //! The database path that you pass in is wherever you - //! want the database to ultimately live. - ObjectManager(const std::string & db_dir = "."); - - //! Using a Schema object which specifies the Tables, - //! Columns, and Indexes for your database, construct - //! the physical database file and open the connection. - //! - //! Returns true if successful, false otherwise. - bool createDatabaseFromSchema( - Schema & schema, - std::unique_ptr db_proxy); - - //! After calling createDatabaseFromSchema(), you may - //! add additional tables with this method. If a table - //! has the same name as an existing table in this database, - //! all of the table columns need to match exactly as - //! well, or this method will throw. If the columns - //! match however, the table will be ignored as it - //! already exists in the schema. - //! - //! Returns true if the provided schema's tables were - //! successfuly added to this ObjectManager's schema, - //! otherwise returns false. - bool appendSchema(Schema & schema); - - //! Open a database connection to an existing database - //! file. The 'db_file' that you pass in should be the - //! full database path, including the file name and - //! extension. For example, "/path/to/my/dir/statistics.db" - //! - //! This 'db_file' is typically one that was given to you - //! from a previous call to ObjectManager::getDatabaseFile() - //! - //! Returns true if successful, false otherwise. - bool connectToExistingDatabase(const std::string & db_file); - - //! Get the full database file name, including its path and - //! file extension. If the database has not been opened or - //! created yet, this will just return the database path. - const std::string & getDatabaseFile() const; - - //! Get the internal database proxy. Will return nullptr - //! if no database connection has been made yet. - const DbConnProxy * getDbConn() const; - - //! Get this database connection's task queue. This - //! object can be used to schedule database work to - //! be executed on a background thread. This never - //! returns null. - AsyncTaskEval * getTaskQueue() const { - return task_queue_.get(); - } - - //! Every ObjectManager has its own AsyncTaskEval - //! which can be used to schedule database work - //! on a background thread. If you have multiple - //! of these ObjectManager's, you may want them - //! all to share the same background thread. In - //! that case, create an AsyncTaskController - //! object, and tell each of your ObjectManager's - //! to "addToTaskController()". All calls that - //! forward work to this ObjectManager's task queue - //! will get rerouted to the same work queue - //! inside your AsyncTaskController. - //! - //! auto controller = std::make_shared(); - //! ObjectManager mgrA("."); - //! ObjectManager mgrB("."); - //! - //! //... set up schemas, set up proxies, ... - //! - //! mgrA.addToTaskController(controller); - //! mgrB.addToTaskController(controller); - //! - //! //Schedule hypothetical WorkerTask objects - //! //on the shared controller's work thread: - //! mgrA.getTaskQueue()->addWorkerTask( - //! std::unique_ptr(new TaskA)); - //! - //! mgrB.getTaskQueue()->addWorkerTask( - //! std::unique_ptr(new TaskB)); - //! - //! //Typically, you would not immediately flush - //! //the worker thread, instead letting it consume - //! //your scheduled work in the background, but - //! //to illustrate use of one of the controller - //! //APIs, the TaskA and TaskB work requests would - //! //get completed with this call, if they aren't - //! //already underway in the background: - //! controller->flushQueue(); - //! - void addToTaskController( - AsyncTaskController * controller); - - //! Get a unique identifier for this database connection. - //! Returns 0 if there is no open connection yet, which - //! happens during a call to "createDatabaseFromSchema()" - int32_t getId() const { - return uuid_; - } - - //! Warnings are printed to stdout. Disable warnings by - //! calling this method. Warnings are enabled by default. - void disableWarningMessages() { - warnings_enabled_ = false; - } - - //! Re-enable disabled warnings. Warnings are enabled - //! by default. - void enableWarningMessages() { - warnings_enabled_ = true; - } - - //! Open database connections will be closed when the - //! destructor is called. - ~ObjectManager(); - - //! All API calls to ObjectManager, ObjectRef, and the - //! other database classes will be executed inside "safe - //! transactions" for exception safety and for better - //! performance. Failed database writes/reads will be - //! retried until successful. This will also improve - //! performance - especially for DB writes - if you - //! have several operations that you need to perform - //! on the database, for example: - //! - //! \code - //! ObjectRef new_customer(...) - //! new_customer.setPropertyString("First", "Bob") - //! new_customer.setPropertyString("Last", "Smith") - //! new_customer.setPropertyInt32("Age", 41) - //! \endcode - //! - //! That would normally be three individual transactions. - //! But if you do this instead (assuming you have an - //! ObjectManager 'obj_mgr' nearby): - //! - //! \code - //! obj_mgr.safeTransaction([&]() { - //! ObjectRef new_customer(...) - //! new_customer.setPropertyString("First", "Bob") - //! new_customer.setPropertyString("Last", "Smith") - //! new_customer.setPropertyInt32("Age", 41) - //! }); - //! \endcode - //! - //! That actually ends up being just *one* database - //! transaction. Not only is this faster (in some - //! scenarios it can be a very significant performance - //! boost) but all three of these individual setProperty() - //! calls would either be committed to the database, or - //! they wouldn't, maybe due to an exception. But the - //! "new_customer" object would not have the "First" - //! property written to the database, while the "Last" - //! and "Age" properties were left unwritten. "Half- - //! written" database objects could result in difficult - //! bugs to track down, or leave your data in an - //! inconsistent state. - typedef std::function TransactionFunc; - void safeTransaction(TransactionFunc transaction) const; - - class ObjectDatabase { - public: - //! Get a wrapper to a Table in this database. - //! Returns nullptr if the table name you provide - //! is not in this database's schema. Call the - //! 'getTableNames()' method to see the available - //! tables (originally taken from the Schema that - //! this ObjectManager used to create the database). - std::unique_ptr getTable( - const std::string & table_name) const; - - //! There are scenarios where a TableRef either cannot - //! be returned by the above 'getTable()' method if the - //! namespace schema has not yet been fully realized, - //! or the schema is realized but the table is still - //! not writable at this time for another reason. - //! - //! Tables can conditionally allow or disallow access - //! in response to triggered events (such as specific - //! conditions in a simulation). - //! - //! Using this method will never return null, and the - //! returned proxy object has an 'isWritable()' method - //! to check its accessibility state before proceeding. - //! The proxy also has a 'getTable()' method of its own - //! which returns the underlying TableRef pointer if - //! the table / namespace is accessible at that time, - //! or null if the table is still inaccessible. - TableProxy * getConditionalTable( - const std::string & table_name) const; - - //! Get a list of Table names in this database - const std::unordered_set & getTableNames() const; - - //! Find any database record by its table name and - //! its unique database ID in that table. - std::unique_ptr findObject( - const std::string & table_name, - const DatabaseID db_id) const; - - //! Find a group of database records in a given table - //! by their unique database ID's in that table. This - //! will not verify that the 'db_ids' you pass in are - //! in fact unique. The returned 'obj_refs' will always - //! be equal in length to the 'db_ids' that you pass in. - //! Database ID's that did not have a record in this table - //! will have a nullptr in that ObjectRef's spot in 'obj_refs'. - //! - //! \code - //! std::vector> obj_refs; - //! obj_mgr.findObjects("Customers", {44, 68, 92}, obj_refs); - //! - //! if (obj_refs[0]) { - //! std::cout << "Customer with id 44 found" << std::endl; - //! } else { - //! std::cout << "Customer with id 44 was NOT found" << std::endl; - //! } - //! //... - //! - //! assert(obj_refs.size() == 3, - //! "No matter what, ObjectManager::findObjects() should " - //! "return a vector of ObjectRef's that is the same length " - //! "as the vector of database IDs that you pass in."); - //! \endcode - //! - //! If you want to get *all* records in the given table, pass in {} - //! for the 'db_ids' input. - void findObjects( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const; - - //! Get the underlying data file location. This is the full path - //! to the file, the file stem, and the file extension. - const std::string & getDatabaseFile() const { - return sim_db_->getDatabaseFile(); - } - - //! Perform constrained queries against the underlying database - //! to find records that match a specific set of criteria. See - //! simdb/include/simdb/utils/ObjectQuery.h for more details - //! on how to use this class. - //! - //! \warning ObjectQuery is currently only supported for SQLite - //! database connections, and will throw if you attempt to use - //! it with another database format such as HDF5. - std::unique_ptr createObjectQueryForTable( - const std::string & table_name) const; - - //! Get the task queue associated with the underlying ObjectManager. - //! The WorkerTask's you give to the task queue will be invoked - //! on a background thread, inside of ObjectManager::safeTransaction() - //! calls. - AsyncTaskEval * getTaskQueue() const; - - //! Get the underlying database. This is not necessarily unique - //! to this one ObjectDatabase; it may be shared among many DB - //! namespaces. - //! - //! \warning This method may be removed in a future release. - //! It is currently here primarily for backwards compatibility. - ObjectManager * getObjectManager() const { - return sim_db_; - } - - //! This method is called when a SimDB namespace has - //! just become available for reads and writes via - //! the TableProxy class objects. - void grantAccess(); - - //! This method is called when a SimDB namespace has - //! just become unavailable for reads and writes via - //! the TableProxy class objects. - void revokeAccess(); - - private: - ObjectDatabase(ObjectManager * sim_db, - const std::string & db_namespace, - DatabaseNamespace * db_namespace_obj = nullptr) : - sim_db_(sim_db), - db_namespace_(db_namespace), - db_namespace_obj_(db_namespace_obj) - { - if (!sim_db_) { - throw DBException("Null ObjectManager given to a ObjectDatabase"); - } - } - - ObjectManager * sim_db_ = nullptr; - std::string db_namespace_; - DatabaseNamespace * db_namespace_obj_ = nullptr; - mutable std::unordered_set table_names_; - mutable std::map> table_proxies_; - bool access_granted_ = true; - friend class DatabaseRoot; - friend class DatabaseNamespace; - friend class ObjectManager; - }; - - /*! - * \brief ObjectManager's can have tables inside the database file - * separated into different namespaces. Here is an example of a - * database schema with four tables in two namespaces, using the - * equivalent C++ pseudo-code to illustrate: - * - * namespace gold { - * struct CustomerInfo { - * string First; - * string Last; - * long AccountNumber; - * }; - * struct RewardsMembers { - * long AccountNumber; - * int YearsActive; - * short RewardsBalance; - * }; - * } - * - * namespace platinum { - * struct CustomerInfo { - * string First; - * string Middle; - * string Last; - * string Occupation; - * long RewardsBalance; - * }; - * } - * - * Say you want to access the RewardsMembers table, but you - * are not sure which namespace it belongs to. You can call - * this method in several ways to get the full table name in - * this database, if it exists: - * - * 1) You don't know the exact namespace, but you do know that - * the RewardsMembers table only exists in one namespace. - * - * auto qualified_table_name = - * obj_mgr.getQualifiedTableName("RewardsMembers"); - * - * In this case, qualified_table_name would be something - * like "gold$RewardsMembers" - a delimiter separates the - * namespace from the table name, but you should not rely - * on this delimiter being any specific character/string. - * The delimiter in use is implementation detail. - * - * 2) You are sure the namespace is "gold", but you don't - * want to hard code the '$' character in your code. - * You want to keep your code robust. - * - * auto qualified_table_name = - * obj_mgr.getQualifiedTableName("RewardsMembers", "gold"); - * - * For this example, and assuming that the delimiter is - * '$', this would return "gold$RewardsMembers" - * - * 3) The table you *really* want is RewardsMembers in the - * 'platinum' namespace. But your schemas are dynamically - * generated and this table sometimes does not exist in - * the 'platinum' namespace. - * - * auto qualified_table_name = - * obj_mgr.getQualifiedTableName("RewardsMembers", "platinum"); - * - * For the above schema, this would return an empty string. - * It would NOT return "gold$RewardsMembers", as that is a - * completely different table, with a different column - * arrangement. - * - * Now let's say that you were looking for the "CustomerInfo" - * table. This table appears in two namespaces. Here are some - * return values for this example: - * - * 1) You are looking for CustomerInfo in the "gold" namespace. - * - * auto qualified_table_name = - * obj_mgr.getQualifiedTableName("CustomerInfo", "gold"); - * - * Assuming a '$' delimiter, this would return "gold$CustomerInfo" - * - * 2) You are looking for CustomerInfo, and you think it only - * exists in one of the database namespaces, but you don't - * know the name of that namespace. - * - * auto qualified_table_name = - * obj_mgr.getQualifiedTableName("CustomerInfo"); - * - * This would return an empty string. There were matches - * found in two namespaces, but since you did not provide - * any namespace hint, ObjectManager cannot know which - * qualified table name to return. - */ - std::string getQualifiedTableName( - const std::string & table_name, - const utils::lowercase_string & namespace_hint = "") const; - - //! Generate table summary snapshot. Columns that support - //! summaries (numeric scalars, and possibly other data types) - //! will have their data values' min/max/average captured and - //! put into separate summary tables, along with any other - //! custom summary/aggregation calculations that were given - //! to the schema's TableSummaries object. - void captureTableSummaries(); - - //! Get the schema this ObjectManager is using. - Schema & getSchema() { - return schema_; - } - - //! Get the schema this ObjectManager is using. - const Schema & getSchema() const { - return schema_; - } - - //! ------- DEPRECATED. Use the ObjectDatabase class above. ------- - std::unique_ptr getTable( - const std::string & table_name) const; - - const std::unordered_set & getTableNames() const; - - std::unique_ptr findObject( - const std::string & table_name, - const DatabaseID db_id) const; - - void findObjects( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const; - //! ---------------------- (end DEPRECATED) ------------------------- - -private: - //Complements ObjectDatabase::getTable() - the table name - //passed in will be fully qualified ("$") - std::unique_ptr getTable_( - const std::string & table_name) const; - - //Complements ObjectDatabase::getTableNames() - the table name - //passed in will be fully qualified ("$") - const std::unordered_set & getTableNames_() const; - - //Complements ObjectDatabase::findObject() - the table name - //passed in will be fully qualified ("$") - std::unique_ptr findObject_( - const std::string & table_name, - const DatabaseID db_id) const; - - //Complements ObjectDatabase::findObjects() - the table name - //passed in will be fully qualified ("$") - void findObjects_( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const; - - //Open the given database file. If the connection is - //successful, this file will be the ObjectManager's - //"db_full_filename_" value. - bool openDbFile_(const std::string & db_file, - const bool create_file); - - //Try to just open an empty database file. This is - //similar to fopen(). - void openDatabaseWithoutSchema_(); - - //This class does not currently allow one ObjectManager - //to be simultaneously connected to multiple databases. - void assertNoDatabaseConnectionOpen_() const; - - //Ask the database what its table names are, and cache - //them in memory for later. - void getDatabaseTableNames_() const; - mutable std::unordered_set table_names_; - mutable std::unordered_set default_table_names_; - - //Helper method to get the full name of the provided - //table in the "Stats" namespace. This is for backwards - //compatibility only and may be removed in a future - //release. - std::string getStatsTableName_( - const std::string & table_name) const; - - //Cached qualified table names. Used to boost performance - //of ObjectManager::getQualifiedTableName() - mutable std::unordered_map< - std::string, //Unqualified table name - std::unordered_map< - std::string, //Namespace - std::string //Qualified table name - >> cached_qualified_table_names_; - - //UUID for this database connection - void getAndStoreDatabaseID_(); - int32_t uuid_ = 0; - - //Warnings are enabled by default. There are APIs in this - //class for disabling/re-enabling warnings. - bool warnings_enabled_ = true; - - //Physical database proxy. Commands (INSERT, UPDATE, etc.) - //are executed against this proxy, not against the lower- - //level database APIs directly. - std::shared_ptr db_proxy_; - - //Registered object factories by table name. - mutable std::unordered_map any_size_record_factories_; - mutable std::unordered_map fixed_size_record_factories_; - - //Keep a list of fixed-size tables. For these tables, all of - //their columns are POD's, and the DbConnProxy subclasses may - //have an optimized object factory implementation for fixed- - //size record creation. - mutable std::unordered_set fixed_size_tables_; - - //Copy of the schema that was given to the ObjectManager's - //createDatabaseFromSchema() method. - Schema schema_; - - //Location where this database lives, e.g. the tempdir - const std::string db_dir_; - - //Task queue associated with this database connection. - //It is instantiated from our constructor, but won't - //have any effect unless its addWorkerTask() method - //is called. That method starts a background thread - //to begin consuming work packets. - std::unique_ptr task_queue_; - - //When this task controller is in use, any WorkerTask - //that gets added to our AsyncTaskEval work queue will - //be rerouted into this controller's own internal work - //queue. This is used to enable many individual database - //connections to write to the same database file using - //just one worker thread. - AsyncTaskController * task_controller_ = nullptr; - - //Full database file name, including the database path - //and file extension - std::string db_full_filename_; - - //Flag used in RAII safeTransaction() calls. This is - //needed to we know whether to tell SQL to "BEGIN - //TRANSACTION" or not (i.e. if we're already in the - //middle of another safeTransaction). - // - //This allows users to freely do something like this: - // - // obj_mgr_.safeTransaction([&]() { - // writeReportHeader_(report); - // }); - // - //Even if their writeReportHeader_() code does the - //same thing: - // - // void CSV::writeReportHeader_(sparta::Report * r) { - // obj_mgr_.safeTransaction([&]() { - // writeReportName_(r); - // writeSimulationMetadata_(sim_); - // }); - // } - mutable bool is_in_transaction_ = false; - - //Logging utility to capture any database warnings or - //other useful messages to a "database.warn" log file - WarningLogger warning_log_; - - friend class TableProxy; -}; - -} // namespace simdb diff --git a/sparta/simdb/include/simdb/ObjectRef.hpp b/sparta/simdb/include/simdb/ObjectRef.hpp deleted file mode 100644 index 7ca3f39943..0000000000 --- a/sparta/simdb/include/simdb/ObjectRef.hpp +++ /dev/null @@ -1,207 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb_fwd.hpp" - -#include "simdb/schema/ColumnTypedefs.hpp" -#include "simdb/ObjectManager.hpp" - -#include -#include -#include - -namespace simdb { - -/*! - * \brief Wrapper around a record (row) in a - * database table. Used for writing and reading - * record properties (column values). - */ -class ObjectRef -{ -public: - //! Construct an ObjectRef for a SimDB table record. Pass - //! in the ObjectManager it belongs to, the name of the - //! table this record belongs to, and the unique database - //! ID for this record (primary key in the given table). - //! - //! Typically, you should get all ObjectRef's - //! from an ObjectManager that you created first, - //! or from a TableRef that you got from such an - //! ObjectManager, instead of making an ObjectRef - //! yourself manually. - ObjectRef(const ObjectManager & obj_mgr, - const std::string & table_name, - const DatabaseID db_id) : - obj_mgr_(obj_mgr), - table_name_(table_name), - db_id_(db_id) - {} - - //! Returns the ObjectManager passed to our constructor - const ObjectManager & getObjectManager() const; - - //! Get this record's unique database ID. It is unique - //! to the table it lives in, not necessarily globally - //! unique across all database tables. - DatabaseID getId() const; - - //! PROPERTY SETTERS --------------------------------- - void setPropertyInt8( - const std::string & prop_name, - const int8_t prop_value); - - void setPropertyUInt8( - const std::string & prop_name, - const uint8_t prop_value); - - void setPropertyInt16( - const std::string & prop_name, - const int16_t prop_value); - - void setPropertyUInt16( - const std::string & prop_name, - const uint16_t prop_value); - - void setPropertyInt32( - const std::string & prop_name, - const int32_t prop_value); - - void setPropertyUInt32( - const std::string & prop_name, - const uint32_t prop_value); - - void setPropertyInt64( - const std::string & prop_name, - const int64_t prop_value); - - void setPropertyUInt64( - const std::string & prop_name, - const uint64_t prop_value); - - void setPropertyString( - const std::string & prop_name, - const std::string & prop_value); - - void setPropertyChar( - const std::string & prop_name, - const char prop_value); - - void setPropertyFloat( - const std::string & prop_name, - const float prop_value); - - void setPropertyDouble( - const std::string & prop_name, - const double prop_value); - - void setPropertyBlob( - const std::string & prop_name, - const Blob & prop_value); - - //! PROPERTY GETTERS --------------------------------- - int8_t getPropertyInt8( - const std::string & prop_name) const; - - uint8_t getPropertyUInt8( - const std::string & prop_name) const; - - int16_t getPropertyInt16( - const std::string & prop_name) const; - - uint16_t getPropertyUInt16( - const std::string & prop_name) const; - - int32_t getPropertyInt32( - const std::string & prop_name) const; - - uint32_t getPropertyUInt32( - const std::string & prop_name) const; - - int64_t getPropertyInt64( - const std::string & prop_name) const; - - uint64_t getPropertyUInt64( - const std::string & prop_name) const; - - std::string getPropertyString( - const std::string & prop_name) const; - - char getPropertyChar( - const std::string & prop_name) const; - - float getPropertyFloat( - const std::string & prop_name) const; - - double getPropertyDouble( - const std::string & prop_name) const; - - //Note here that blobs are returned in a vector - //of bytes that you create and own at the call - //site: - // - // std::vector record_values; - // obj_ref->getPropertyBlob("RawDataValues", record_values); - // - //The ObjectRef will only copy the bytes into - //the vector you provide; it will not cache - //anything under the hood. Your vector will - //be resized to exactly fit the record blob, - //which could of course be 0 elements. - template - void getPropertyBlob( - const std::string & prop_name, - std::vector & prop_bytes) const; - -private: - //Called at the beginning of getPropertyBlob() - // - not inlined in the template code below - void prepGetPropertyBlob_( - const std::string & prop_name, - void ** statement, - Blob & blob_descriptor) const; - - //Called at the end of getPropertyBlob() - // - not inlined in the template code below - void finalizeGetPropertyBlob_( - void * statement) const; - - const ObjectManager & obj_mgr_; - const std::string table_name_; - const DatabaseID db_id_; -}; - -//! Inlined template code for getPropertyBlob() -//! (non-template / common code is in the .cpp file) -template -void ObjectRef::getPropertyBlob(const std::string & prop_name, - std::vector & prop_value) const -{ - obj_mgr_.safeTransaction([&]() { - //Get the blob void* and number of bytes - Blob blob_descriptor; - void * statement = nullptr; - prepGetPropertyBlob_(prop_name, &statement, blob_descriptor); - - const void * blob_ptr = blob_descriptor.data_ptr; - const size_t blob_num_bytes = blob_descriptor.num_bytes; - if (blob_num_bytes > 0) { - prop_value.resize(blob_num_bytes / sizeof(BlobDataT)); - void * dest = &prop_value[0]; - memcpy(dest, blob_ptr, blob_num_bytes); - } else { - prop_value.clear(); - //Just to be extra safe, let's shrink the output - //property value vector. We may choose not to do - //this for performance reasons later on, but for - //now we can play it safe. - prop_value.shrink_to_fit(); - } - - //Destroy the prepared statement - we are done with it. - finalizeGetPropertyBlob_(statement); - }); -} - -} // namespace simdb diff --git a/sparta/simdb/include/simdb/TableProxy.hpp b/sparta/simdb/include/simdb/TableProxy.hpp deleted file mode 100644 index 9c47a4d820..0000000000 --- a/sparta/simdb/include/simdb/TableProxy.hpp +++ /dev/null @@ -1,79 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/DatabaseRoot.hpp" -#include "simdb/TableRef.hpp" -#include "simdb_fwd.hpp" - -#include -#include - -namespace simdb { - -/*! - * \class TableProxy - * - * \brief When SimDB is used with table access triggers, - * there may be times when a table in the database is not - * in a writable state. Instead of returning null TableRef - * objects to signal that the table is not available, we - * instead return TableProxy objects which are never null. - * When the table connection is available, it can be accessed - * through this proxy. - */ -class TableProxy -{ -public: - //! Construct with the table name and ObjectManager - //! this table belongs to. - TableProxy(const std::string & table_name, - const ObjectManager & obj_mgr, - DatabaseNamespace * db_namespace) : - table_name_(table_name), - obj_mgr_(obj_mgr), - db_namespace_(db_namespace) - {} - - //! Check if this proxy is in a writable state. - bool isWritable() const { - refreshAccess_(); - return table_ref_ != nullptr; - } - - //! Get the underlying TableRef from this proxy. This - //! will return null without warnings or exceptions if - //! the table is requested when it is unavailable. - TableRef * getTable() const { - refreshAccess_(); - return table_ref_.get(); - } - - //! Method called when the table becomes accessible. - void grantAccess() { - if (!table_ref_) { - table_ref_ = obj_mgr_.getTable_(table_name_); - } - } - - //! Method called when the table becomes inaccessible. - void revokeAccess() { - table_ref_.reset(); - } - -private: - void refreshAccess_() const { - if (db_namespace_) { - db_namespace_->getDatabase(); - } - } - - std::string table_name_; - const ObjectManager & obj_mgr_; - mutable DatabaseNamespace * db_namespace_ = nullptr; - std::unique_ptr table_ref_; - friend class DatabaseRoot; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/TableRef.hpp b/sparta/simdb/include/simdb/TableRef.hpp deleted file mode 100644 index daaffb74fc..0000000000 --- a/sparta/simdb/include/simdb/TableRef.hpp +++ /dev/null @@ -1,637 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/ColumnValueContainer.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" -#include "simdb/ObjectFactory.hpp" -#include "simdb/utils/Stringifiers.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb_fwd.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace simdb { - -/*! - * \brief Wrapper around a SimDB table - */ -class TableRef -{ -public: - //! Construct a TableRef for a SimDB table with the - //! given name, and the ObjectManager it belongs to. - //! - //! Typically, you should get a TableRef object - //! from an ObjectManager that you created first, - //! instead of making a TableRef yourself manually. - TableRef(const std::string & table_name, - const ObjectManager & obj_mgr) : - table_name_(table_name), - obj_mgr_(obj_mgr) - {} - - //! Create a new record in this table. Returns a - //! wrapper around the new record. - std::unique_ptr createObject(); - - //! Intermediate class used in UPDATE statements: - //! - //! table->updateRowValues("MyInt32", 100, "MyString", "bar"). - //! forRecordsWhere("MyInt32", constraints::equal, 85); - //! - //! This would update a table with these records: - //! - //! MyInt32 MyString MyDouble - //! --------- ---------- ---------- - //! 80 hello 3.45 - //! 85 world 4.56 <-- (match) - //! 85 foo 5.67 <-- (match) - //! 90 bar 6.78 - //! - //! And the new table records would then be: - //! - //! MyInt32 MyString MyDouble - //! --------- ---------- ---------- - //! 80 hello 3.45 - //! 100 bar 4.56 <-- (new values) - //! 100 bar 5.67 <-- (new values) - //! 90 bar 6.78 - class RecordFinder - { - public: - //! WHERE clause in UPDATE statements for numeric - //! column constraints. - template - typename std::enable_if< - std::is_fundamental::value, - size_t>::type - forRecordsWhere(const char * col_name, - const constraints constraint, - const ColumnT col_val) - { - update_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - return finalize_callback_(); - } - - //! WHERE clause in UPDATE statements for string - //! column constraints. - template - typename std::enable_if< - std::is_same::type, const char*>::value, - size_t>::type - forRecordsWhere(const char * col_name, - const constraints constraint, - ColumnT col_val) - { - update_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - return finalize_callback_(); - } - - //! WHERE clause in UPDATE statements for string - //! column constraints. - template - typename std::enable_if< - std::is_same::value, - size_t>::type - forRecordsWhere(const char * col_name, - const constraints constraint, - const ColumnT & col_val) - { - update_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - return finalize_callback_(); - } - - //! WHERE clause in UPDATE statements for {num,num,...} and - //! {string,string,...} column constraints ("IN SET" / "NOT - //! IN SET"). - template - size_t forRecordsWhere(const char * col_name, - const constraints constraint, - const std::initializer_list & col_val) - { - update_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - return finalize_callback_(); - } - - //! WHERE clause in UPDATE statements for multi-column - //! selection constraints. - template - size_t forRecordsWhere(const char * col_name, - const constraints constraint, - const ColumnT & col_val, - Args &&... args) - { - update_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - return forRecordsWhere(std::forward(args)...); - } - - //! Unconstrained UPDATE statements terminate the call to - //! TableRef::updateRowValues() with this method: - //! - //! table->updateRowValues("MyString", "foobar", "MyDouble", 5.6). - //! forAllRecords(); - size_t forAllRecords() - { - return finalize_callback_(); - } - - private: - //Maintain a list of WHERE clauses for the UPDATE. - ColumnValueContainer update_where_clauses_; - - //Only to be used by TableRef. Cannot be created outside - //of an UPDATE action. - RecordFinder(const std::function & finalize_callback) : - finalize_callback_(finalize_callback) - {} - friend class TableRef; - - //Callback set by TableRef when the forRecordsWhere() - //parameter pack has been unrolled. - std::function finalize_callback_; - }; - - //! UPDATE value clause for integer and floating-point columns. - template - typename std::enable_if< - std::is_fundamental::value, - RecordFinder&>::type - updateRowValues(const char * col_name, - const ColumnT col_val) - { - col_values_.add(col_name, col_val); - return makeRecordFinder_(); - } - - //! UPDATE value clause for string columns. - template - typename std::enable_if< - std::is_same::type, const char*>::value, - RecordFinder&>::type - updateRowValues(const char * col_name, - ColumnT col_val) - { - col_values_.add(col_name, col_val); - return makeRecordFinder_(); - } - - //! UPDATE value clause for string columns. - template - typename std::enable_if< - std::is_same::value, - RecordFinder&>::type - updateRowValues(const char * col_name, - const ColumnT & col_val) - { - return updateRowValues(col_name, col_val.c_str()); - } - - //! UPDATE value clause for blob columns (as an STL container) - template - typename std::enable_if< - is_container::value and is_contiguous::value, - RecordFinder&>::type - updateRowValues(const char * col_name, - const ColumnT & col_val) - { - col_values_.add(col_name, col_val); - return makeRecordFinder_(); - } - - //! UPDATE value clause for blob columns (as a struct holding - //! the void* and number of bytes for the raw data) - template - typename std::enable_if< - std::is_same::value, - RecordFinder&>::type - updateRowValues(const char * col_name, - const ColumnT & col_val) - { - col_values_.add(col_name, col_val); - return makeRecordFinder_(); - } - - //! UPDATE value clause for overwriting multiple column - //! values in the same statement. - template - RecordFinder & updateRowValues(const char * col_name, - const ColumnT & col_val, - Args &&... args) - { - col_values_.add(col_name, col_val); - return updateRowValues(std::forward(args)...); - } - - template - typename std::enable_if< - std::is_arithmetic::value, - std::unique_ptr>::type - createObjectWithVals(const ColumnT col_val) - { - appendRawValue_(col_val); - return finalizeCreationStatement_(); - } - - template - typename std::enable_if< - std::is_arithmetic::value, - std::unique_ptr>::type - createObjectWithVals( - const ColumnT col_val, - Args &&... args) - { - appendRawValue_(col_val); - return createObjectWithVals(std::forward(args)...); - } - - template - typename std::enable_if< - std::is_trivially_constructible::value, - std::unique_ptr>::type - createObjectFromStruct(const ColumnT & structure) - { - appendRawValue_(structure); - return finalizeCreationStatement_(); - } - - //! Create a new record in this table, setting - //! one of the record's column values at the time - //! of creation. - //! - //! This method is also used as the base function in - //! the N-argument creation method below. - template - typename std::enable_if< - std::is_fundamental::value, - std::unique_ptr>::type - createObjectWithArgs(const char * col_name, - const ColumnT col_val) - { - col_values_.add(col_name, col_val); - return finalizeCreationStatement_(); - } - - //! Same as single-argument creation method above, but - //! specific to const char* column values. - template - typename std::enable_if< - std::is_same::type, const char*>::value, - std::unique_ptr>::type - createObjectWithArgs(const char * col_name, - ColumnT col_val) - { - col_values_.add(col_name, col_val); - return finalizeCreationStatement_(); - } - - //! Same as single-argument creation method above, but - //! specific to std::string column values. - template - typename std::enable_if< - std::is_same::value, - std::unique_ptr>::type - createObjectWithArgs(const char * col_name, - const ColumnT & col_val) - { - return createObjectWithArgs(col_name, col_val.c_str()); - } - - //! Same as single-argument creation mentioned above, but - //! specific to std::vector column values. - template - typename std::enable_if< - is_container::value and is_contiguous::value, - std::unique_ptr>::type - createObjectWithArgs(const char * col_name, - const ColumnT & col_val) - { - col_values_.add(col_name, col_val); - return finalizeCreationStatement_(); - } - - //! Create a new record in this table, setting one - //! or more column values at the time of creation. - template - std::unique_ptr createObjectWithArgs( - const char * col_name, - const ColumnT & col_val, - Args &&... args) - { - col_values_.add(col_name, col_val); - return createObjectWithArgs(std::forward(args)...); - } - - //! Delete one or more records from this table which - //! match the provided constraint. - //! - //! This method is also used as the base function in - //! the N-argument deletion method below. - template - typename std::enable_if< - std::is_fundamental::value and - !std::is_same::value, - void>::type - deleteObjectsWhere(const char * col_name, - const constraints constraint, - const ColumnT col_val) - { - delete_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - finalizeDeletionStatement_(); - } - - //! Same as single-argument record deletion method above, - //! but specific to const char* constraint values. - template - typename std::enable_if< - std::is_same::type, const char*>::value, - void>::type - deleteObjectsWhere(const char * col_name, - const constraints constraint, - ColumnT col_val) - { - delete_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - finalizeDeletionStatement_(); - } - - //! Same as single-argument record deletion method above, - //! but specific to std::string constraint values. - template - typename std::enable_if< - std::is_same::value, - void>::type - deleteObjectsWhere(const char * col_name, - const constraints constraint, - const ColumnT & col_val) - { - delete_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - finalizeDeletionStatement_(); - } - - //! Same as single-argument record deletion method above, - //! but specific to initializer list constraints. Supports - //! constraints like {10,24,26} (integer) and {"a","b","c"} - //! (string literals). - template - void deleteObjectsWhere(const char * col_name, - const constraints constraint, - const std::initializer_list & col_val) - { - delete_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - finalizeDeletionStatement_(); - } - - //! Delete one or more records from this table which - //! match the provided constraints. - template - void deleteObjectsWhere(const char * col_name, - const constraints constraint, - const ColumnT & col_val, - Args &&... args) - { - delete_where_clauses_.add(col_name, col_val) - ->setConstraint(constraint); - deleteObjectsWhere(std::forward(args)...); - } - - //! Delete **ALL** records in this table. This operation - //! cannot be undone! - void deleteAllObjects() - { - finalizeDeletionStatement_(); - } - - //! The various create*() methods will return ObjectRef - //! wrappers around the newly created record by default. - //! Disable that behavior with a call to this method. - //! The create*() methods will return nullptr until - //! this TableRef object is told to do otherwise. - void neverReturnObjectRefsOnCreate() { - explicit_return_object_ = ExplicitReturnObject::NEVER_RETURN; - } - - //! Make a call to this method if you want the various - //! create*() method calls to return ObjectRef wrappers - //! around the newly created records. This is the default - //! behavior. The create*() methods will return ObjectRef - //! wrappers until this TableRef object is told to do - //! otherwise. - void alwaysReturnObjectRefsOnCreate() { - explicit_return_object_ = ExplicitReturnObject::ALWAYS_RETURN; - } - - //! Some tables are configured to be able to capture - //! summaries of their record value(s). Some columns - //! do not support summaries, such as blobs and strings, - //! but numeric columns may have summary support. This - //! method returns true if the summary snapshot was - //! successful, false otherwise. Note that returning - //! false does not mean that an error occurred, but - //! that this table did not have any columns that were - //! able to be summarized. - //! - //! Also note that tables only have at most one summary - //! record associated with them. Calling this method - //! on a table that had previously been summarized will - //! overwrite the summary record with the updated values. - bool captureSummary(); - -private: - //! At the end of a TableRef::updateRowValues() function - //! call, this method returns a RecordFinder which is bound - //! to 'this' TableRef: - //! - //! table->updateRowValues("MyString", "helloWorld"). - //! forRecordsWhere("MyDouble", simdb::constraints::equal, 4.5); - //! - //! The RecordFinder class provides TableRef access through - //! friendship to its constructor, and a public set of API's - //! that let the outside world build the WHERE clause of the - //! UPDATE statement. - RecordFinder & makeRecordFinder_() - { - std::function finalize_callback = [&]() { - return finalizeUpdateStatement_(); - }; - record_finder_for_update_.reset(new RecordFinder(finalize_callback)); - is_in_update_statement_ = true; - return *record_finder_for_update_; - } - - //! When using the createObjectWithArgs() API's, this - //! finalize method gets called when all creation args - //! in the parameter pack have been unrolled. - std::unique_ptr finalizeCreationStatement_(); - - //! When using the deleteObjectsWhere() API's, this finalize - //! method gets called when all deletion args/clauses in the - //! parameter pack have been unrolled. - void finalizeDeletionStatement_(); - - //! When using the updateRowValues() API's, this finalize - //! method gets called when all update args/clauses in the - //! parameter pack have been unrolled. - size_t finalizeUpdateStatement_(); - std::unique_ptr record_finder_for_update_; - - //! Object creation method used by createObject() and - //! createObjectWithArgs() public APIs. - std::unique_ptr createDefaultObject_(); - - //! ObjectRef's are returned from the various create*() - //! methods by default. There are separate APIs which - //! lets the caller say whether record wrappers should - //! always be returned, or never returned. Those APIs - //! set this member variable. - enum class ExplicitReturnObject { - ALWAYS_RETURN, - NEVER_RETURN, - DEFAULT - }; - ExplicitReturnObject explicit_return_object_ = - ExplicitReturnObject::DEFAULT; - - //! Calls to updateRowValues() use the same member variables - //! as createObjectWithArgs() do, but calls to these API's cannot - //! be mixed. Here are separate UPDATE and INSERT calls done the - //! correct way: - //! - //! table->updateRowValues("MyString", "foo"). - //! forRecordsWhere("MyDouble", constraints::equal, 50); - //! - //! table->createObjectWithArgs("MyString", "bar", - //! "MyDouble", 45.89); - //! - //! Those two calls would not affect each other. However, here - //! are calls that would result in incorrect data values in the - //! end: - //! - //! auto & updater = table->updateRowValues("MyString", "foo"); - //! - //! table->createObjectWithArgs("MyString", "bar", - //! "MyDouble", 45.89); - //! - //! updater.forRecordsWhere("MyDouble", constraints::equal, 50); - //! - //! This is invalid because a createObjectWithArgs() function - //! call is made when the UPDATE statement was being built. - //! This flag lets us throw in the case where UPDATE/INSERT - //! API's are mixed. - bool is_in_update_statement_ = false; - - //! DELETE clauses we hold onto while unrolling a - //! parameter pack for deleteObjectsWhere() calls. - ColumnValueContainer delete_where_clauses_; - - //! Common table member variables for all INSERT, - //! UPDATE, and DELETE actions. - const std::string table_name_; - const ObjectManager & obj_mgr_; - - //! Proxy back-pointer that was given to us by the - //! ObjectManager. - std::shared_ptr db_proxy_; - - //! List of column names and their data types - std::vector col_descriptors_; - - //! Map of table summary calculation functions - //! by summary function name. For instance: - //! - //! {{ "min", [](const std::vector & vals) { - //! return vals.empty() ? NAN : - //! *std::min_element(vals.begin(), vals.end()); - //! }, - //! { "max", [](const std::vector & vals) { - //! return vals.empty() ? NAN : - //! *std::max_element(vals.begin(), vals.end()); - //! }} - NamedSummaryFunctions summary_fcns_; - - //! Record factory that was given to us by the - //! ObjectManager. - AnySizeObjectFactory any_size_record_factory_; - FixedSizeObjectFactory fixed_size_record_factory_; - - //! This container holds onto column values and an - //! enumeration which gives the column data type. - //! Values are accessible via the ColumnValue's - //! getAs() method. - ColumnValueContainer col_values_; - - //! To support data writes using only raw bytes, - //! without cluttering the APIs with column names - //! that we don't strictly need to use, we will - //! hold onto the raw bytes in this char vector. - std::vector raw_bytes_for_obj_create_; - - //! Append to the 'raw bytes' member variable during - //! calls to createObjectWithVals() - template - typename std::enable_if< - std::is_arithmetic::value, - void>::type - appendRawValue_(const ColumnT col_val) { - const size_t cur_num_bytes = raw_bytes_for_obj_create_.size(); - const size_t new_num_bytes = cur_num_bytes + sizeof(ColumnT); - raw_bytes_for_obj_create_.resize(new_num_bytes); - - memcpy(&raw_bytes_for_obj_create_[cur_num_bytes], - &col_val, sizeof(ColumnT)); - } - - //! Append to the 'raw bytes' member variable during - //! calls to createObjectWithVals() - template - typename std::enable_if< - std::is_trivially_constructible::value and - not std::is_arithmetic::value, - void>::type - appendRawValue_(const ColumnT & structure) { - const size_t cur_num_bytes = raw_bytes_for_obj_create_.size(); - const size_t new_num_bytes = cur_num_bytes + sizeof(ColumnT); - raw_bytes_for_obj_create_.resize(new_num_bytes); - - memcpy(&raw_bytes_for_obj_create_[cur_num_bytes], - &structure, sizeof(ColumnT)); - } - - //! Private constructor, just for ObjectManager to call. - TableRef(const std::string & table_name, - const ObjectManager & obj_mgr, - const std::shared_ptr & db_proxy, - const std::vector & col_descriptors, - const NamedSummaryFunctions & summary_fcns, - AnySizeObjectFactory any_size_record_factory, - FixedSizeObjectFactory fixed_size_record_factory) : - table_name_(table_name), - obj_mgr_(obj_mgr), - db_proxy_(db_proxy), - col_descriptors_(col_descriptors), - summary_fcns_(summary_fcns), - any_size_record_factory_(any_size_record_factory), - fixed_size_record_factory_(fixed_size_record_factory) - {} - - friend class ObjectManager; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/async/AsyncTaskEval.hpp b/sparta/simdb/include/simdb/async/AsyncTaskEval.hpp deleted file mode 100644 index 6f75e901bf..0000000000 --- a/sparta/simdb/include/simdb/async/AsyncTaskEval.hpp +++ /dev/null @@ -1,630 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/async/TimerThread.hpp" -#include "simdb/async/ConcurrentQueue.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/Errors.hpp" - -#include -#include -#include -#include -#include - -namespace simdb { - -/*! - * \brief Base class used for all tasks that are given - * to the worker task queue. The completeTask() method - * will be called when this task's turn is up on the - * worker thread. - */ -class WorkerTask -{ -public: - virtual ~WorkerTask() {} - virtual void completeTask() = 0; - - uint64_t getId() const { - return id_; - } - -protected: - WorkerTask() : id_(incId_()) - {} - -private: - static uint64_t incId_() { - static uint64_t id = 0; - return id++; - } - const uint64_t id_; -}; - -/*! - * \brief Specialized worker task used in order to break - * out of the consumer thread without synchronously asking - * it to do so. - */ -class WorkerInterrupt : public WorkerTask -{ -protected: - //! WorkerTask implementation - void completeTask() override { - throw InterruptException(); - } -}; - -/*! - * \brief If you want to register your class objects for - * pre-flush notifications from the AsyncTaskEval, you - * should subclass from this Notifiable class. - * - * The notifyTaskQueueAboutToFlush() method will get - * called right before end-of-simulation task queue - * flushes. - */ -class Notifiable -{ -public: - virtual ~Notifiable() = default; - - std::weak_ptr getWeakPtr() { - return std::weak_ptr(self_ptr_); - } - - virtual void notifyTaskQueueAboutToFlush() = 0; - -protected: - Notifiable() : self_ptr_(this, [](Notifiable*){}) - {} - -private: - std::shared_ptr self_ptr_; -}; - -/*! - * \brief Thread-safe task queue. Used together with the - * AsyncTaskEval class to create a single-producer, single- - * consumer queue of work requests. - * - * -> main thread (producer) ==> (work to do) ==> (put in queue) - * -> main thread (producer) ==> (work to do) ==> (put in queue) - * | - * | - * [work packet] - * [work packet] - * ... - * - * -> work thread (consumer) ==================> ^^^^^^^^^^^^^^^^^ - * (consume data queue) - * | - * [lots of work at once] - * / \ - * [Database] - */ -class WorkerTaskQueue -{ -public: - //! Add a task you wish to evaluate off the main thread - void addTask(std::unique_ptr task) { - task_queue_.emplace(task.release()); - } - - //! Evaluate every task that has been queued up. This is - //! typically called by a worker thread, but may be called - //! from the main thread at synchronization points like - //! simulation pause/stop. - void flushQueue() { - std::unique_ptr task; - while (task_queue_.try_pop(task)) { - task->completeTask(); - } - } - -private: - // Give the AsyncTaskController the ability to pop work - // packets out of this queue, and evaluate them itself. - // This is needed for scenarios where there are several - // ObjectManager's writing to different database files, - // yet all WorkerTask's are in this single queue: - // - // ObjMgrA ... TaskA - - // | - // ----> [taskA1, taskB1, taskA2, ...] - // | - // ObjMgrB ... TaskB - - // - // It needs to "deinterleave" the tasks so that it can - // do two separate database transactions (two *in this - // example* - there could be more): - // - // ObjMgrA.safeTransaction([&]() { - // for task in TaskQueueA { - // task->completeTask() - // } - // }); - // - // ObjMgrB.safeTransaction([&]() { - // for task in TaskQueueB { - // task->completeTask() - // } - // }); - bool popQueue_(std::unique_ptr & task) { - return task_queue_.try_pop(task); - } - friend class AsyncTaskController; - - ConcurrentQueue> task_queue_; -}; - -class AsyncTaskEval; - -/*! - * \class AsyncTaskController - * - * \brief This class is used in order to allow more than - * one database file to be written to in the same process. - * Without this controller, you would put WorkerTask's onto - * your ObjectManager's AsyncTaskEval, and there would be - * a thread dedicated to that AsyncTaskEval to consume the - * work for that database file. But a 1-to-1 link between - * an ObjectManager and a thread (TimerThread) poses some - * performance problems - too many threads for little or - * no gain, being bottlenecked around disk I/O. - * - * To prevent performance from degrading, all ObjectManager's - * can be added to just a single AsyncTaskController, and - * they will all share the same background thread for their - * individual tasks. See ObjectManager.h for more details - * on how to link these two classes. - */ -class AsyncTaskController -{ -public: - explicit AsyncTaskController(const double interval_seconds = 0.1) : - timed_eval_(new TimedEval(interval_seconds, this)) - {} - - ~AsyncTaskController() = default; - - //! Allow users of this thread object to register themselves - //! for notifications that the worker queue is about to be - //! flushed. - void registerForPreFlushNotifications(Notifiable & notif) { - pre_flush_listeners_.emplace_back(notif.getWeakPtr()); - } - - //! Send out a notification to all registered listeners that - //! we are about to flush the worker queue. - void emitPreFlushNotification(); - - //! Force a synchronous flush of all WorkerTask's that - //! are currently in this queue. - void flushQueue() { - flushQueues_(); - } - - //! Wait for the worker queue to be flushed / consumed, - //! and stop the consumer thread. - //! - //! \warning DO NOT call this method from any WorkerTask - //! subclass' completeTask() method. If the completeTask() - //! method was being invoked from this controller's own - //! consumer thread (which is usually the case), this - //! method will hang. It is safest to call this method - //! from code that you know is always on the main thread, - //! for example in setup or teardown / post-processing - //! code in a simulation. - void stopThread() { - //Put a special interrupt packet in the queue. This - //does nothing but throw an interrupt exception when - //its turn is up. - std::unique_ptr interrupt(new WorkerInterrupt); - task_queue_.addTask(std::move(interrupt)); - - //Join the thread and just wait forever... (until the - //exception is thrown). - timed_eval_->stop(); - } - -private: - //! TimerThread implementation. Called at regular - //! intervals on a worker thread. - class TimedEval : public TimerThread - { - private: - TimedEval(const double interval_seconds, - AsyncTaskController * task_eval) : - TimerThread(TimerThread::Interval::FIXED_RATE, - interval_seconds), - task_eval_(task_eval) - {} - - void execute_() override { - task_eval_->flushQueues_(); - } - - AsyncTaskController *const task_eval_; - friend class AsyncTaskController; - }; - - //! Add the provided AsyncTaskEval. These AsyncTaskEval.'s are - //! basically back pointers we can use to emit pre-flush - //! notifications to any client code that registered itself - //! for these notifications. - void addTaskQueue_(AsyncTaskEval * task_queue) - { - client_task_queues_.insert(task_queue); - } - - //! When a WorkerTask is added to an ObjectManager's task - //! queue, if that ObjectManager belongs to this controller - //! it will reroute the task to our work queue through this - //! method. - void addWorkerTask_( - ObjectManager * sim_db, - std::unique_ptr task) - { - std::lock_guard lock(mutex_); - sim_dbs_by_task_id_[task->getId()] = sim_db; - task_queue_.addTask(std::move(task)); - - if (!timed_eval_->isRunning()) { - timed_eval_->start(); - } - } - - //! TimerThread (base class) implementation. Called - //! periodically on a background thread. - void flushQueues_() { - std::unique_ptr task; - - //This controller has just a single queue of work requests. - //The queue can have work that belongs to more than one - //database file. For example, simultaneous SPARTA statistics - //logging (SQLite) and branch prediction (HDF5) are completely - //independent, yet they share this one queue (and one thread). - //We "deinterleave" the queued work so that we can put all - //of one database's WorkerTask into its own safeTransaction(), - //then do the same for the next ObjectManager we are serving, - //and so on. - std::unordered_map< - ObjectManager*, - std::vector>> db_tasks; - - //There can be WorkerTask's that were added to our queue - //without any ObjectManager to go with it. An example of - //when this could happen is a simulator that pushes a - //post-simulation flush/interrupt WorkerTask into the - //queue, even though nobody ever made an ObjectManager - //(there were no --report options, no -z options, etc.) - // - //Making calling code keep track of whether they can do - //something as harmless as flushing or interrupting an - //empty task queue is not very user-friendly. And there - //is also always a chance that this async task queue is - //being used for non-SimDB background work (reusing SimDB - //for its threading utilities without needing a database - //for anything). - std::vector> no_db_tasks; - - { - //Grab our mutex to protect the sim_dbs_by_task_id_ member variable - std::lock_guard lock(mutex_); - - while (task_queue_.popQueue_(task)) { - auto simdb_iter = sim_dbs_by_task_id_.find(task->getId()); - if (simdb_iter == sim_dbs_by_task_id_.end()) { - no_db_tasks.emplace_back(task.release()); - } else { - const auto id = task->getId(); - db_tasks[sim_dbs_by_task_id_[id]].emplace_back(task.release()); - } - } - } - - std::unordered_map< - std::string, - std::vector>> db_tasks_by_db_file; - - std::unordered_map< - std::string, - ObjectManager*> db_conns_by_db_file; - - //Perform the deinterleave. We'll be left with one queue (vector) - //of WorkerTask's for each database file (ObjectManager) currently - //in use. - for (auto & db_task_queue : db_tasks) { - const auto & db_file = db_task_queue.first->getDatabaseFile(); - auto & src_tasks = db_task_queue.second; - auto & dst_tasks = db_tasks_by_db_file[db_file]; - - for (auto & src_task : src_tasks) { - dst_tasks.emplace_back(src_task.release()); - } - db_conns_by_db_file[db_file] = db_task_queue.first; - } - - assert(db_tasks_by_db_file.size() == db_conns_by_db_file.size()); - - //Perform a high-level safeTransaction() for each database - //currently in use, and inside each transaction, only evaluate - //the WorkerTask's that belong to the ObjectManager that is - //invoking the safeTransaction() - auto task_queue_iter = db_tasks_by_db_file.begin(); - auto db_conn_iter = db_conns_by_db_file.begin(); - - //For example, say that we have three databases in use, - //and there are work requests for all of them right here... - //ObjMgrA, ObjMgrB, and ObjMgrC. - while (task_queue_iter != db_tasks_by_db_file.end()) { - db_conn_iter->second->safeTransaction([&]() { - try { - //Say we are inside the safeTransaction() - //for ObjMgrB. Every WorkerTask in this - //vector that we are looping over here - //is strictly for ObjMgrB's database. - for (auto & current_task : task_queue_iter->second) { - current_task->completeTask(); - } - } catch (const InterruptException &) { - //Do not report this "exception". It does - //not mean anything went wrong; interrupts - //are purposely put in the queue when the - //queue (or its owning class) is asked to - //stop the thread. - } - }); - - //In the above example, we just committed the - //safeTransaction() for ObjMgrB... the next - //iteration in this while loop would be for - //ObjMgrC/ObjMgrA (it's a map keyed off of - //raw pointers, so there is no real ordering) - ++task_queue_iter; - ++db_conn_iter; - } - - //Note that there is no safeTransaction() wrapping these - //last WorkerTask's, since they are not associated with - //any ObjectManager. - try { - for (auto & current_task : no_db_tasks) { - current_task->completeTask(); - } - } catch (const InterruptException &) { - timed_eval_->stop(); - } - } - - std::unordered_map< - uint64_t, - ObjectManager*> sim_dbs_by_task_id_; - - std::unordered_set< - AsyncTaskEval*> client_task_queues_; - - WorkerTaskQueue task_queue_; - std::vector> pre_flush_listeners_; - std::unique_ptr timed_eval_; - mutable std::recursive_mutex mutex_; - friend class AsyncTaskEval; - friend class TimedEval; -}; - -/*! - * \brief Use this class to evaluate code asynchronously. - * - * IMPORTANT: Every one of these objects will get their own - * background thread. Don't create too many of them! One - * of these objects can serve an unlimited number of - * WorkerTask's, so typically you will only create one - * AsyncTaskEval object, and add all of your tasks to it - * one by one during simulation. - * - * \note There is a default limit for the total number of - * these objects you can make. Query the methods to find - * out if there is room for another: - * - * - getMaxTaskThreadsAllowed() - * - getCurrentNumTaskThreadsCreated() - */ -class AsyncTaskEval -{ -public: - //! Construct a task evaluator that will execute every - //! 'interval_seconds' that you specify. - explicit AsyncTaskEval(const double interval_seconds = 0.1) : - task_queue_(new WorkerTaskQueue), - timed_eval_(new TimedEval(interval_seconds, this)) - {} - - ~AsyncTaskEval() { - timed_eval_->stop(); - } - - static uint64_t getMaxTaskThreadsAllowed() { - return TimerThread::getMaxTaskThreadsAllowed(); - } - - static uint64_t getCurrentNumTaskThreadsCreated() { - return TimerThread::getCurrentNumTaskThreadsCreated(); - } - - //! Give this AsyncTaskEval shared ownership of the provided - //! database object. This is used in order to group together - //! WorkerTask items on the background thread and put them - //! inside larger periodic safeTransaction() calls. For some - //! database (DbConnProxy) implementations such as SQLite, - //! this typically results in much faster throughput for - //! database writes. - //! . . . . . . . . . . . . . . . . . . . - //! - //! Main thread Worker thread - //! =============== ================= - //! (interval 1) ====>> safeTransaction([&]() { - //! --> task1 task1->completeTask() - //! --> task2 task2->completeTask() - //! --> task3 task3->completeTask() - //! --> ... }) - //! --> ... - //! ...... - //! - //! (interval 2) ====>> safeTransaction([&]() { - //! --> task137 task137->completeTask() - //! --> task138 task138->completeTask() - //! --> ... }) - //! --> ... - //! ...... - //! - void setSimulationDatabase(ObjectManager * obj_mgr) { - sim_db_ = obj_mgr; - } - - //! Tell this task queue to forward all future WorkerTask's - //! it is given into the shared AsyncTaskController you provide. - //! If this AsyncTaskEval already had launched its own consumer - //! thread, it will be torn down. The controller object will start - //! its own thread that we can leverage. - bool addToTaskController(AsyncTaskController * ctrl) { - if (ctrl == nullptr || sim_db_ == nullptr) { - return false; - } - - if (sim_db_ && sim_db_->getDbConn()) { - flushQueue(); - if (timed_eval_->isRunning()) { - stop_(); - } - } - - task_controller_ = ctrl; - task_controller_->addTaskQueue_(this); - - return true; - } - - //! Allow users of this thread object to register themselves - //! for notifications that the worker queue is about to be - //! flushed. - void registerForPreFlushNotifications(Notifiable & notif) { - pre_flush_listeners_.emplace_back(notif.getWeakPtr()); - } - - //! Send out a notification to all registered listeners that - //! we are about to flush the worker queue. - void emitPreFlushNotification() { - for (auto & listener : pre_flush_listeners_) { - if (auto callback_obj = listener.lock()) { - callback_obj->notifyTaskQueueAboutToFlush(); - } - } - } - - //! Add a task for asynchronous eval. This will start - //! the worker thread if it is the first added task. - void addWorkerTask(std::unique_ptr task) { - if (task_controller_ != nullptr) { - task_controller_->addWorkerTask_(sim_db_, std::move(task)); - } else { - task_queue_->addTask(std::move(task)); - if (!timed_eval_->isRunning()) { - timed_eval_->start(); - } - } - } - - //! Evaluate all pending tasks. - void flushQueue() { - //If this consumer is going to a database, wrap the - //entire flush in a high-level transaction. This - //typically gives much better performance than - //doing database commits one at a time. - if (sim_db_ != nullptr) { - sim_db_->safeTransaction([&]() { - task_queue_->flushQueue(); - }); - } else { - task_queue_->flushQueue(); - } - } - - //! Stop the consumer thread. This triggers a flush on - //! the queue of tasks, evaluating each completeTask() - //! method as it does so. - //! - //! IMPORTANT: Do not try to call 'stop()' from inside - //! any of your WorkerTask's completeTask() methods or - //! the thread will wait forever and block whatever - //! thread you call this method from (typically the - //! main thread). - void stopThread() { - stop_(); - } - -private: - //! TimerThread implementation. Called at regular - //! intervals on a worker thread. - class TimedEval : public TimerThread - { - private: - TimedEval(const double interval_seconds, - AsyncTaskEval * task_eval) : - TimerThread(TimerThread::Interval::FIXED_RATE, - interval_seconds), - task_eval_(task_eval) - {} - - void execute_() override { - task_eval_->flushQueue(); - } - - AsyncTaskEval *const task_eval_; - friend class AsyncTaskEval; - }; - - //! Teardown consumer / work thread. - void stop_() { - if (task_controller_ != nullptr) { - task_controller_->stopThread(); - } else { - //Put a special interrupt packet in the queue. This - //does nothing but throw an interrupt exception when - //its turn is up. - std::unique_ptr interrupt(new WorkerInterrupt); - task_queue_->addTask(std::move(interrupt)); - - //Join the thread and just wait forever... (until the - //exception is thrown). - timed_eval_->stop(); - } - } - - AsyncTaskController * task_controller_ = nullptr; - std::unique_ptr task_queue_; - ObjectManager * sim_db_ = nullptr; - std::unique_ptr timed_eval_; - std::vector> pre_flush_listeners_; - friend class TimedEval; -}; - -//! Send out a notification to all registered listeners that -//! we are about to flush the worker queue. Implemented down -//! here so we can call AsyncTaskEval methods. -inline void AsyncTaskController::emitPreFlushNotification() -{ - for (auto & listener : pre_flush_listeners_) { - if (auto callback_obj = listener.lock()) { - callback_obj->notifyTaskQueueAboutToFlush(); - } - } - for (auto & task_queue : client_task_queues_) { - task_queue->emitPreFlushNotification(); - } -} - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/async/ConcurrentQueue.hpp b/sparta/simdb/include/simdb/async/ConcurrentQueue.hpp deleted file mode 100644 index 1598b64b00..0000000000 --- a/sparta/simdb/include/simdb/async/ConcurrentQueue.hpp +++ /dev/null @@ -1,58 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include - -namespace simdb { - -/*! - * \brief Thread-safe wrapper around std::queue (FIFO) - */ -template -class ConcurrentQueue -{ -public: - ConcurrentQueue() = default; - - //! Push an item to the back of the queue. - void push(const DataT & item) { - std::lock_guard guard(mutex_); - queue_.emplace(std::move(item)); - } - - //! Emplace an item to the back of the queue. - template - void emplace(Args&&... args) { - std::lock_guard guard(mutex_); - queue_.emplace(std::forward(args)...); - } - - //! Get the item at the front of the queue. This - //! returns true if successful, or false if there - //! was no data in the queue. - bool try_pop(DataT & item) { - std::lock_guard guard(mutex_); - if (queue_.empty()) { - return false; - } - std::swap(item, queue_.front()); - queue_.pop(); - return true; - } - - //! How many data points are in the queue? - size_t size() const { - std::lock_guard guard(mutex_); - return queue_.size(); - } - -private: - mutable std::mutex mutex_; - std::queue queue_; -}; - -} // namespace simdb - - diff --git a/sparta/simdb/include/simdb/async/TimerThread.hpp b/sparta/simdb/include/simdb/async/TimerThread.hpp deleted file mode 100644 index 8a52c1c5a0..0000000000 --- a/sparta/simdb/include/simdb/async/TimerThread.hpp +++ /dev/null @@ -1,246 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Errors.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace simdb { - -/*! - * \brief Interruption class. We put one of these into the - * AsyncTaskEval's work queue. This throws an exception - * when the worker thread gets to this item in the queue, - * and this exception type is caught in order to break - * out of the infinite consumer loop. - */ -class InterruptException : public std::exception -{ -public: - const char * what() const noexcept override { - return "Infinite consumer loop has been interrupted"; - } - -private: - //Not to be created by anyone but the WorkerInterrupt - InterruptException() = default; - friend class WorkerInterrupt; -}; - -/*! - * \brief Thread utility used for fixed-interval execution - * of asynchronous tasks. - */ -class TimerThread -{ -public: - //! Types of timer intervals. Currently, this utility - //! only supports fixed-rate execution. - enum class Interval : int8_t { - FIXED_RATE - }; - - //! Give the timer the fixed wall clock interval in - //! seconds. Your execute_() method will be called - //! every 'n' seconds like so: - //! - //! \code - //! class HelloWorld : public TimerThread { - //! public: - //! HelloWorld() : TimerThread(2.5) - //! {} - //! private: - //! void execute_() override { - //! std::cout << "Hello, world!" << std::endl; - //! } - //! }; - //! - //! HelloWorld obj; - //! obj.start(); - //! // "Hello, world!" printed after 2.5 seconds - //! // "Hello, world!" printed after 5.0 seconds - //! // "Hello, world!" printed after 7.5 seconds - //! \endcode - //! - //! Notes: - //! - The execute_() method is called for the very first - //! time after the interval has elapsed. It is not called - //! immediately from the TimerThread constructor. - //! - The timer interval is not guaranteed and may vary - //! at runtime and show different intervals from one - //! program execution to another. - //! - If your execute_() callback takes more than the - //! interval specified, your callback will be called - //! again immediately. It will not sleep before calling - //! your method. - TimerThread(const Interval interval, const double seconds) : - interval_seconds_(seconds) - { - (void) interval; - } - - //! When the timer goes out of scope, the execute_() callbacks - //! will be stopped. - virtual ~TimerThread() { - if (stress_testing_) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - stop(); - } - - static uint64_t getMaxTaskThreadsAllowed() { - return TimerThread::max_task_threads_allowed_; - } - - static uint64_t getCurrentNumTaskThreadsCreated() { - return TimerThread::current_num_task_threads_; - } - - //! Bugs uncovered in this class will typically be sporadic - //! due to timing of the background thread(s). This method - //! is used for testing purposes only. It injects pauses to - //! purposely draw out otherwise very sporadic bugs. - //! - //! \warning Calling this method in production code will - //! slow down execution of your program. - static void enableStressTesting() { - stress_testing_ = true; - } - - //! Disable stress testing. See the comment above about - //! this test-only setting. - static void disableStressTesting() { - stress_testing_ = false; - } - - //! Call this method from the main thread to start - //! timed execution of your execute_() method. - void start() { - if (thread_ == nullptr) { - is_running_ = true; - thread_.reset(new std::thread([&]() { - start_(); - })); - - if (TimerThread::current_num_task_threads_ >= - TimerThread::max_task_threads_allowed_) - { - throw DBException( - "Too many task thread objects have been created (the current " - "limit is ") << TimerThread::max_task_threads_allowed_ << ")"; - } - - ++TimerThread::current_num_task_threads_; - } - } - - //! Call this method from the main thread to stop - //! timed execution of your execute_() method. - //! You may NOT call this method from inside your - //! execute_() callback, or the timer thread will - //! not be able to be torn down. - void stop() { - is_running_ = false; - if (thread_ != nullptr) { - thread_->join(); - thread_.reset(); - - if (TimerThread::current_num_task_threads_ > 0) { - --TimerThread::current_num_task_threads_; - } - } - } - - //! Ask if this timer is currently executing at - //! regular intervals on the background thread. - //! This does not mean that it is in the middle - //! of calling the client background thread code, - //! just that the thread itself is still alive. - bool isRunning() const { - return thread_ != nullptr; - } - -private: - //! The timer's delayed start callback - void start_() { - sleepUntilIntervalEnd_(); - intervalFcn_(); - } - - //! The timer's own interval callback which includes - //! your execute_() implementation and sleep duration - //! calculation. - void intervalFcn_() { - while (is_running_) { - //Get the time before calling the user's code - const Time interval_start = getCurrentTime_(); - - try { - execute_(); - } catch (const InterruptException &) { - is_running_ = false; - continue; - } - - //Take the amount of time it took to execute the user's - //code, and use that info to sleep for the amount of time - //that puts the next call to execute_() close to the fixed - //interval. - auto interval_end = getCurrentTime_(); - std::chrono::duration user_code_execution_time = - interval_end - interval_start; - - const double num_seconds_into_this_interval = - user_code_execution_time.count(); - sleepUntilIntervalEnd_(num_seconds_into_this_interval); - } - } - - //! Go to sleep until the current time interval has expired. - //! - //! |----------------|----------------|----------------| - //! ^ - //! (sleeps until........^) - //! - //! |----------------|----------------|----------------| - //! ^ - //! (sleeps until...^) - void sleepUntilIntervalEnd_(const double offset_seconds = 0) { - const double sleep_seconds = interval_seconds_ - offset_seconds; - if (sleep_seconds > 0) { - auto sleep_ms = std::chrono::milliseconds(static_cast(sleep_seconds * 1000)); - std::this_thread::sleep_for(sleep_ms); - } - } - - typedef std::chrono::time_point Time; - - //! Get the current time. Used in sleep_for calculation. - Time getCurrentTime_() const { - return std::chrono::high_resolution_clock::now(); - } - - //! User-supplied method which will be called at regular - //! intervals on a worker thread. - virtual void execute_() = 0; - - const double interval_seconds_; - std::unique_ptr thread_; - bool is_running_ = false; - - static constexpr uint64_t max_task_threads_allowed_ = 2; - static std::atomic current_num_task_threads_; - static bool stress_testing_; -}; - -} // namespace simdb - - diff --git a/sparta/simdb/include/simdb/impl/hdf5/DataTypeUtils.hpp b/sparta/simdb/include/simdb/impl/hdf5/DataTypeUtils.hpp deleted file mode 100644 index 27979190de..0000000000 --- a/sparta/simdb/include/simdb/impl/hdf5/DataTypeUtils.hpp +++ /dev/null @@ -1,228 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/ColumnMetaStructs.hpp" -#include "simdb/Errors.hpp" - -#include -#include -#include -#include -#include - -namespace simdb { - -//! \class HDF5ScopedDataType -//! -//! \brief Utility class which closes HDF5 data type -//! resources when it goes out of scope. -class HDF5ScopedDataType -{ -public: - //! \brief Construction - //! - //! \param dtype HDF5 data type ID - //! - //! \param close_on_destroy Flag telling this object - //! if it should invoke H5Tclose() on destruction - HDF5ScopedDataType(const hid_t dtype, - const bool close_on_destroy) : - dtype_(dtype), - close_on_destroy_(close_on_destroy) - {} - - //! \brief Call H5Tclose() if we were instructed to - //! do so on destruction - ~HDF5ScopedDataType() { - if (close_on_destroy_ && dtype_ > 0) { - H5Tclose(dtype_); - } - } - - //! \return HDF5 data type ID - hid_t getDataTypeId() const { - return dtype_; - } - -private: - hid_t dtype_ = 0; - bool close_on_destroy_ = false; -}; - -//! \brief Get the equivalent H5T_NATIVE_* data type ID -//! for the given schema column -//! -//! \warning This method only supports built-in data types. -//! Types such as string will return -1, which is an invalid -//! identifier for all of the H5T*() APIs. -hid_t getNativeDTypeIdForHDF5(const Column & col) -{ - using dt = ColumnDataType; - - switch (col.getDataType()) { - case dt::char_t: { - return H5T_NATIVE_CHAR; - } - case dt::int8_t: { - return H5T_NATIVE_CHAR; - } - case dt::uint8_t: { - return H5T_NATIVE_UCHAR; - } - case dt::int16_t: { - return H5T_NATIVE_SHORT; - } - case dt::uint16_t: { - return H5T_NATIVE_USHORT; - } - case dt::int32_t: { - return H5T_NATIVE_INT; - } - case dt::uint32_t: { - return H5T_NATIVE_UINT; - } - case dt::int64_t: { - return H5T_NATIVE_LONG; - } - case dt::uint64_t: { - return H5T_NATIVE_ULONG; - } - case dt::float_t: { - return H5T_NATIVE_FLOAT; - } - case dt::double_t: { - return H5T_NATIVE_DOUBLE; - } - default: { - return -1; - } - } -} - -//! \brief Translate a SimDB column object into an equivalent -//! HDF5ScopedDataType object. This takes into account the base -//! data type of the column as well as its dimensionality. -//! -//! \param col Schema object to turn into the HDF5 equivalent -//! -//! \return Object holding the data type ID that matches the -//! data type of the column -HDF5ScopedDataType getScopedDTypeForHDF5(const Column & col) -{ - const std::vector & dims = col.getDimensions(); - const hid_t hdtype = getNativeDTypeIdForHDF5(col); - if (hdtype < 0) { - throw DBException("Unsupported data type encountered"); - } - - std::vector hdims; - if (!dims.empty()) { - hdims.reserve(dims.size()); - for (auto dimsval : dims) { - hdims.emplace_back(dimsval); - } - } else { - hdims.resize(1); - hdims[0] = 1; - } - - //The base data type is a POD type, and we do not have - //to call H5Tclose() for these predefined data types - //HDF5 provides out of the box. However, if the column - //dimensions were specified as non-scalar, then we do - //have to create/register our own custom data type, and - //therefore we do have to tell the HDF5ScopedDataType - //to take ownership of the data type handle, and close - //it from its destructor. - if (std::accumulate(hdims.begin(), hdims.end(), 1, - std::multiplies()) == 1) - { - const bool take_ownership = false; - return HDF5ScopedDataType(hdtype, take_ownership); - } - - const auto hdims_sz = static_cast(hdims.size()); - const auto hdims_ptr = hdims.data(); - const bool take_ownership = true; - - return HDF5ScopedDataType( - H5Tarray_create(hdtype, hdims_sz, hdims_ptr), - take_ownership); -} - -//! \brief Utility method used in errors and warnings -//! -//! \param col Schema column to stringize -//! -//! \return Stringized column data type. Takes into -//! account the column base type as well as dimensions. -std::string getColumnDTypeStr(const Column & col) -{ - std::ostringstream oss; - oss << col.getName(); - - const auto & dims = col.getDimensions(); - if (dims.empty()) { - return oss.str(); - } - - oss << "("; - if (dims.size() == 1) { - oss << dims[0]; - } else { - for (size_t idx = 0; idx < dims.size() - 1; ++idx) { - oss << dims[idx] << ","; - } - oss << dims.back(); - } - oss << ")"; - return oss.str(); -} - -//! \brief Get the SimDB column data type enumeration that -//! is equivalent to the provided HDF5 identifier. HDF5 SimDB -//! currently only supports native data types, which includes -//! integers and floating-point types. -//! -//! \warning This method throws if the provided identifier is -//! none of the supported types, such as string or blob/opaque. -//! -//! \param tid HDF5 data type ID -//! -//! \return simdb::ColumnDataType equivalent -ColumnDataType getPODColumnDTypeFromHDF5(const hid_t tid) -{ - using dt = ColumnDataType; - - if (H5Tequal(tid, H5T_NATIVE_CHAR)) { - return dt::char_t; - } else if (H5Tequal(tid, H5T_NATIVE_SCHAR)) { - return dt::int8_t; - } else if (H5Tequal(tid, H5T_NATIVE_UCHAR)) { - return dt::uint8_t; - } else if (H5Tequal(tid, H5T_NATIVE_SHORT)) { - return dt::int16_t; - } else if (H5Tequal(tid, H5T_NATIVE_USHORT)) { - return dt::uint16_t; - } else if (H5Tequal(tid, H5T_NATIVE_INT)) { - return dt::int32_t; - } else if (H5Tequal(tid, H5T_NATIVE_UINT)) { - return dt::uint32_t; - } else if (H5Tequal(tid, H5T_NATIVE_LONG)) { - return dt::int64_t; - } else if (H5Tequal(tid, H5T_NATIVE_ULONG)) { - return dt::uint64_t; - } else if (H5Tequal(tid, H5T_NATIVE_FLOAT)) { - return dt::float_t; - } else if (H5Tequal(tid, H5T_NATIVE_DOUBLE)) { - return dt::double_t; - } - - //HDF5 SimDB currently only supports POD scalars - //and POD arrays/matrices. - throw DBException("Unrecognized data type encountered"); -} - -} - diff --git a/sparta/simdb/include/simdb/impl/hdf5/HDF5ConnProxy.hpp b/sparta/simdb/include/simdb/impl/hdf5/HDF5ConnProxy.hpp deleted file mode 100644 index ee8f80453b..0000000000 --- a/sparta/simdb/include/simdb/impl/hdf5/HDF5ConnProxy.hpp +++ /dev/null @@ -1,223 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/DbConnProxy.hpp" - -#include - -namespace simdb { - -/*! - * \class HDF5ConnProxy - * - * \brief Proxy object which forwards database commands - * to the physical database APIs. ObjectManager does not - * provide direct access to the database handle; going - * through this proxy prevents misuse of database commands, - * and throws if it sees a command that is disallowed. - */ -class HDF5ConnProxy : public DbConnProxy -{ -public: - //! \brief HDF5 database files have "*.h5" file extension - //! - //! \return Hard-coded ".h5" extension - const char * getDatabaseFileExtension() const override { - return ".h5"; - } - - //! \brief This validate method gets called when a schema is - //! given to an ObjectManager to use with an HDF5 connection. - //! - //! \param schema Schema object to validate - void validateSchema(const Schema & schema) const override; - - //! \brief Turn a Schema object into an actual database - //! connection - //! - //! \param schema Schema object to realize - //! - //! \param obj_mgr SimDB ObjectManager that this DbConnProxy - //! object belongs to - void realizeSchema( - const Schema & schema, - const ObjectManager & obj_mgr) override; - - //! \brief Try to open a connection to an existing database file - //! - //! \param db_file Name of the HDF5 file - //! - //! \return Return true on successful connection, false otherwise - bool connectToExistingDatabase(const std::string & db_file) override; - - //! \brief Get the full database filename being used. This - //! includes the database path, stem, and extension. Returns - //! empty if no connection is open. - //! - //! \return Full filename, such as "/tmp/abcd-1234.h5" - std::string getDatabaseFullFilename() const override; - - //! \return Returns true if this database is open and ready to - //! take read and write requests, false otherwise - bool isValid() const override; - - //! \brief HDF5 does not support or need atomic transactions - //! - //! \return False, hard-coded - bool supportsAtomicTransactions() const override { - return false; - } - - //! \brief Not implemented. Do not call - this will throw. - void performDeletion( - const std::string &, - const ColumnValues &) const override - { - throw DBException("Not implemented"); - } - - //! \brief Not implemented. Do not call - this will throw. - size_t performUpdate( - const std::string &, - const ColumnValues &, - const ColumnValues &) const override - { - throw DBException("Not implemented"); - } - - //! \brief For tables that only have fixed-size columns, - //! we can read the raw bytes for the requested record - //! properties directly. The HDF5 library will give us the - //! byte offset to the column (prop_name) in question, - //! and the DatabaseID is just a linear offset into the - //! table itself. - //! - //! struct MyFoo { - //! int16_t x; - //! int16_t y; - //! }; - //! - //! "Give me prop_name 'y' for element #14" - //! - //! --> "Go to the element offset 13, and offset - //! further another 2 bytes" - //! - //! \param table_name Name of the table containing data - //! we want to read - //! - //! \param prop_name Name of the specific property (column / - //! field) in that table being read - //! - //! \param db_id Unique database ID for the requested - //! record. Equivalent to rowid in SQL. - //! - //! \param dest_ptr Preallocated memory the raw bytes from - //! the database should be written into - //! - //! \param num_bytes Number of bytes of preallocated memory - //! the dest_ptr is pointing to - //! - //! \return Number of bytes read - size_t readRawBytes( - const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - void * dest_ptr, - const size_t num_bytes) const override; - - //! \brief Provide an object/record factory for the given table - //! - //! \param table_name Name of the table we want to create objects - //! for (table rows) - //! - //! \return Factory suitable for creating records for the given - //! table - AnySizeObjectFactory getObjectFactoryForTable( - const std::string & table_name) const override; - - //! \brief Provide an object/record factory for the given table. - //! This returns a factory that can only work with tables of - //! fixed-size column data types (POD's). - //! - //! \note Currently, HDF5 SimDB only supports scalar/matrix POD's. - //! The AnySizeObjectFactory objects returned above and the - //! FixedSizeObjectFactory objects returned here do the same - //! thing at the moment. Though working with FixedSizeObjectFactory - //! should be preferred for fixed-size schema tables since it is - //! considerably faster than the AnySizeObjectFactory equivalent. - //! - //! \param table_name Name of the table we want to create objects - //! for (table rows) - //! - //! \return Factory suitable for creating fixed-size records for - //! the given table - FixedSizeObjectFactory getFixedSizeObjectFactoryForTable( - const std::string & table_name) const override; - - //! \brief Respond when our AnySizeObjectFactory is invoked. - //! Create a new object with the provided column values. - //! - //! \param table_name Name of the table we want to create an - //! object for (table row) - //! - //! \param values Vector of objects which are holding the - //! column names and their values for the new record - //! - //! \return Database ID of the newly created record - DatabaseID createObject( - const std::string & table_name, - const ColumnValues & values); - - //! \brief Respond when our FixedSizeObjectFactory is invoked. - //! Create a new object with the provided raw bytes. Since the - //! table is fixed in its column(s) width, the raw bytes array - //! passed in contains a fixed, known number of bytes that the - //! HDF5 library can read from. This byte array has all of the - //! new record's column value(s) all packed together. - //! - //! \param table_name Name of the table we want to create an - //! object for (table row) - //! - //! \param raw_bytes_ptr Flat byte array containing all of the - //! columns' values for the new record - //! - //! \return Database ID of the newly created record - DatabaseID createFixedSizeObject( - const std::string & table_name, - const void * raw_bytes_ptr, - const size_t num_raw_bytes); - - HDF5ConnProxy(); - ~HDF5ConnProxy() override; - -private: - //! First-time database file open. - std::string openDbFile_( - const std::string & db_dir, - const std::string & db_file, - const bool open_file) override; - - //! Not implemented. - void prepareStatement_(const std::string &, - void **) const override - { - throw DBException("Not implemented"); - } - - //! Until HDF5ConnProxy supports ObjectQuery, calls to - //! ObjectManager::findObject() will reroute to this - //! method. - bool hasObjectImpl_( - const std::string & table_name, - const DatabaseID db_id) const override; - - //! Rest of the implementation is hidden from view. This - //! is to reduce unnecessary includes and to hide the - //! specific database tech being used. - class Impl; - std::shared_ptr impl_; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/impl/hdf5/Resources.hpp b/sparta/simdb/include/simdb/impl/hdf5/Resources.hpp deleted file mode 100644 index 057da884d6..0000000000 --- a/sparta/simdb/include/simdb/impl/hdf5/Resources.hpp +++ /dev/null @@ -1,188 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "hdf5.h" - -/*! - * \file Resources.h - * \brief The HDF5 library returns opaque resource handles - * as integer IDs from many of its APIs. Things like file - * handles and dataspace handles are returned when you open - * or create a file/dataspace in an HDF5 database, and they - * must be closed/released using a separate HDF5 API. Failure - * to do so can leak these resources, and eventually you may - * see errors coming from HDF5 saying there are no more IDs - * available. The classes in this file can be thought of as - * smart pointers for all of the HDF5 resource types SimDB - * uses. - */ - -#define INVALID_HID_T -1 - -namespace simdb { - -//! Default behavior is to take no action when an HDF5 -//! resource goes out of scope. -struct H5DefaultDeleter { - void operator()(const hid_t id) const { - (void) id; - } -}; - -/*! - * \class H5Resource - * \brief This class holds onto an HDF5 resource ID, calling - * the appropriate resource deleter when H5Resource goes out - * of scope. - */ -template -class H5Resource -{ -public: - //! Create an H5Resource without a resource handle. - H5Resource() : H5Resource(INVALID_HID_T) - {} - - //! Create an H5Resource associated with the given identifier. - H5Resource(const hid_t id) : id_(id) - {} - - //! No copies, no moves. - H5Resource(const H5Resource &) = delete; - H5Resource & operator=(const H5Resource &) = delete; - - //! Move constructor. The newly constructed resource - //! will be solely responsible for closing the handle. - H5Resource(H5Resource && rhs) : id_(rhs.id_) { - rhs.id_ = INVALID_HID_T; - } - - //! Move assignment operator. The newly assigned resource - //! will be solely responsible for closing the handle. If - //! the resource being assigned to was already holding an - //! HDF5 resource identifier, it will be closed/released. - H5Resource & operator=(H5Resource && rhs) { - if (id_ != INVALID_HID_T && id_ != rhs.id_) { - del_(id_); - } - id_ = rhs.id_; - rhs.id_ = INVALID_HID_T; - return *this; - } - - //! Close the resource on destroy. - ~H5Resource() { - if (id_ != INVALID_HID_T) { - del_(id_); - } - } - - //! Whether or not this is usable - bool good() const { - return id_ != INVALID_HID_T; - } - - //! Assignment from a raw HDF5 identifier. If the resource - //! being assigned to was already holding an HDF5 resource - //! identifier, it will be closed/released. - H5Resource & operator=(const hid_t id) { - if (id_ != INVALID_HID_T && id_ != id) { - del_(id_); - } - id_ = id; - return *this; - } - - //! Cast to the underlying HDF5 identifier value. - operator hid_t() const { - return id_; - } - - bool operator==(const H5Resource & rhs) const { - return id_ == rhs.id_; - } - - bool operator!=(const H5Resource & rhs) const { - return id_ != rhs.id_; - } - - bool operator<(const H5Resource & rhs) const { - return id_ < rhs.id_; - } - - bool operator>(const H5Resource & rhs) const { - return id_ > rhs.id_; - } - - bool operator==(const hid_t rhs) const { - return id_ == rhs; - } - - bool operator!=(const hid_t rhs) const { - return id_ != rhs; - } - - bool operator<(const hid_t rhs) const { - return id_ < rhs; - } - - bool operator>(const hid_t rhs) const { - return id_ > rhs; - } - -private: - DeleterT del_; - hid_t id_ = INVALID_HID_T; -}; - -//! H5Resource deleter for HDF5 file handles -struct H5FDeleter { - void operator()(const hid_t id) const { - H5Fclose(id); - } -}; -typedef H5Resource H5FResource; - -//! H5Resource deleter for HDF5 group handles -struct H5GDeleter { - void operator()(const hid_t id) const { - H5Gclose(id); - } -}; -typedef H5Resource H5GResource; - -//! H5Resource deleter for HDF5 dataset handles -struct H5DDeleter { - void operator()(const hid_t id) const { - H5Dclose(id); - } -}; -typedef H5Resource H5DResource; - -//! H5Resource deleter for HDF5 data type handles -struct H5TDeleter { - void operator()(const hid_t id) const { - H5Tclose(id); - } -}; -typedef H5Resource H5TResource; - -//! H5Resource deleter for HDF5 dataspace handles -struct H5SDeleter { - void operator()(const hid_t id) const { - H5Sclose(id); - } -}; -typedef H5Resource H5SResource; - -//! H5Resource deleter for HDF5 property list handles -struct H5PDeleter { - void operator()(const hid_t id) const { - H5Pclose(id); - } -}; -typedef H5Resource H5PResource; - -} - diff --git a/sparta/simdb/include/simdb/impl/sqlite/Errors.hpp b/sparta/simdb/include/simdb/impl/sqlite/Errors.hpp deleted file mode 100644 index 33108f5301..0000000000 --- a/sparta/simdb/include/simdb/impl/sqlite/Errors.hpp +++ /dev/null @@ -1,21 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Errors.hpp" - -#include - -namespace simdb { - -//! General-purpose SQLite exception used for interrupts -class SqliteInterrupt : public DatabaseInterrupt -{ -protected: - virtual std::string getExceptionDetails_() const override { - return "SQLite"; - } -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/impl/sqlite/SQLiteConnProxy.hpp b/sparta/simdb/include/simdb/impl/sqlite/SQLiteConnProxy.hpp deleted file mode 100644 index c32a148e26..0000000000 --- a/sparta/simdb/include/simdb/impl/sqlite/SQLiteConnProxy.hpp +++ /dev/null @@ -1,166 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/DbConnProxy.hpp" - -#include - -namespace simdb { - -/*! - * \brief Proxy object which forwards database commands - * to the physical database APIs. ObjectManager does not - * provide direct access to the database handle; going - * through this proxy prevents misuse of database commands, - * and throws if it sees a command that is disallowed. - */ -class SQLiteConnProxy : public DbConnProxy -{ -public: - //! SQLite database files have "*.db" file extension. - const char * getDatabaseFileExtension() const override { - return ".db"; - } - - //! SQLite has some schema restrictions related to - //! column dimensionality. This validate method gets - //! called when a schema is given to an ObjectManager - //! to use with a SQLite connection. - void validateSchema(const Schema & schema) const override; - - //! Turn a Schema object into an actual database connection. - void realizeSchema( - const Schema & schema, - const ObjectManager & obj_mgr) override; - - //! Try to open a connection to an existing database file - bool connectToExistingDatabase(const std::string & db_file) override; - - //! Get the full database filename being used. This includes - //! the database path, stem, and extension. Returns empty if - //! no connected is open. - std::string getDatabaseFullFilename() const override; - - //! Execute the provided statement against the database - //! connection. This will validate the command, and throw - //! if this command is disallowed. - void eval(const std::string & command) const; - - //! Execute a SELECT statement against the database - //! connection. The provided callback will be invoked - //! once for each record found. Example: - //! - //! struct CallbackHandler { - //! int handle(int argc, char ** argv, char ** col_names) { - //! ... - //! return 0; - //! } - //! }; - //! - //! CallbackHandler handler; - //! - //! db_proxy->evalSelect( - //! "SELECT First,Last FROM Customers", - //! +[](void * handler, int argc, char ** argv, char ** col_names) { - //! return (CallbackHandler*)(handler)->handle(argc, argv, col_names); - //! }, - //! &handler); - //! - //! See TransactionUtils.h for more details about the callback - //! arguments that are passed to your SELECT handler. - void evalSelect( - const std::string & command, - int (*callback)(void *, int, char **, char **), - void * callback_obj) const; - - //! Check for nullptr database handle. - bool isValid() const override; - - //! Override the default way ObjectManager gets the table - //! names. It defaults to whatever Table objects were in the - //! Schema object it was originally given, but we may want to - //! override it so we can make some tables private/unqueryable - //! for internal use only. - void getTableNames( - std::unordered_set & table_names) override; - - //! SQLite gets a performance boost by grouping statements - //! in BEGIN TRANSACTION / COMMIT TRANSACTION pairs. - bool supportsAtomicTransactions() const override { - return true; - } - - //! Issue BEGIN TRANSACTION - void beginAtomicTransaction() const override; - - //! Issue COMMIT TRANSACTION - void commitAtomicTransaction() const override; - - //! Take the object constraints we are given, and delete - //! any records from the given table which match those - //! constraints. - void performDeletion( - const std::string & table_name, - const ColumnValues & where_clauses) const override; - - //! Take the object constraints we are given, and update - //! all records from the given table which match those - //! constraints. - //! - //! Returns the total number of updated records. - size_t performUpdate( - const std::string & table_name, - const ColumnValues & col_values, - const ColumnValues & where_clauses) const override; - - //! Provide an object/record factory for the given table. - AnySizeObjectFactory getObjectFactoryForTable( - const std::string & table_name) const override; - - //! Respond when our ObjectFactory is invoked. Create a new - //! object with the provided arguments. - DatabaseID createObject( - const std::string & table_name, - const ColumnValues & values); - - SQLiteConnProxy(); - ~SQLiteConnProxy() override; - -private: - //! This proxy is intended to be used directly with - //! the core SimDB classes. - friend class ObjectManager; - friend class ObjectRef; - friend class ObjectQuery; - friend class TableRef; - - //! First-time database file open. - std::string openDbFile_( - const std::string & db_dir, - const std::string & db_file, - const bool open_file) override; - - //! Create a prepared statement for the provided command. - //! The specific pointer type of the output void** is tied - //! to the database tech being used. This is intended to - //! be implementation detail, so this method is accessible - //! to friends only. - void prepareStatement_( - const std::string & command, - void ** statement) const override; - - //! Lookup optimization via ObjectQuery is enabled only - //! for SQLite SimDB today. This method will be removed - //! in the future. - bool supportsObjectQuery_() const override final { - return true; - } - - //! Rest of the implementation is hidden from view. - class Impl; - std::shared_ptr impl_; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/impl/sqlite/Schema.hpp b/sparta/simdb/include/simdb/impl/sqlite/Schema.hpp deleted file mode 100644 index 6f7fdd7885..0000000000 --- a/sparta/simdb/include/simdb/impl/sqlite/Schema.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/Schema.hpp" - -#include - -namespace simdb { - -//! Stream operator used when creating a SQL command from an ostringstream. -inline std::ostream & operator<<(std::ostream & os, const ColumnDataType dtype) -{ - using dt = ColumnDataType; - - switch (dtype) { - case dt::fkey_t: - case dt::char_t: - case dt::int8_t: - case dt::uint8_t: - case dt::int16_t: - case dt::uint16_t: - case dt::int32_t: - case dt::uint32_t: - case dt::int64_t: - case dt::uint64_t: { - os << "INT"; break; - } - - case dt::string_t: { - os << "TEXT"; break; - } - - case dt::float_t: - case dt::double_t: { - os << "FLOAT"; break; - } - - case dt::blob_t: { - os << "BLOB"; break; - } - } - - return os; -} - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/impl/sqlite/TransactionUtils.hpp b/sparta/simdb/include/simdb/impl/sqlite/TransactionUtils.hpp deleted file mode 100644 index 297863d808..0000000000 --- a/sparta/simdb/include/simdb/impl/sqlite/TransactionUtils.hpp +++ /dev/null @@ -1,112 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Errors.hpp" -#include "simdb_fwd.hpp" - -//Standard headers -#include - -namespace simdb { - -//! \brief SQLite may return error codes when evaluating a SQL -//! statement, i.e. with either "eval_sql()" or "eval_sql_select()" -//! below. We may trap these exceptions and keep retrying the -//! SQL statement until successful, or decide to rethrow the -//! exception: -//! -//! SQLite gives error code: SimDB throws: -//! -------------------------- ----------------------------- -//! SQLITE_BUSY simdb::SqlFileLockedException() -//! SQLITE_LOCKED simdb::SqlTableLockedException() - -class SqlFileLockedException : public DBAccessException -{ -public: - const char * what() const noexcept override { - return "The database file is locked"; - } -}; - -class SqlTableLockedException : public DBAccessException -{ -public: - const char * what() const noexcept override { - return "A table in the database is locked"; - } -}; - -//! \brief Evaluate a SQL command on an ObjectManager's connection proxy. -void eval_sql(const SQLiteConnProxy * db_proxy, - const std::string & command); - -//! \brief Optional callback that will be forwarded to sqlite when -//! executing a SELECT command. Use this with eval_sql_select() -//! calls. Your callback will be called once for each matching -//! record. Example usage: -//! -//! Say we want to execute the following statement: -//! SELECT (First,Last,Age) FROM Customers WHERE Balance > 1000 -//! -//! This statement could return any number of records, or none. -//! We could set up a SELECT callback like this: -//! -//! \code -//! struct SelectCustomersCallback { -//! std::vector> matches; -//! -//! int processMatchingRecord(int argc, char ** argv, char ** col_names) { -//! //Should be 3 columns returned for this statement -//! assert(argc == 3); -//! -//! //We might not care about the column names, but they should be: -//! assert(col_names[0] == std::string("First")); -//! assert(col_names[1] == std::string("Last")); -//! assert(col_names[2] == std::string("Age")); -//! -//! //The most important part... the column values: -//! std::string first_name = argv[0]; -//! std::string last_name = argv[1]; -//! int age = atoi(argv[2]); -//! matches.emplace_back(first_name, last_name, age); -//! } -//! }; -//! -//! //IMPORTANT - Your object's callback function must return 'int' -//! // or it will fail to compile ("int processMatchingRecord...") -//! -//! //Make one of these callback objects: -//! SelectCustomersCallback select_cb; -//! -//! //Say we have an SQLiteConnProxy 'db_proxy' nearby... now call: -//! eval_sql_select(db_proxy, -//! "SELECT (First,Last,Age) FROM Customers WHERE Balance > 1000", -//! +[](void * callback_obj, int argc, char ** argv, char ** col_names) { -//! return static_cast(callback_obj)-> -//! processMatchingRecord(argc, argv, col_names); -//! }, -//! &select_cb); -//! -//! //Now the 'select_cb.matches' member variable has your results. -//! \endcode -//! -typedef int //Return 0 to indicate success (*see note below) -(*sqlite_select_callback)( - void * caller_ptr, //Object provided in the 4th argument of eval_sql_select() - int argc, //The number of columns in row - char ** argv, //An array of strings representing fields in the row - char ** col_names); //An array of strings representing column names - // - // *If something goes wrong when the SELECT statement calls - // your callback, throw an exception. Do not return specific - // SQL error codes. - -//! \brief Evaluate a SELECT SQL command on an open sqlite connection. -void eval_sql_select(const SQLiteConnProxy * db_proxy, - const std::string & command, - sqlite_select_callback callback, - void * callback_obj); - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/schema/ColumnMetaStructs.hpp b/sparta/simdb/include/simdb/schema/ColumnMetaStructs.hpp deleted file mode 100644 index 72509bcfab..0000000000 --- a/sparta/simdb/include/simdb/schema/ColumnMetaStructs.hpp +++ /dev/null @@ -1,299 +0,0 @@ -// -*- C++ -*- - -#pragma once - -/*! - * \brief Metaprogramming utilities for use - * with SimDB schemas. - */ - -#include "simdb/schema/GeneralMetaStructs.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" -#include "simdb/Errors.hpp" -#include "simdb/utils/CompatUtils.hpp" - -#include -#include -#include -#include - -namespace simdb { - -//! Base template for column_info structs -template -struct column_info; - -//! int8_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::int8_t; - } - using value_type = int8_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! uint8_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::uint8_t; - } - using value_type = uint8_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! int16_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::int16_t; - } - using value_type = int16_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! uint16_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::uint16_t; - } - using value_type = uint16_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! int32_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::int32_t; - } - using value_type = int32_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! uint32_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::uint32_t; - } - using value_type = uint32_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! int64_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::int64_t; - } - using value_type = int64_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! uint64_t -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::uint64_t; - } - using value_type = uint64_t; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! float -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::float_t; - } - using value_type = float; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! double -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::double_t; - } - using value_type = double; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! string -template -struct column_info::value or - std::is_same::type, const char*>::value>::type> -{ - static ColumnDataType data_type() { - return ColumnDataType::string_t; - } - using value_type = ColumnT; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! char -template <> -struct column_info { - static ColumnDataType data_type() { - return ColumnDataType::char_t; - } - using value_type = char; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! Vectors of raw bytes are stored as blobs (void* / opaque) -template -struct column_info::value>::type> -{ - static ColumnDataType data_type() { - return ColumnDataType::blob_t; - } - using value_type = typename is_container::value_type; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! Blob descriptor -template -struct column_info::value>::type> -{ - static ColumnDataType data_type() { - return ColumnDataType::blob_t; - } - using value_type = Blob; - static constexpr bool is_fixed_size = utils::is_pod::value; -}; - -//! See if the given column data type has a fixed number -//! of bytes, as determined by utils::is_pod -inline bool getColumnIsFixedSize(const ColumnDataType dtype) -{ - using dt = ColumnDataType; - - switch (dtype) { - case dt::char_t: { - return column_info::is_fixed_size; - } - case dt::int8_t: { - return column_info::is_fixed_size; - } - case dt::uint8_t: { - return column_info::is_fixed_size; - } - case dt::int16_t: { - return column_info::is_fixed_size; - } - case dt::uint16_t: { - return column_info::is_fixed_size; - } - case dt::int32_t: { - return column_info::is_fixed_size; - } - case dt::uint32_t: { - return column_info::is_fixed_size; - } - case dt::int64_t: { - return column_info::is_fixed_size; - } - case dt::uint64_t: { - return column_info::is_fixed_size; - } - case dt::float_t: { - return column_info::is_fixed_size; - } - case dt::double_t: { - return column_info::is_fixed_size; - } - case dt::string_t: { - return column_info::is_fixed_size; - } - case dt::blob_t: { - return column_info::is_fixed_size; - } - case dt::fkey_t: { - return true; - } - } - - simdb_throw("Unreachable") - return false; -} - -//! Get the number of bytes for a fixed-size column data type. -//! This method throws if getColumnIsFixedSize() returns -//! false for the given data type. -inline size_t getFixedNumBytesForColumnDType( - const ColumnDataType dtype, - const std::vector & dims = {1}) -{ - using dt = ColumnDataType; - - if (!getColumnIsFixedSize(dtype)) { - throw DBException("Data type is not fixed-size"); - } - - const size_t dims_mult = - dims.empty() ? 1UL : - std::accumulate(dims.begin(), dims.end(), - 1, std::multiplies()); - - switch (dtype) { - case dt::char_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::int8_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::uint8_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::int16_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::uint16_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::int32_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::uint32_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::int64_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::uint64_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::float_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::double_t: { - return dims_mult * sizeof(column_info::value_type); - } - case dt::fkey_t: { - return sizeof(DatabaseID); - } - case dt::string_t: - case dt::blob_t: { - //Data types that are not fixed width should have thrown - //an exception at the top of this method. This should be - //unreachable. - simdb_throw("Unreachable") - break; - } - } - - simdb_throw("Unreachable") - return 0; -} - -} - diff --git a/sparta/simdb/include/simdb/schema/ColumnTypedefs.hpp b/sparta/simdb/include/simdb/schema/ColumnTypedefs.hpp deleted file mode 100644 index a4150fcf9e..0000000000 --- a/sparta/simdb/include/simdb/schema/ColumnTypedefs.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include - -namespace simdb { - -//! Data types supported by SimDB schemas -enum class ColumnDataType : int8_t { - int8_t, - uint8_t, - int16_t, - uint16_t, - int32_t, - uint32_t, - int64_t, - uint64_t, - float_t, - double_t, - char_t, - string_t, - blob_t, - fkey_t -}; - -//! From a table's perspective, each column can be uniquely -//! described by its column name and its data type. -using ColumnDescriptor = std::pair; - -//! Blob descriptor used for writing and reading raw bytes -//! to/from the database. -struct Blob { - const void * data_ptr = nullptr; - size_t num_bytes = 0; -}; - -} - diff --git a/sparta/simdb/include/simdb/schema/ColumnValue.hpp b/sparta/simdb/include/simdb/schema/ColumnValue.hpp deleted file mode 100644 index c1f3eed96a..0000000000 --- a/sparta/simdb/include/simdb/schema/ColumnValue.hpp +++ /dev/null @@ -1,325 +0,0 @@ -// -*- C++ -*- - -#pragma once - -//! SimDB column values can be numeric, strings, or -//! blobs, and in all cases they can be represented -//! with a data type enumeration, a void* that can -//! be casted to the actual type (int16_t, double, -//! etc.) and the name of the column. - -#include "simdb/schema/ColumnMetaStructs.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/Errors.hpp" - -#include -#include - -namespace simdb { - -//! This object holds onto the minimum information needed -//! to get a column's value. It uses a void* to the data -//! and is able to cast it to the correct underlying type. -class ColumnValueBase -{ -public: - virtual ~ColumnValueBase() = default; - - const std::string & getColumnName() const { - return name_; - } - - ColumnDataType getDataType() const { - return dt_; - } - - const void * getDataPtr() const { - return getValuePtrAt_(0); - } - - template - typename std::enable_if< - !std::is_same::value and - !std::is_same::value and - !std::is_same::type, const char*>::value, - T>::type - getAs() const { - if (column_info::data_type() != dt_) { - throw DBException("Invalid call to ColumnValueBase::getAs() ") - << "- attempt to cast to invalid data type"; - } - const void * val = getValuePtrAt_(0); - return *static_cast::value_type*>(val); - } - - template - typename std::enable_if< - std::is_same::type, const char*>::value, - T>::type - getAs() const { - if (column_info::data_type() != dt_) { - throw DBException("Invalid call to ColumnValueBase::getAs() ") - << "- attempt to cast to invalid data type"; - } - const void * val = getValuePtrAt_(0); - return static_cast::value_type>(val); - } - - template - typename std::enable_if< - std::is_same::value, - T>::type - getAs() const { - const char * charval = getAs(); - return std::string(charval); - } - - template - typename std::enable_if< - std::is_same::value, - T>::type - getAs() const { - if (column_info::data_type() != dt_) { - throw DBException("Invalid call to ColumnValueBase::getAs() ") - << "- attempt to cast to invalid data type"; - } - - Blob blob_descriptor; - const void * val = getValuePtrAt_(0); - const Blob * my_descriptor = static_cast(val); - blob_descriptor.data_ptr = my_descriptor->data_ptr; - blob_descriptor.num_bytes = my_descriptor->num_bytes; - return blob_descriptor; - } - - template - typename std::enable_if< - !std::is_same::value and - !std::is_same::value and - !std::is_same::type, const char*>::value, - T>::type - getAs(const size_t idx) const { - if (column_info::data_type() != dt_) { - throw DBException("Invalid call to ColumnValueBase::getAs() ") - << "- attempt to cast to invalid data type"; - } - - using col_type = typename column_info::value_type; - const void * val = getValuePtrAt_(idx); - return *static_cast(val); - } - - template - typename std::enable_if< - std::is_same::type, const char*>::value, - T>::type - getAs(const size_t idx) const { - if (column_info::data_type() != dt_) { - throw DBException("Invalid call to ColumnValueBase::getAs() ") - << "- attempt to cast to invalid data type"; - } - - const void * val = getValuePtrAt_(idx); - return static_cast::value_type>(val); - } - - template - typename std::enable_if< - std::is_same::value, - T>::type - getAs(const size_t idx) const { - const char * charval = getAs(idx); - return std::string(charval); - } - - //! ColumnValue objects might be holding onto a set of - //! values, for example: - //! - //! UPDATE Accounts SET Active=0 - //! WHERE LastName IN ('Smith','Thompson') - //! - //! Call this method to get the number of column values - //! this object is holding. - size_t getNumValues() const { - return vals_.size(); - } - - //! For ColumnValue objects that are used when building - //! up a database WHERE clause, tack on the value constraint. - void setConstraint(const constraints constraint) { - if (constraint == constraints::INVALID) { - throw DBException( - "Cannot call ColumnValue::setConstraint() passing " - "in constraints::INVALID"); - } - constraint_ = constraint; - } - - //! For ColumnValue objects that are used when building - //! up a database WHERE clause, get the value constraint. - //! This throws if the setConstraint() method was never - //! called. Check hasConstraint() before calling this - //! method if you are unsure. - constraints getConstraint() const { - if (constraint_ == constraints::INVALID) { - throw DBException("ColumnValue::getConstraint() called ") - << "on an object whose constraint has not been set"; - } - return constraint_; - } - - //! See if this ColumnValue has a constraint attached to - //! it. This applies to ColumnValue objects that are used - //! when building a database WHERE clause for an UPDATE or - //! DELETE. - bool hasConstraint() const { - return constraint_ != constraints::INVALID; - } - - ColumnValueBase(const ColumnValueBase &) = default; - ColumnValueBase(ColumnValueBase &&) = default; - -protected: - ColumnValueBase(const std::string & name, - const ColumnDataType dt) : - name_(name), - dt_(dt) - {} - - void setValuePtr_(const void * valptr) { - vals_.emplace_back(valptr); - } - - void setValuePtrs_(std::vector && valptrs) { - vals_.insert(vals_.end(), valptrs.begin(), valptrs.end()); - } - -private: - //! Private data access. ColumnValueContainer is - //! only outside class who can get to this pointer - //! directly. Everyone else must go through the - //! getAs() methods. - inline const void * getValuePtr_() const { - return getValuePtrAt_(0); - } - inline const void * getValuePtrAt_(const size_t idx) const { - return vals_.at(idx); - } - friend class ColumnValueContainer; - - const std::string name_; - std::vector vals_; - const ColumnDataType dt_; - constraints constraint_ = constraints::INVALID; -}; - -//! Lightweight "copy" of table column's value. Identified -//! by its data type and a pointer to its value. Use the -//! data type to cast the pointer accordingly. -using ColumnValues = std::deque; - -//! Base template class for enable_if's below -template -class ColumnValue; - -//! Integer and floating-point values -template -class ColumnValue::value>::type> - : public ColumnValueBase -{ -public: - ColumnValue(const std::string & name, - const void * valptr) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtr_(valptr); - } - - ColumnValue(const std::string & name, - std::vector && valptrs) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtrs_(std::move(valptrs)); - } -}; - -//! String literal values -template -class ColumnValue::type, const char*>::value>::type> - : public ColumnValueBase -{ -public: - ColumnValue(const std::string & name, - const void * valptr) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtr_(valptr); - } - - ColumnValue(const std::string & name, - std::vector && valptrs) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtrs_(std::move(valptrs)); - } -}; - -//! std::string values -template -class ColumnValue::value>::type> - : public ColumnValueBase -{ -public: - ColumnValue(const std::string & name, - const void * valptr) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtr_(valptr); - } - - ColumnValue(const std::string & name, - std::vector && valptrs) : - ColumnValueBase(name, column_info::data_type()) - { - setValuePtrs_(std::move(valptrs)); - } -}; - -//! Blob values -template -class ColumnValue::value>::type> - : public ColumnValueBase -{ -public: - ColumnValue(const std::string & name, - const void * valptr) : - ColumnValueBase(name, ColumnDataType::blob_t) - { - setValuePtr_(valptr); - } -}; - -//! STL container values -template -class ColumnValue::value and - is_contiguous::value>::type> - : public ColumnValueBase -{ -public: - ColumnValue(const std::string & name, - const void * valptr) : - ColumnValueBase(name, ColumnDataType::blob_t) - { - setValuePtr_(valptr); - } -}; - -} - diff --git a/sparta/simdb/include/simdb/schema/ColumnValueContainer.hpp b/sparta/simdb/include/simdb/schema/ColumnValueContainer.hpp deleted file mode 100644 index ac79b7e609..0000000000 --- a/sparta/simdb/include/simdb/schema/ColumnValueContainer.hpp +++ /dev/null @@ -1,302 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/ColumnValue.hpp" -#include "simdb_fwd.hpp" - -#include -#include - -namespace simdb { - -/*! - * \brief This container holds onto column values and an - * enumeration which gives the column data type. - * Values are accessible via the ColumnValue's - * getAs() method. - */ -class ColumnValueContainer -{ -public: - //! Add a POD column value - template - typename std::enable_if< - std::is_fundamental::value, - ColumnValueBase*>::type - add(const char * col_name, - const ColumnT col_val) - { - const void * valptr = getColumnDataValuePtr_(col_val); - if (valptr) { - ColumnValue col(col_name, valptr); - col_values_.emplace_back(col); - return &col_values_.back(); - } - return nullptr; - } - - //! Add a string value (as string literal) - template - typename std::enable_if< - std::is_same::type, const char*>::value, - ColumnValueBase*>::type - add(const char * col_name, - ColumnT col_val) - { - const void * valptr = getColumnDataValuePtr_(col_val); - if (valptr) { - ColumnValue col(col_name, valptr); - col_values_.emplace_back(col); - return &col_values_.back(); - } - return nullptr; - } - - //! Add a string value (as std::string) - template - typename std::enable_if< - std::is_same::value, - ColumnValueBase*>::type - add(const char * col_name, - const ColumnT & col_val) - { - return add(col_name, col_val.c_str()); - } - - //! Add a blob value (as std::vector, where T is a scalar POD) - template - typename std::enable_if< - is_container::value and is_contiguous::value, - ColumnValueBase*>::type - add(const char * col_name, - const ColumnT & col_val) - { - const void * valptr = getColumnDataValuePtr_(col_val); - if (valptr) { - ColumnValue col(col_name, valptr); - col_values_.emplace_back(col); - return &col_values_.back(); - } - return nullptr; - } - - //! Add a blob value (as simdb::Blob) - template - typename std::enable_if< - std::is_same::value, - ColumnValueBase*>::type - add(const char * col_name, - const ColumnT & col_val) - { - const void * valptr = getColumnDataValuePtr_(col_val); - if (valptr) { - ColumnValue col(col_name, valptr); - col_values_.emplace_back(col); - return &col_values_.back(); - } - return nullptr; - } - - //! Add a set of values in an initialization list - template - ColumnValueBase * add(const char * col_name, - const std::initializer_list & col_vals) - { - std::vector valptrs; - for (const auto val : col_vals) { - const void * valptr = getColumnDataValuePtr_(val); - if (valptr) { - valptrs.emplace_back(valptr); - } - } - - if (!valptrs.empty()) { - ColumnValue col(col_name, std::move(valptrs)); - col_values_.emplace_back(col); - return &col_values_.back(); - } - return nullptr; - } - - //! Get the underlying ColumnValue objects. Access the - //! column values using the ColumnValue::getAs() - //! method. - const ColumnValues & getValues() const { - return col_values_; - } - - //! Ask if there are any column values in this container. - bool empty() const { - return col_values_.empty(); - } - - //! Clear all column value objects in this container. - void clear() { - for (const auto & col : col_values_) { - if (col.getDataType() == ColumnDataType::blob_t) { - const void * valptr = col.getValuePtr_(); - if (valptr) { - delete static_cast(valptr); - } - } - } - - col_values_.clear(); - held_8bit_int_values_.clear(); - held_16bit_int_values_.clear(); - held_32bit_int_values_.clear(); - held_64bit_int_values_.clear(); - held_float_values_.clear(); - held_double_values_.clear(); - } - -private: - //! Return a void* to 8-bit integer column values - template - typename std::enable_if< - sizeof(ColumnT) == sizeof(int8_t) and - std::is_integral::value and - not std::is_same::type, const char*>::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_8bit_int_values_.emplace_back(static_cast(col_val)); - return &held_8bit_int_values_.back(); - } - - //! Return a void* to 16-bit integer column values - template - typename std::enable_if< - sizeof(ColumnT) == sizeof(int16_t) and - std::is_integral::value and - not std::is_same::type, const char*>::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_16bit_int_values_.emplace_back((int16_t)col_val); - return &held_16bit_int_values_.back(); - } - - //! Return a void* to 32-bit integer column values - template - typename std::enable_if< - sizeof(ColumnT) == sizeof(int32_t) and - std::is_integral::value and - not std::is_same::type, const char*>::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_32bit_int_values_.emplace_back((int32_t)col_val); - return &held_32bit_int_values_.back(); - } - - //! Return a void* to 64-bit integer column values - template - typename std::enable_if< - sizeof(ColumnT) == sizeof(int64_t) and - std::is_integral::value and - not std::is_same::type, const char*>::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_64bit_int_values_.emplace_back((int64_t)col_val); - return &held_64bit_int_values_.back(); - } - - //! Return a void* to std::string column values - template - typename std::enable_if< - std::is_same::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT & col_val) { - return col_val.c_str(); - } - - //! Return a void* to string literal column values - template - typename std::enable_if< - std::is_same::type, const char*>::value, - const void*>::type - getColumnDataValuePtr_(ColumnT col_val) { - return col_val; - } - - //! Return a void* to float column values - template - typename std::enable_if< - std::is_same::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_float_values_.emplace_back(col_val); - return &held_float_values_.back(); - } - - //! Return a void* to double column values - template - typename std::enable_if< - std::is_same::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT col_val) { - held_double_values_.emplace_back(col_val); - return &held_double_values_.back(); - } - - //! Return a void* to data values held in contiguous - //! containers. For these column data types, we need - //! both the void* to the actual data, and the number - //! of bytes' worth of data values, so we return a - //! new'd Blob descriptor. We'll delete it later. - template - typename std::enable_if< - is_container::value and - is_contiguous::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT & col_val) { - if (col_val.empty()) { - return nullptr; - } - - constexpr auto nbytes = - sizeof(typename is_contiguous::value_type); - - Blob * blob_descriptor = new Blob; - blob_descriptor->data_ptr = &col_val[0]; - blob_descriptor->num_bytes = col_val.size() * nbytes; - return blob_descriptor; - } - - //! Return a void* to data values held in contiguous containers. - //! This is similar to the method above but is specifically for - //! data passed in as simdb::Blob's, as opposed to STL containers. - template - typename std::enable_if< - std::is_same::value, - const void*>::type - getColumnDataValuePtr_(const ColumnT & col_val) { - if (col_val.data_ptr == nullptr || col_val.num_bytes == 0) { - return nullptr; - } - - Blob * blob_descriptor = new Blob; - blob_descriptor->data_ptr = col_val.data_ptr; - blob_descriptor->num_bytes = col_val.num_bytes; - return blob_descriptor; - } - - //! Vector of objects holding onto column values. All values - //! are in this data structure as a void* pointing somewhere - //! else, along with the data type enumeration that can be - //! used to cast that void* to the right type. - ColumnValues col_values_; - - //! When we are given POD column values, we copy them into - //! deques so they remain in scope while we hand out void* - //! to the outside world (such as TableRef, or DbConnProxy - //! subclasses). - std::deque held_8bit_int_values_; - std::deque held_16bit_int_values_; - std::deque held_32bit_int_values_; - std::deque held_64bit_int_values_; - std::deque held_float_values_; - std::deque held_double_values_; -}; - -} - diff --git a/sparta/simdb/include/simdb/schema/DatabaseRoot.hpp b/sparta/simdb/include/simdb/schema/DatabaseRoot.hpp deleted file mode 100644 index c192d63e16..0000000000 --- a/sparta/simdb/include/simdb/schema/DatabaseRoot.hpp +++ /dev/null @@ -1,650 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/ObjectManager.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/utils/StringUtils.hpp" -#include "simdb/Errors.hpp" - -#include -#include -#include -#include - -namespace simdb { - -//! Signature of user-defined schema creation method. -using SchemaBuildFcn = std::function; - -//! Factory function which returns a DbConnProxy subclass. -using ProxyCreateFcn = std::function; - -/*! - * \class DatabaseNamespace - * - * \brief Database files are organized as follows: - * - * root (DatabaseRoot) - * -> namespace1 (DatabaseNamespace) - * --> table - * --> table - * -> namespace2 (DatabaseNamespace) - * --> table - * --> table - * - * This class is a container wrapping the namespace - * nodes in the database hierarchy. - */ -class DatabaseNamespace -{ -public: - /*! - * \brief Invoke registered schema builder methods. - * This will be triggered when using the GET_DB_* - * macros, but it also works if you want to inline - * your schema creation code like so: - * - * \code - * DatabaseNamespace * db_namespace = ... - * - * db_namespace->addToSchema([](simdb::Schema & schema) { - * schema.addTable("FizzBuzz") - * .addColumn("Fizz", simdb::ColumnDataType::int32_t) - * .addColumn("Buzz", simdb::ColumnDataType::int32_t); - * }); - * \endcode - * - * \param schema_builder Callable code which builds the - * schema from the SchemaBuildFcn you provide. - */ - void addToSchema(const SchemaBuildFcn & schema_builder) - { - simdb::Schema schema; - schema_builder(schema); - addSchema_(schema); - } - - /* - * \brief Check to see if there is a table in the underlying - * database with this name, taking into account our namespace - * as well. - * - * \param table_name Name of the target table you are looking for - * - * \return Returns true if found, false otherwise. - */ - bool hasTableNamed(const std::string & table_name) const { - return schema_.getTableNamed( - db_namespace_ + Table::NS_DELIM + table_name) != nullptr; - } - - /*! - * \brief Get the table by the given name, if it exists. - * - * \param table_name Name of the target table you are looking for - * - * \note The table name you pass in ideally should not contain - * the namespace delimiter '$' as it can lead to exceptions. - * For example, say we had a namespace called "Stats", with - * a table called "ReportTimeseries": - * - * \code - * DatabaseNamespace * ns = ... - * - * //This will never throw - * if (auto table = ns->getTableNamed("ReportTimeseries")) { - * table->... - * } - * - * //This *happens* to not throw, since the namespace - * //matched this object's namespace - * if (auto table = ns->getTableNamed("Stats$ReportTimeseries")) { - * table->... - * } - * - * //This *would* throw, since the namespace is "Stats", - * //not "Statistics" - * if (auto table = ns->getTableNamed("Statistics$ReportTimeseries")) { - * table->... - * } - * \endcode - * - * \note As long as you do not include the namespace delimiter - * '$' in the table name you pass in, this method will never - * throw; it returns null if the table is not found. - */ - const Table * getTableNamed(const std::string & table_name) const { - std::string requested_table_name; - auto delim_idx = table_name.find(std::string(1, Table::NS_DELIM)); - if (delim_idx != std::string::npos) { - const utils::lowercase_string requested_namespace_lower = - table_name.substr(0, delim_idx); - - const std::string requested_namespace = requested_namespace_lower; - - requested_table_name = - (delim_idx + 1 < table_name.size()) ? - table_name.substr(delim_idx + 1) : ""; - - if (!requested_namespace.empty() && - requested_namespace != db_namespace_) - { - throw DBException("Invalid namespace. This DatabaseNamespace ") - << "is named '" << db_namespace_ << "', but the requested namespace " - << "was '" << requested_namespace << "'"; - } - } else { - requested_table_name = table_name; - } - - return schema_.getTableNamed( - db_namespace_ + Table::NS_DELIM + requested_table_name); - } - - /* - * \brief Check to see if our underlying schema has any tables at all. - */ - bool hasSchema() const { - return schema_.hasTables(); - } - - /* - * \return Returns whether or not this database has an open - * connection to a database file. - */ - bool databaseConnectionEstablished() const; - - /* - * \brief Get an object which has many of the same ObjectManager - * APIs, and return an "intermediate" class object. This class - * sits right between the calling code (SPARTA simulator, unit - * tests, etc.) and the lower level objects that run SELECT / - * UPDATE / etc. database commands. - */ - ObjectManager::ObjectDatabase * getDatabase(); - - ~DatabaseNamespace() = default; - -private: - DatabaseNamespace(const utils::lowercase_string & db_namespace, - DatabaseRoot * db_root, - AsyncTaskController * task_controller) : - db_namespace_(db_namespace), - db_root_(db_root), - task_controller_(task_controller) - {} - - void addSchema_(Schema & schema) - { - schema.setNamespace_(db_namespace_); - for (auto & table : schema) { - if (Table * existing_table = schema_.getTableNamed(table.getName())) { - if (*existing_table != table) { - throw DBException("Invalid table added to schema. The table ") - << "has the same name as an existing schema table, but has " - << "a different column configuration. The offending table " - << "is '" << table.getName() << "'."; - } - continue; - } - schema_.addTable(table); - } - - //If we already have a database connection open, we - //need to forward these new schema tables to the - //ObjectManager we are associated with. Note that - //we also need to make the call to appendSchema() - //before unsetting the schema's namespace below. - if (db_root_) { - appendSchemaToConnectionIfOpen_(*db_root_, schema); - } - - schema.setNamespace_(""); - } - - void grantAccess_() { - if (cached_db_) { - cached_db_->grantAccess(); - } - access_granted_ = true; - } - - void revokeAccess_() { - if (cached_db_) { - cached_db_->revokeAccess(); - } - access_granted_ = false; - } - - void appendSchemaToConnectionIfOpen_( - DatabaseRoot & db_root, - Schema & schema); - - std::string db_namespace_; - Schema schema_; - DatabaseRoot * db_root_ = nullptr; - AsyncTaskController * task_controller_ = nullptr; - std::unique_ptr cached_db_; - bool access_granted_ = true; - friend class DatabaseRoot; -}; - -/*! - * \class DatabaseRoot - * - * \brief SimDB organizes its schema into namespaces. This class - * is at the top of the database hierarchy; it is a collection of - * SimDB namespaces. - */ -class DatabaseRoot -{ -public: - //! Construct with a database directory. All ObjectManager's - //! created underneath this DatabaseRoot will put their database - //! file(s)/artifacts in this directory. - DatabaseRoot(const std::string & db_dir = ".") : - db_dir_(db_dir), - task_controller_(new AsyncTaskController) - {} - - ~DatabaseRoot() = default; - - //! Access a SimDB namespace by the given name. The first time - //! this is called for a particular namespace, it will be created - //! for you. If a SchemaBuildFcn was registered with SimDB for - //! this namespace, it will be invoked. Otherwise, you will have - //! to populate the namespace schema yourself using the method - //! "DatabaseNamespace::addToSchema()" on the returned object. - DatabaseNamespace * getNamespace(const utils::lowercase_string & db_namespace) { - auto ns_iter = namespaces_.find(db_namespace); - if (ns_iter != namespaces_.end()) { - return ns_iter->second.get(); - } - - auto type_iter = db_types_by_namespace_.find(db_namespace); - if (type_iter == db_types_by_namespace_.end()) { - throw DBException("Unable to get namespace named '") - << db_namespace << "'. This namespace was not registered " - << "with SimDB."; - } - - std::unique_ptr ns(new DatabaseNamespace( - db_namespace, this, task_controller_.get())); - - auto build_iter = schema_builders_by_namespace_.find(db_namespace); - if (build_iter != schema_builders_by_namespace_.end()) { - for (auto & builder : build_iter->second) { - ns->addToSchema(builder); - } - } - - auto ret = ns.get(); - namespaces_[db_namespace].reset(ns.release()); - return ret; - } - - AsyncTaskController * getTaskController() const { - return task_controller_.get(); - } - - //! Let SimDB know the database type that should be used to - //! instantiate the schema for the given namespace. - //! - //! \param db_namespace Namespace to register. Case insenstive. - //! - //! \param db_type Database type for this namespace. Examples - //! include "sqlite" and "hdf5". Case insensitive. - //! - //! \warning This will throw if a namespace is passed in that - //! has already been registered *with a different db_type*. - //! - //! \note It is recommended that you use the macros found at the - //! bottom of this file instead of calling this method directly. - //! See the macro comments for details. - static void registerDatabaseNamespace( - const utils::lowercase_string & db_namespace, - const utils::lowercase_string & db_type) - { - auto iter = db_types_by_namespace_.find(db_namespace); - if (iter != db_types_by_namespace_.end()) { - if (iter->second != db_type) { - throw DBException("SimDB has already been registered with a ") - << "conflicting database type. Namespace '" << db_namespace << "' " - << "is registered for database type '" << iter->second << "', " - << "which conflicts with the new type '" << db_type << "'."; - } - } - db_types_by_namespace_[db_namespace] = db_type; - } - - //! Optionally give one of the SimDB namespaces a schema - //! build function/callback. When this namespace is accessed - //! for the first time, this callback will be invoked to - //! populate the namespace schema with the appropriate - //! empty tables. - //! - //! \param db_namespace Namespace of the schema build function - //! you are registering. Case insenstive. - //! - //! \param build_fcn Schema build function to register for - //! this namespace. - //! - //! \warning This will throw if the provided namespace already - //! has a schema build function registered for it, *and the new - //! build function is different than the existing build function*. - //! In order to determine if the new and existing schema builders - //! are identical, the build function will be invoked with a - //! blank Schema object, and the resulting built schema will be - //! compared against the existing schema. - //! - //! \note If you do not provide a schema build function for - //! your namespace, you have to call the DatabaseNamespace - //! "addToSchema()" method manually before you can start - //! writing any records into that namespace. Registering - //! your schema build callback is just a convenience so you - //! don't have to make the call yourself. - //! - //! \note It is recommended that you use the macros found at the - //! bottom of this file instead of calling this method directly. - //! See the macro comments for details. - static void registerSchemaBuilderForNamespace( - const utils::lowercase_string & db_namespace, - const SchemaBuildFcn & build_fcn) - { - schema_builders_by_namespace_[db_namespace].emplace_back(build_fcn); - } - - //! Give SimDB a DbConnProxy factory function for the given - //! database type. - //! - //! \param db_type Database type for this proxy factory. - //! Examples include "sqlite" and "hdf5". Case insensitive. - //! - //! \param create_fcn Schema builder callback function. Note - //! that this builder code may also be inlined with a lambda - //! at the call site. - //! - //! \note If there is already a proxy factory registered for - //! the provided database type, it will be overwritten. A warning - //! will be printed to stdout, but it will not reject it or throw. - //! - //! \note It is recommended that you use the macros found at the - //! bottom of this file instead of calling this method directly. - //! See the macro comments for details. - static void registerProxyCreatorForDatabaseType( - const utils::lowercase_string & db_type, - const ProxyCreateFcn & create_fcn) - { - if (proxy_creators_by_db_type_.find(db_type) != - proxy_creators_by_db_type_.end()) - { - std::cout << " [simdb] Warning: Database type '" << db_type << "' " - << "already has a DbConnProxy factory registered for it. " - << "The registered factory is being overwritten with the " - << "new factory." << std::endl; - } - proxy_creators_by_db_type_[db_type] = create_fcn; - } - -private: - // Return the database type for the given namespace. Database - // types are registered with SimDB using the registration - // macros at the bottom of this file. Examples of these strings - // include "SQLite" and "HDF5". - const std::string & getDatabaseTypeForNamespace_( - const utils::lowercase_string & db_namespace) const - { - auto type_iter = db_types_by_namespace_.find(db_namespace); - if (type_iter == db_types_by_namespace_.end()) { - throw DBException("No registered database type found ") - << "for namespace '" << db_namespace << "'"; - } - return type_iter->second; - } - - // Return the DbConnProxy factory for the given namespace. - // Throws if the namespace was not registered with SimDB - // beforehand. - const ProxyCreateFcn & getProxyCreatorForNamespace_( - const utils::lowercase_string & db_namespace) const - { - const std::string & db_type = getDatabaseTypeForNamespace_(db_namespace); - auto creation_iter = proxy_creators_by_db_type_.find(db_type); - if (creation_iter == proxy_creators_by_db_type_.end()) { - throw DBException("No registered DbConnProxy factory ") - << "found for namespace '" << db_namespace << "'"; - } - return creation_iter->second; - } - - // Return the ObjectManager associated with the given namespace. - // The first time this is called for a particular namespace, the - // ObjectManager will be created, the appropriate DbConnProxy will - // be instantiated, and its namespace schema will be realized with - // empty tables. - ObjectManager * getObjectManagerForNamespace_( - const utils::lowercase_string & db_namespace, - simdb::Schema & namespace_schema) - { - ObjectManager * ret = nullptr; - if (!hasObjectManagerForNamespace_(db_namespace)) { - std::unique_ptr sim_db(new ObjectManager(db_dir_)); - ret = sim_db.get(); - const ProxyCreateFcn & proxy_creator = - getProxyCreatorForNamespace_(db_namespace); - - namespace_schema.setNamespace_(db_namespace); - std::unique_ptr db_proxy(proxy_creator()); - ret->createDatabaseFromSchema(namespace_schema, std::move(db_proxy)); - namespace_schema.setNamespace_(""); - - sim_dbs_by_db_type_[db_types_by_namespace_[db_namespace]] = std::move(sim_db); - } else { - ret = sim_dbs_by_db_type_[db_types_by_namespace_[db_namespace]].get(); - - simdb::Schema combined_schema; - combined_schema.setNamespace_(db_namespace); - namespace_schema.setNamespace_(db_namespace); - - auto build_iter = schema_builders_by_namespace_.find(db_namespace); - if (build_iter != schema_builders_by_namespace_.end()) { - for (auto & builder : build_iter->second) { - builder(combined_schema); - } - } - combined_schema += namespace_schema; - - simdb::Schema pruned_schema; - pruned_schema.setNamespace_(db_namespace); - const auto & curr_realized_tables = ret->getTableNames(); - for (const auto & new_table : combined_schema) { - if (!curr_realized_tables.count(new_table.getName())) { - pruned_schema.addTable(new_table); - } - } - - if (pruned_schema.hasTables()) { - ret->appendSchema(pruned_schema); - } - namespace_schema.setNamespace_(""); - } - return ret; - } - - // See if the given namespace already has an ObjectManager created for it. - bool hasObjectManagerForNamespace_(const utils::lowercase_string & db_namespace) const { - auto type_iter = db_types_by_namespace_.find(db_namespace); - if (type_iter == db_types_by_namespace_.end()) { - throw DBException("No registered database type found for ") - << "namespace '" << db_namespace << "'"; - } - - auto db_iter = sim_dbs_by_db_type_.find(type_iter->second); - if (db_iter == sim_dbs_by_db_type_.end()) { - return false; - } - - auto sim_db = db_iter->second.get(); - if (sim_db == nullptr) { - throw DBException("Unexpectedly found a null ObjectManager"); - } - if (sim_db->getDbConn() == nullptr) { - throw DBException( - "Unexpectedly found an ObjectManager with a null DbConnProxy"); - } - return true; - } - - std::map> namespaces_; - std::string db_dir_; - std::map> sim_dbs_by_db_type_; - std::unique_ptr task_controller_; - static std::map db_types_by_namespace_; - static std::map> schema_builders_by_namespace_; - static std::map proxy_creators_by_db_type_; - friend class DatabaseNamespace; -}; - -inline bool DatabaseNamespace::databaseConnectionEstablished() const -{ - if (db_root_) { - return db_root_->hasObjectManagerForNamespace_(db_namespace_); - } - return false; -} - -inline ObjectManager::ObjectDatabase * DatabaseNamespace::getDatabase() -{ - if (cached_db_) { - if (access_granted_) { - cached_db_->grantAccess(); - } else { - cached_db_->revokeAccess(); - } - return cached_db_.get(); - } - - if (db_root_) { - auto sim_db = db_root_->getObjectManagerForNamespace_(db_namespace_, schema_); - sim_db->addToTaskController(task_controller_); - cached_db_.reset(new ObjectManager::ObjectDatabase(sim_db, db_namespace_, this)); - - if (access_granted_) { - cached_db_->grantAccess(); - } else { - cached_db_->revokeAccess(); - } - } - - return cached_db_.get(); -} - -inline void DatabaseNamespace::appendSchemaToConnectionIfOpen_( - DatabaseRoot & db_root, - Schema & schema) -{ - if (db_root.hasObjectManagerForNamespace_(db_namespace_)) { - Schema unused_schema; - auto sim_db = db_root.getObjectManagerForNamespace_( - db_namespace_, unused_schema); - - Schema pruned_schema; - schema.setNamespace_(""); - for (const auto & table : schema) { - if (sim_db->getQualifiedTableName(table.getName(), db_namespace_).empty()) { - pruned_schema.addTable(table); - } - } - pruned_schema.setNamespace_(db_namespace_); - sim_db->appendSchema(pruned_schema); - } -} - -//! Let SimDB know about your database namespace, and the type -//! of the database that goes with it. -//! -//! \param db_namespace Name of the registered namespace. Case -//! insensitive. -//! -//! \param db_type Database type for this namespace. Examples -//! include "sqlite" and "hdf5". Case insensitive. -#define REGISTER_SIMDB_NAMESPACE(db_namespace, db_type) \ - simdb::DatabaseRoot::registerDatabaseNamespace(#db_namespace, #db_type) - -//! Register a DbConnProxy factory for the given database type. -//! -//! \param db_type Database type for this proxy factory. -//! Examples include "sqlite" and "hdf5". Case insensitive. -//! -//! \param create_fcn Schema builder callback function. Note -//! that this builder code may also be inlined with a lambda -//! at the call site. -#define REGISTER_SIMDB_PROXY_CREATE_FUNCTION(db_type, proxy_creator) \ - simdb::DatabaseRoot::registerProxyCreatorForDatabaseType(#db_type, proxy_creator) - -//! Optionally provide SimDB with a schema build function -//! for the given namespace. Example: -//! -//! \code -//! void buildStatsSchema(simdb::Schema & schema) -//! { -//! schema.addTable("ReportHeader") -//! .addColumn(...) -//! .addColumn(...); -//! schema.addTable("ReportTimeseries") -//! .addColumn(...) -//! .addColumn(...); -//! } -//! REGISTER_SIMDB_SCHEMA_BUILDER(Stats, buildStatsSchema); -//! \endcode -//! -//! This method would be invoked the first time we request -//! the "Stats" namespace from SimDB: -//! -//! \code -//! DatabaseRoot * db_root = ... -//! DatabaseNamespace * stats_ns = db_root->getNamespace("Stats"); -//! \endcode -//! -//! In this example, we only provided a schema build function -//! for the "Stats" namespace. We can still manually add tables -//! to namespaces: -//! -//! \code -//! DatabaseRoot * db_root = ... -//! DatabaseNamespace * pipeline_ns = db_root->getNamespace("Pipeline"); -//! if (!pipeline_ns->hasSchema()) { -//! pipeline_ns->addToSchema([](simdb::Schema & schema) { -//! schema.addTable(...) -//! .addColumn(...) -//! .addColumn(...); -//! }); -//! } -//! \endcode -//! -//! You can also specify part of the namespace schema using -//! a regisered build callback, and add more tables manually: -//! -//! \code -//! DatabaseRoot * db_root = ... -//! DatabaseNamespace * stats_ns = db_root->getNamespace("Stats"); -//! assert(stats_ns->hasTableNamed("ReportHeader")); -//! assert(stats_ns->hasTableNamed("ReportTimeseries")); -//! -//! assert(!stats_ns->hasTableNamed("ReportVerifySummary")); -//! if (report_verif_enabled_) { -//! stats_ns->addToSchema([](simdb::Schema & schema) { -//! schema.addTable("ReportVerifySummary") -//! .addColumn(...) -//! .addColumn(...); -//! }); -//! } -//! \endcode -#define REGISTER_SIMDB_SCHEMA_BUILDER(db_namespace, schema_builder) \ - simdb::DatabaseRoot::registerSchemaBuilderForNamespace(#db_namespace, schema_builder) - -} - diff --git a/sparta/simdb/include/simdb/schema/DatabaseTypedefs.hpp b/sparta/simdb/include/simdb/schema/DatabaseTypedefs.hpp deleted file mode 100644 index 4a27d1cb9b..0000000000 --- a/sparta/simdb/include/simdb/schema/DatabaseTypedefs.hpp +++ /dev/null @@ -1,11 +0,0 @@ -// -*- C++ -*- - -#pragma once - -namespace simdb { - -//! Unique database ID's for all records in a SimDB table. -typedef int DatabaseID; - -} - diff --git a/sparta/simdb/include/simdb/schema/GeneralMetaStructs.hpp b/sparta/simdb/include/simdb/schema/GeneralMetaStructs.hpp deleted file mode 100644 index 963e1b108a..0000000000 --- a/sparta/simdb/include/simdb/schema/GeneralMetaStructs.hpp +++ /dev/null @@ -1,80 +0,0 @@ -// -*- C++ -*- - -#pragma once - -/*! - * \brief Metaprogramming utilities for use - * with SimDB schemas. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace simdb { - -//! Base case: is_container is FALSE -template -struct is_container : std::false_type {}; - -//! Vectors -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Lists -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Forward lists -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Sets -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Unordered sets -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Deques -template -struct is_container> : std::true_type { - using value_type = ColumnT; -}; - -//! Base case: is_initializer_list is FALSE -template -struct is_initializer_list : std::false_type {}; - -//! Initializer lists for supported types: {"a","b"} / {4,5,3,6} / etc. -template -struct is_initializer_list> : std::true_type {}; - -//! Base case: is_contiguous is FALSE -template -struct is_contiguous : std::false_type {}; - -//! Vectors are the only supported contiguous types for blob retrieval -template -struct is_contiguous> : std::true_type { - using value_type = ColumnT; -}; - -} - diff --git a/sparta/simdb/include/simdb/schema/ObjectProxy.hpp b/sparta/simdb/include/simdb/schema/ObjectProxy.hpp deleted file mode 100644 index cb44f6ffe7..0000000000 --- a/sparta/simdb/include/simdb/schema/ObjectProxy.hpp +++ /dev/null @@ -1,99 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include - -namespace simdb { - -class ObjectProxy -{ -public: - - -protected: - ObjectProxy(const std::string & table_name, - const std::vector & column_names, - ObjectManager & obj_mgr) : - table_name_(table_name), - column_names_(column_names), - obj_mgr_(obj_mgr) - {} - -private: - std::function getter_char_{ - std::bind(ObjectProxy::getPropertyChar_, this)}; - - std::function getter_int8_{ - std::bind(ObjectProxy::getPropertyInt8_, this)}; - - std::function getter_uint8_{ - std::bind(ObjectProxy::getPropertyUInt8_, this)}; - - std::function getter_int16_{ - std::bind(ObjectProxy::getPropertyInt16_, this)}; - - std::function getter_uint16_{ - std::bind(ObjectProxy::getPropertyUInt16_, this)}; - - std::function getter_int32_{ - std::bind(ObjectProxy::getPropertyInt32_, this)}; - - std::function getter_uint32_{ - std::bind(ObjectProxy::getPropertyUInt32_, this)}; - - std::function getter_int64_{ - std::bind(ObjectProxy::getPropertyInt64_, this)}; - - std::function getter_uint64_{ - std::bind(ObjectProxy::getPropertyUInt64_, this)}; - - std::function getter_float_{ - std::bind(ObjectProxy::getPropertyFloat_, this)}; - - std::function getter_double_{ - std::bind(ObjectProxy::getPropertyDouble_, this)}; - - char getPropertyChar_(const std::string & col_name) const { - } - - int8_t getPropertyInt8_(const std::string & col_name) const { - } - - uint8_t getPropertyUInt8_(const std::string & col_name) const { - } - - int16_t getPropertyInt16_(const std::string & col_name) const { - } - - uint16_t getPropertyUInt16_(const std::string & col_name) const { - } - - int32_t getPropertyInt32_(const std::string & col_name) const { - } - - uint32_t getPropertyUInt32_(const std::string & col_name) const { - } - - int64_t getPropertyInt64_(const std::string & col_name) const { - } - - uint64_t getPropertyUInt64_(const std::string & col_name) const { - } - - float getPropertyFloat_(const std::string & col_name) const { - } - - double getPropertyDouble_(const std::string & col_name) const { - } - - std::string getPropertyString_(const std::string & col_name) const { - } - - std::string table_name_; - std::vector column_names_; - ObjectManager & obj_mgr_; - mutable std::shared_ptr realized_obj_; -}; - diff --git a/sparta/simdb/include/simdb/schema/Schema.hpp b/sparta/simdb/include/simdb/schema/Schema.hpp deleted file mode 100644 index 4ff5ab06a9..0000000000 --- a/sparta/simdb/include/simdb/schema/Schema.hpp +++ /dev/null @@ -1,1061 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/ColumnMetaStructs.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" -#include "simdb/schema/TableTypedefs.hpp" -#include "simdb/schema/TableSummaries.hpp" -#include "simdb/utils/Stringifiers.hpp" -#include "simdb/Errors.hpp" - -#include -#include -#include -#include -#include - -namespace simdb { - -//! This class is used to add columns to a fixed-size -//! schema Table (one whose columns are all fixed-size -//! POD's). It is intended to be used with the FOFFSET -//! macro below, for example: -//! -//! struct MyStruct { -//! int32_t x; -//! double y; -//! }; -//! -//! simdb::Schema schema; -//! schema.addTable("MyStruct") -//! .addColumn("x", dt::int32_t, FOFFSET(MyStruct,x)) -//! .addColumn("y", dt::double_t, FOFFSET(MyStruct,y)); -class FieldAdder -{ -public: - FieldAdder(const size_t byte_offset) : - byte_offset_(byte_offset) - {} - - FieldAdder(const FieldAdder &) = default; - FieldAdder & operator=(const FieldAdder &) = default; - -private: - size_t byte_offset_ = 0; - friend class Table; -}; - -//! This macro is used to create a FieldAdder object -//! that holds the byte offset to a field in a struct. -//! See the doxygen for FieldAdder for an example. -//! -//! The macro name stands for "Field Offset". -#define FOFFSET(s,f) simdb::FieldAdder(offsetof(s,f)) - -//! Compression enumeration specifying various levels -//! of compression which may be available in the SimDB -//! implementation. -enum class CompressionType : int8_t { - NONE, - DEFAULT_COMPRESSION, - BEST_COMPRESSION_RATIO, - BEST_COMPRESSION_SPEED -}; - -//! Column class used for creating SimDB tables -class Column -{ -public: - //! Construct a column with a name and one of the supported data types. - //! The column name must not be empty. - Column(const std::string & column_name, const ColumnDataType dt) : - name_(column_name), - dt_(dt) - { - if (name_.empty()) { - throw DBException( - "You cannot create a database column with no name"); - } - } - - //! Default copies, default assignments - Column(const Column &) = default; - Column & operator=(const Column &) = default; - - //! Equivalence against another Column - bool operator==(const Column & rhs) const { - if (getName() != rhs.getName()) { - return false; - } - if (getDataType() != rhs.getDataType()) { - return false; - } - return true; - } - - //! Equivalence against another Column - bool operator!=(const Column & rhs) const { - return !(*this == rhs); - } - - //! Name of this table column - const std::string & getName() const { - return name_; - } - - //! Data type of this table column - ColumnDataType getDataType() const { - return dt_; - } - - //! Dimensions of this table column - const std::vector & getDimensions() const { - return dims_; - } - - //! See if this column is indexed, either by itself or - //! indexed against other Columns too. - bool isIndexed() const { - return !indexed_properties_.empty(); - } - - //! If indexed, return the list of indexed Column names. - //! Returns an empty vector if this Column is not indexed. - const std::vector & getIndexedProperties() const { - return indexed_properties_; - } - - //! See if this Column has a default value set or not. - bool hasDefaultValue() const { - return !default_val_string_.empty(); - } - - //! Get this Column's default value. These are returned as - //! strings since the schema creation code is implementation- - //! specific (SQLite builds table creation statements one way, - //! HDF5 builds them another way, etc.) and default values - //! are only allowed on simple column data types (any type - //! that isn't a blob) so these strings can be lexical-casted - //! back to their native form if needed ("123" -> 123, etc.) - const std::string & getDefaultValueAsString() const { - return default_val_string_; - } - - //! Tables whose columns were populated using the addField() - //! method will know their byte offset in each of the table's - //! rows. If this column hasByteOffset(), it is part of a - //! table with fixed-size records. - bool hasByteOffset() const { - return byte_offset_.second; - } - - //! If hasByteOffset() returns true, ask for the byte offset - //! value with this method. Throws if hasByteOffset() returns - //! false. - size_t getByteOffset() const { - if (!hasByteOffset()) { - throw DBException("Cannot call Column::getByteOffset() - ") - << "check Column::hasByteOffset() beforehand"; - } - return byte_offset_.first; - } - - //! SimDB columns may or may not be able to be summarized, - //! but for those that do support column summary, they still - //! may have been explicitly removed from summarization: - //! - //! simdb::Schema schema; - //! schema.addTable("MyInts") - //! .addColumn("A", dt::int32_t) - //! .addColumn("B", dt::int32_t) - //! ->noSummary() - //! .addColumn("C", dt::int32_t); - //! - bool isSummaryDisabled() const { - return summary_disabled_; - } - -private: - // Columns that are added to their table object via - // the FOFFSET macro will be assigned their byte - // offset value through this constructor. - Column(const std::string & column_name, - const ColumnDataType dt, - const size_t byte_offset) : - Column(column_name, dt) - { - byte_offset_ = std::make_pair(byte_offset, true); - } - - // Optionally specify a default value for this Column. - // Defaults for Blob data types are not allowed and will - // throw if you attempt to set a Blob default value. - // The DefaultValueT template type needs to be able to - // be lexical-casted to a std::string, or this method - // will throw. - template - void setDefaultValue_(const DefaultValueT & val) { - if (dt_ == ColumnDataType::blob_t) { - throw DBException( - "Cannot set default value for a database " - "column with blob data type"); - } - std::ostringstream ss; - ss << val; - default_val_string_ = ss.str(); - if (default_val_string_.empty()) { - throw DBException("Unable to convert default value ") - << val << " into a std::string"; - } - } - - // You can tell the database to create indexes on specific - // table columns for faster queries later on. For example: - // - // using dt = simdb::ColumnDataType; - // simdb::Schema schema; - // - // schema.addTable("Customers") - // .addColumn("Last", dt::string_t) - // ->index(); - // - // This results in fast lookup performance for queries like this: - // - // SELECT * FROM Customers WHERE Last = 'Smith' - // - // If you want to build indexes based on multiple columns' - // values, pass in those other Column objects like this: - // - // schema.addTable("Customers") - // .addColumn("First", dt::string_t) - // .addColumn("Last", dt::string_t) - // ->indexAgainst("First"); - // - // This results in fast lookup performance for queries like this: - // - // SELECT * FROM Customers WHERE Last = 'Smith' AND Age > 40 - // - void setIsIndexed_(const std::vector & indexed_columns = {}) - { - indexed_properties_.clear(); - indexed_properties_.emplace_back(this); - - indexed_properties_.insert( - indexed_properties_.end(), - indexed_columns.begin(), - indexed_columns.end()); - - indexed_properties_.erase( - std::unique(indexed_properties_.begin(), indexed_properties_.end()), - indexed_properties_.end()); - } - - // Set dimensions for this column. Defaults to scalar - // dimensions of {1}. - void setDimensions_(const std::vector & dims) { - dims_ = dims; - } - - std::string name_; - ColumnDataType dt_; - std::vector dims_ = {1}; - std::string default_val_string_; - std::vector indexed_properties_; - std::pair byte_offset_ = std::make_pair(0, false); - bool summary_disabled_ = false; - - friend class Table; -}; - -//! Table class used for creating SimDB schemas -class Table -{ -public: - //! \brief Construct a table with a name. The table name - //! must not be empty. - //! - //! \param table_name Name of the database table - //! - //! \param compression Optionally request that the table - //! contents be compressed in the database. Note that the - //! specific implementation (DbConnProxy subclass) may not - //! support compression, in which case this option would be - //! ignored. - Table(const std::string & table_name, - const CompressionType compression = CompressionType::BEST_COMPRESSION_RATIO) : - name_(table_name), - compression_(compression) - { - if (name_.empty()) { - throw DBException( - "You cannot create a database table with no name"); - } - if (name_.find(std::string(1, Table::NS_DELIM)) != std::string::npos) { - throw DBException("Cannot call Table constructor with ") - << "a table name that includes the '" - << Table::NS_DELIM << "' character"; - } - } - - //! Construct without a name. - Table() = default; - - //! Default copies, default assignments - Table(const Table &) = default; - Table & operator=(const Table &) = default; - - //! Equivalence against another Table - bool operator==(const Table & rhs) const { - if (getName() != rhs.getName()) { - return false; - } - if (columns_.size() != rhs.columns_.size()) { - return false; - } - for (size_t idx = 0; idx < columns_.size(); ++idx) { - if (*columns_[idx] != *rhs.columns_[idx]) { - return false; - } - } - return true; - } - - //! Equivalence against another Table - bool operator!=(const Table & rhs) const { - return !(*this == rhs); - } - - //! Delimiter used to concatenate table names with - //! the database namespace each table is used in, - //! for instance: - //! - //! namespace: "stats" - //! table name: "Timeseries" - //! -> qualified table name: "stats$Timeseries" - //! - //! NS_DELIM stands for "namespace delimiter" - static constexpr char NS_DELIM = '$'; - - //! Get this Table's name - std::string getName() const { - if (name_prefix_.empty()) { - return name_; - } - return name_prefix_ + NS_DELIM + name_; - } - - //! \return Table's compression setting - CompressionType getCompression() const { - return compression_; - } - - //! See if this table is composed only of columns - //! of fixed size (POD's) - bool isFixedSize() const { - return is_fixed_size_; - } - - //! Add a Column to this Table. - Table & addColumn( - const std::string & name, - const ColumnDataType dt) - { - if (is_fixed_size_) { - is_fixed_size_ = getColumnIsFixedSize(dt); - } - columns_.emplace_back(new Column(name, dt)); - columns_by_name_[name] = columns_.back(); - column_modifier_.reset(); - return *this; - } - - //! If your table represents a contiguous struct of - //! fixed-size data fields, you can define the table - //! layout like this: - //! - //! simdb::Schema schema; - //! schema.addTable("MyStruct") - //! .addField("a", dt::int8_t, FOFFSET(s,a)) - //! .addField("b", dt::double_t, FOFFSET(s,b)) - //! .addField("c", dt::uint16_t, FOFFSET(s,c)); - //! - //! In the above example, this would equate to the - //! following C struct: - //! - //! struct MyStruct { - //! int8_t a; - //! double b; - //! uint16_t c; - //! }; - Table & addField( - const std::string & name, - const ColumnDataType dt, - const FieldAdder & adder) - { - if (!getColumnIsFixedSize(dt)) { - throw DBException("Cannot call Table::addField() for a column ") - << "whose data type is variable length"; - } - columns_.emplace_back(new Column(name, dt, adder.byte_offset_)); - columns_by_name_[name] = columns_.back(); - column_modifier_.reset(); - return *this; - } - - //! Iterator access - std::vector>::const_iterator begin() const { - return columns_.begin(); - } - - //! Iterator access - std::vector>::const_iterator end() const { - return columns_.end(); - } - - //! Ask if this Table has any Columns yet - bool hasColumns() const { - return !columns_.empty(); - } - - //! This class serves as a level of indirection which allows - //! the schema creation call site to look something like this: - //! - //! Schema schema; - //! using dt = simdb::ColumnDataType; - //! - //! schema.addTable("Customers") - //! .addColumn("First", dt::string_t) - //! ->index() - //! .addColumn("Last", dt::string_t) - //! ->indexAgainst("First") - //! .addColumn("RewardsBal", dt::double_t) - //! ->setDefaultValue(50); - //! - //! And so on. The index(), indexAgainst() and setDefaultValue() - //! calls will apply to the column that was just added on the - //! line of code above it in addColumn(). - class ColumnPropsModifier { - public: - Table & index() { - table_->setIndexedColumn_NO_THROW_(col_name_); - return *table_; - } - - Table & indexAgainst(const std::string & other_column) { - table_->setCompoundIndexedColumn_NO_THROW_(col_name_, {other_column}); - return *table_; - } - - Table & indexAgainst(const std::vector & other_columns) { - table_->setCompoundIndexedColumn_NO_THROW_(col_name_, other_columns); - return *table_; - } - - template - Table & setDefaultValue(const DefaultValueT & val) { - table_->setDefaultValue_(col_name_, val); - return *table_; - } - - Table & setDimensions(const std::vector & dims) { - table_->setDimensions_(col_name_, dims); - return *table_; - } - - Table & noSummary() { - table_->disableSummary_(col_name_); - return *table_; - } - - private: - ColumnPropsModifier(Table * tbl, const std::string & col_name) : - table_(tbl), - col_name_(col_name) - {} - - Table * table_ = nullptr; - std::string col_name_; - friend class Table; - }; - - //! Overloaded operator->() so we know when to switch - //! into "column modification" mode during a call that - //! looks like this: - //! - //! schema.addTable("Customers") - //! .addColumn("FirstName", dt::string_t) - //! .addColumn("LastName", dt::string_t) - //! ->indexAgainst("FirstName") - //! .addColumn(...) - //! ... ... - //! ........................................... - //! - //! The call to '->indexAgainst("First")' actually means: - //! "Go back to the previous column 'LastName', and create a - //! compound index for it together with the 'FirstName' column." - //! - //! The subsequent call to addColumn() will tip off the - //! Table object that column modification mode is done, - //! at least until operator->() gets called again. - //! - //! These modification APIs can be strung together too, like this: - //! - //! schema.addTable("RewardsAccounts") - //! .addColumn("FirstName", dt::string_t) - //! .addColumn("LastName", dt::string_t) - //! .addColumn("Amount", dt::double_t) - //! ->indexAgainst("LastName") - //! ->setDefaultValue(100) - //! .addColumn("DaysToExpiration", dt::int32_t) - //! ->setDefaultValue(365); - //! - ColumnPropsModifier * operator->() { - if (columns_.empty()) { - throw DBException("Invalid use of the schema creation utility. ") - << "An attempt was made to modify a table's column indexing or " - << "default values, but the table does not have any columns to " - << "modify. The offending table was '" << getName() << "'."; - } - - column_modifier_.reset(new ColumnPropsModifier( - this, columns_.back()->getName())); - - return column_modifier_.get(); - } - -private: - // Optionally specify a default value for the given Column - template - void setDefaultValue_(const std::string & column_name, - const DefaultValueT & val) - { - auto iter = columns_by_name_.find(column_name); - if (iter == columns_by_name_.end()) { - throw DBException("Table::setDefaultValue_() called with ") - << "a column name that does not exist: '" - << column_name << "'"; - } - iter->second->setDefaultValue_(val); - } - - // Set the column by the given name to be indexed - void setIndexedColumn_(const std::string & column_name) { - auto iter = columns_by_name_.find(column_name); - if (iter == columns_by_name_.end()) { - throw DBException("Table::setIndexedColumn_() called with ") - << "a column name that does not exist: '" - << column_name << "'"; - } - iter->second->setIsIndexed_(); - } - - // Set the column by the given name to be indexed - // together with one or more other columns. This - // is used to enable fast queries that look like - // this: - // - // SELECT * FROM Customers WHERE Last='Smith' AND Age>50 - void setCompoundIndexedColumn_( - const std::string & primary_column_name, - const std::vector & other_indexed_column_names) - { - auto primary_iter = columns_by_name_.find(primary_column_name); - if (primary_iter == columns_by_name_.end()) { - throw DBException("Column '") << primary_column_name - << "' does not exist in table '" << getName() << "'"; - } - - std::vector other_indexed_columns; - for (const auto & other_col_name : other_indexed_column_names) { - auto other_iter = columns_by_name_.find(other_col_name); - if (other_iter == columns_by_name_.end()) { - throw DBException("Column '") << primary_column_name - << "' does not exist in table '" << getName() << "'"; - } - other_indexed_columns.emplace_back(other_iter->second.get()); - } - - primary_iter->second->setIsIndexed_(other_indexed_columns); - } - - // Column dimensionality defaults to scalar. Some database - // implementations such as SQLite do not support non-scalar - // data types like double{3,4} but others such as HDF5 do - // support N-D columns. Set the dimensions with this API, - // or use the Schema object like so: - // - // Schema schema; - // using dt = simdb::ColumnDataType; - // - // schema.addTable("Numbers") - // .addColumn("MyMatrix", dt::double_t) - // ->setDimensions({3,4}); - // - void setDimensions_( - const std::string & column_name, - const std::vector & dims) - { - auto column_iter = columns_by_name_.find(column_name); - if (column_iter == columns_by_name_.end()) { - throw DBException("Column '") << column_name - << "' does not exist in table '" << getName() << "'"; - } - - column_iter->second->setDimensions_(dims); - } - - // The ColumnPropsModifier class may need to inform us if a schema - // call site looks like this: - // - // Schema schema; - // using dt = simdb::ColumnDataType; - // - // schema.addTable("Customers") - // .addColumn("LastName", dt::string_t) - // ->indexAgainst("FirstName") - // .addColumn("FirstName", dt::string_t); - // - // The indexAgainst("FirstName") call would be made before the - // column "FirstName" was even added to the table. It would not - // be very user-friendly to throw an exception and enforce the - // addColumn() calls be ordered in a specific way. - // - // Aside from user-unfriendly behavior, it would also prevent - // indexes like the following completely: - // - // schema.addTable("Sales") - // .addColumn("Amount", dt::double_t) - // ->indexAgainst("LastName") - // .addColumn("LastName", dt::string_t) - // ->indexAgainst("FirstName") - // .addColumn("FirstName", dt::string_t) - // ->indexAgainst("Amount"); - // - void setIndexedColumn_NO_THROW_(const std::string & column_name) - { - if (columns_by_name_.find(column_name) == columns_by_name_.end()) { - unresolved_column_idxs_[column_name] = {}; - return; - } else { - setIndexedColumn_(column_name); - } - } - - // No-throw API for setting a compound index. Called by ColumnPropsModifier. - void setCompoundIndexedColumn_NO_THROW_( - const std::string & primary_column_name, - const std::vector & other_indexed_column_names) - { - // Note that we only allow the indexed *against* columns to - // be unresolved, not the primary column. This is valid: - // - // using dt = simdb::ColumnDataType; - // - // schema.addTable("Customers") - // .addColumn("LastName", dt::string_t) - // ->indexAgainst({"FirstName", "AccountActive"} - // .addColumn("FirstName", dt::string_t) - // .addColumn("AccountActive", dt::int32_t); - // - // Here, the primary column name would be "LastName", which the - // Table object (this) explicitly gave to the ColumnPropsModifier - // ahead of time when Table::operator->() was called. - // - // There is no way ColumnPropsModifier would pass in a primary - // column name that the Table did not already have, unless it - // ignored the column name we just gave it, and turned around - // and gave us another column name instead. - // - // The *secondary* columns in the compound index are allowed to - // be unresolved until the Schema is given to an ObjectManager. - // In the above example, "FirstName" and "AccountActive" are the - // secondary columns in the index. - // - // SELECT * FROM Customers WHERE - // LastName='Smith' AND FirstName='Bob' AND AccountActive=1 - // - // ---------------- ----------------------------------- - // (primary) (secondary) - // - bool all_resolved = true; - for (const auto & secondary_col_name : other_indexed_column_names) { - if (columns_by_name_.find(secondary_col_name) == columns_by_name_.end()) { - all_resolved = false; - break; - } - } - - if (!all_resolved) { - unresolved_column_idxs_[primary_column_name] = other_indexed_column_names; - return; - } else { - setCompoundIndexedColumn_( - primary_column_name, other_indexed_column_names); - } - } - - // This method gets called during schema construction - // for columns that are explicitly removed from table - // summarization: - // - // simdb::Schema schema; - // schema.addTable("MyInts") - // .addColumn("A", dt::int32_t) - // .addColumn("B", dt::int32_t) - // ->noSummary() - // .addColumn("C", dt::int32_t); - // - void disableSummary_(const std::string & column_name) { - auto iter = columns_by_name_.find(column_name); - if (iter != columns_by_name_.end()) { - iter->second->summary_disabled_ = true; - } - } - - // Called by the Schema object this Table belongs to when - // the Schema is given to an ObjectManager for database - // instantiation. Schema is a friend of this class to - // make this private call. - void finalizeTable_() { - std::unordered_map> resolved_column_idxs; - - for (const auto & unresolved : unresolved_column_idxs_) { - auto primary_column_iter = columns_by_name_.find(unresolved.first); - - if (primary_column_iter == columns_by_name_.end()) { - throw DBException("Unrecognized column '") << - unresolved.first << "' encountered in the " << - "SimDB table '" << getName() << "'."; - } - - Column * primary_column_obj = primary_column_iter->second.get(); - resolved_column_idxs[primary_column_obj] = {}; - - for (const auto & unresolved_secondary : unresolved.second) { - auto secondary_column_iter = columns_by_name_.find( - unresolved_secondary); - - if (secondary_column_iter == columns_by_name_.end()) { - throw DBException("Unrecognized column '") << - unresolved_secondary << "' encountered in the " << - "SimDB table '" << getName() << "'."; - } - - resolved_column_idxs[primary_column_obj].emplace_back( - secondary_column_iter->second.get()); - } - } - - // If we got this far, the resolved column indexes map has - // all the objects we need to finalize the schema. The keys - // in this map are the primary columns, and the values they - // point to are the secondary columns that go with it to - // make the compound index. - for (auto & resolved : resolved_column_idxs) { - resolved.first->setIsIndexed_(resolved.second); - } - unresolved_column_idxs_.clear(); - } - - bool is_fixed_size_ = true; - std::string name_; - std::string name_prefix_; - CompressionType compression_ = CompressionType::BEST_COMPRESSION_RATIO; - std::vector> columns_; - std::shared_ptr column_modifier_; - std::unordered_map> columns_by_name_; - std::unordered_map> unresolved_column_idxs_; - - friend class ColumnPropsModifier; - friend class Schema; -}; - -//! Schema class used for creating SimDB databases -class Schema -{ -public: - Schema() = default; - - //! \brief Create a new Table in this Schema with the given name - //! - //! \param table_name Name of the database table - //! - //! \param compression Table-specific compression setting. - //! This setting will be ignored if the DbConnProxy subclass - //! being used does not support compression. - //! - //! \warning Throws if the provided table name contains the - //! character '$' - this is a reserved delimiter. - //! - //! \return Reference to the added table - Table & addTable( - const std::string & table_name, - const CompressionType compression = CompressionType::BEST_COMPRESSION_RATIO) - { - if (table_name.find(std::string(1, Table::NS_DELIM)) != std::string::npos) { - throw DBException("Cannot call Schema::addTable() ") - << "with a table name that includes the '" - << Table::NS_DELIM << "' character"; - } - tables_.emplace_back(table_name, compression); - if (!pending_namespace_.empty()) { - tables_.back().name_prefix_ = pending_namespace_; - } - return tables_.back(); - } - - //! \brief Create a new Table in this Schema, copied from the - //! Table object passed in. - //! - //! \param rhs Source table object intended to be added to the schema - //! - //! \warning Throws if the provided table name contains the - //! character '$' - this is a reserved delimiter. - //! - //! \return Returns a reference to the newly created table, or - //! returns a reference to an existing table in this schema that - //! matched the incoming table (same table name, same colum names, - //! and the same column data types). - Table & addTable(const Table & rhs) { - if (Table * existing_table = getTableNamed(rhs.getName())) { - if (*existing_table != rhs) { - throw DBException("Cannot add table '") - << rhs.getName() << "' to schema. A table " - << "with that name already exists."; - } - return *existing_table; - } - - tables_.emplace_back(rhs); - if (!pending_namespace_.empty()) { - tables_.back().name_prefix_ = pending_namespace_; - } - return tables_.back(); - } - - /*! - * \brief Combine this schema with the tables from another - * schema. Any clashes between the new table(s) and the - * existing table(s) will throw an exception: - * - * 1) Added schema has table called "Customers". The - * existing schema already has a table by the same - * name. However, the column configuration for both - * tables is identical. Will NOT throw. The table - * would be ignored. - * - * 2) Added schema has table called "Customers". The - * existing schema already has a table by the same - * name. The column configurations of the two tables - * are different. WILL throw. Columns are considered - * different if they have a different name (case- - * sensitive) and/or a different ColumnDataType. - */ - Schema & operator+=(const Schema & rhs) { - for (const auto & table : rhs) { - addTable(table); - } - return *this; - } - - /*! - * \brief Get a pointer to the schema table with the - * given name - * - * \param table_name Name of the table in this schema - * you want to access - * - * \return Pointer to the table with the given name. - * Returns null if no table by that name exists. - */ - Table * getTableNamed(const std::string & table_name) { - for (auto & lhs : tables_) { - if (lhs.getName() == table_name) { - return &lhs; - } - } - return nullptr; - } - - /*! - * \brief Get a pointer to the schema table with the - * given name - * - * \param table_name Name of the table in this schema - * you want to access - * - * \return Pointer to the table with the given name. - * Returns null if no table by that name exists. - */ - const Table * getTableNamed(const std::string & table_name) const { - for (auto & lhs : tables_) { - if (lhs.getName() == table_name) { - return &lhs; - } - } - return nullptr; - } - - //! Iterator access - std::deque::const_iterator begin() const { - return tables_.begin(); - } - - //! Iterator access - std::deque
::const_iterator end() const { - return tables_.end(); - } - - //! Ask if this Schema has any Tables yet - bool hasTables() const { - return !tables_.empty(); - } - - //! Equivalence check - bool operator==(const Schema & rhs) const { - if (tables_.size() != rhs.tables_.size()) { - return false; - } - for (size_t idx = 0; idx < tables_.size(); ++idx) { - if (tables_[idx] != rhs.tables_[idx]) { - return false; - } - } - return true; - } - - //! Equivalence check - bool operator!=(const Schema & rhs) const { - return !(*this == rhs); - } - - //! Give this schema a TableSummaries object it can - //! use to enable the TableRef::captureSummary() method - //! for the finalized database. This method should be - //! called prior to giving this schema to an ObjectManager - //! via the createDatabaseFromSchema() method. - void setTableSummaryConfig(TableSummaries && summary_config) { - summary_config_ = std::move(summary_config); - } - -private: - std::deque
tables_; - std::string pending_namespace_; - - //Prepend a namespace to this schema. This is called by - //friend classes DatabaseRoot and DatabaseNamespace. - void setNamespace_(const std::string & namespace_name) { - if (tables_.empty()) { - pending_namespace_ = namespace_name; - } else { - for (auto & tbl : tables_) { - tbl.name_prefix_ = namespace_name; - } - } - } - TableSummaries summary_config_; - - struct TableSummaryQueryInfo { - struct SourceTableInfo { - std::string table_name; - std::vector table_columns; - }; - - std::vector source_tables; - NamedSummaryFunctions summary_fcns; - }; - TableSummaryQueryInfo summary_query_info_structs_; - - void getSummarizeableColumnsForTable_( - std::vector & summarizeable_cols, - const Table & table) const - { - for (const auto & col : table) { - switch (col->getDataType()) { - case ColumnDataType::fkey_t: - case ColumnDataType::string_t: - case ColumnDataType::blob_t: { - continue; - } - default: { - if (!col->isSummaryDisabled()) { - summarizeable_cols.emplace_back( - col->getName(), col->getDataType()); - } - } - } - } - } - - //When this Schema is given to an ObjectManager, it will call back - //into this method to give us a chance to finalize the Schema and - //throw any exceptions if we need to. - void finalizeSchema_() { - std::vector //Table columns - >> summarizeable_columns_by_table; - - const NamedSummaryFunctions * summary_fcns = nullptr; - if (!summary_config_.named_summary_fcns_.empty()) { - summary_fcns = &summary_config_.named_summary_fcns_; - } - - for (auto & tbl : tables_) { - if (!pending_namespace_.empty()) { - tbl.name_prefix_ = pending_namespace_; - } - tbl.finalizeTable_(); - - if (summary_fcns) { - std::vector summarizeable_columns; - summarizeable_columns_by_table.emplace_back( - std::make_pair(tbl.getName(), summarizeable_columns)); - - getSummarizeableColumnsForTable_( - summarizeable_columns_by_table.back().second, tbl); - - if (summarizeable_columns_by_table.back().second.empty()) { - summarizeable_columns_by_table.pop_back(); - } - } - } - - for (const auto & summarizeable : summarizeable_columns_by_table) { - Table summary_table(summarizeable.first + "_Summary"); - for (const auto & summarizeable_column : summarizeable.second) { - for (const auto & summary_fcn : *summary_fcns) { - summary_table.addColumn( - summarizeable_column.first + "_" + summary_fcn.first, - ColumnDataType::double_t); - } - } - - if (summary_table.hasColumns()) { - tables_.emplace_back(summary_table); - TableSummaryQueryInfo::SourceTableInfo source_table; - source_table.table_name = summarizeable.first; - source_table.table_columns = summarizeable.second; - summary_query_info_structs_.source_tables.emplace_back(source_table); - } - } - - if (!summary_query_info_structs_.source_tables.empty()) { - summary_query_info_structs_.summary_fcns = *summary_fcns; - } - pending_namespace_.clear(); - } - - bool shouldSummarizeTable_(const std::string & table_name) const { - return summary_config_.excluded_tables_.find(table_name) == - summary_config_.excluded_tables_.end(); - } - - friend class ObjectManager; - friend class DatabaseRoot; - friend class DatabaseNamespace; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/schema/TableSummaries.hpp b/sparta/simdb/include/simdb/schema/TableSummaries.hpp deleted file mode 100644 index 97580b5895..0000000000 --- a/sparta/simdb/include/simdb/schema/TableSummaries.hpp +++ /dev/null @@ -1,83 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/Schema.hpp" -#include "simdb/schema/TableTypedefs.hpp" - -#include - -namespace simdb { - -/*! - * \class TableSummaries - * \brief This utility lets SimDB users define named aggregation - * methods to summarize table columns. TableRef and ObjectManager - * have APIs that let you trigger a table summary, which invokes - * all registered aggregation methods one by one. Things like min, - * max, avg, etc. can be captured easily this way. - */ -class TableSummaries -{ -public: - TableSummaries & define(const std::string & algo_name, - const SummaryFunction & algo_impl) - { - named_summary_fcns_[algo_name] = algo_impl; - return *this; - } - - const NamedSummaryFunctions & getSummaryAlgos() const { - return named_summary_fcns_; - } - - template - typename std::enable_if< - std::is_same::value, - TableSummaries &>::type - excludeTables(const T & table_name) { - excluded_tables_.insert(table_name); - return *this; - } - - template - typename std::enable_if< - std::is_same::type, const char*>::value, - TableSummaries &>::type - excludeTables(T table_name) { - excluded_tables_.insert(table_name); - return *this; - } - - template - typename std::enable_if< - std::is_same::value, - TableSummaries &>::type - excludeTables(const T & table_name, - Args &&... args) { - excludeTables(table_name); - excludeTables(std::forward(args)...); - return *this; - } - - template - typename std::enable_if< - std::is_same::type, const char*>::value, - TableSummaries &>::type - excludeTables(T table_name, - Args &&... args) { - excludeTables(table_name); - excludeTables(std::forward(args)...); - return *this; - } - -private: - NamedSummaryFunctions named_summary_fcns_; - std::unordered_set excluded_tables_; - - friend class DbConnProxy; - friend class Schema; -}; - -} - diff --git a/sparta/simdb/include/simdb/schema/TableTypedefs.hpp b/sparta/simdb/include/simdb/schema/TableTypedefs.hpp deleted file mode 100644 index 2ecdf6bd48..0000000000 --- a/sparta/simdb/include/simdb/schema/TableTypedefs.hpp +++ /dev/null @@ -1,92 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb_fwd.hpp" - -#include -#include -#include -#include - -namespace simdb { - -//! Table summary functions are applied to the table's columns -//! that support summarization. The input arguments to this -//! function are as follows: -//! -//! std::vector vals -> All of the column's record -//! values that have not yet -//! been summarized. -//! -//! ObjectProxy * current_summary -> Existing record holding all -//! of the summarized table values -//! for this column. -//! -//! To illustrate the ObjectProxy input argument, say we wanted -//! to capture the average value of all records in a table for -//! each column in that table. One way to do this would be be -//! to gather all records in the table since the start of the -//! program, and give all of them to the registered averaging -//! function. But this gets slower each time the TableRef's -//! captureSummary() method is called, since the table itself -//! is presumably growing over time. -//! -//! Here is another way which lets the averaging function make -//! use of previously-summarized records: -//! -//! double calcAverage(const std::vector & new_vals, -//! const ObjectProxy * existing_summary) -//! { -//! double avg = NAN; -//! if (new_vals.empty()) { -//! if (existing_summary == nullptr) { -//! return avg; -//! } -//! avg = existing_summary->getPropertyDouble("avg"); -//! } else { -//! double total_avg_numerator = std::accumulate( -//! new_vals.begin(), new_vals.end(), 0, std::plus()); -//! size_t total_avg_denominator = new_vals.size(); -//! if (existing_summary) { -//! total_avg_numerator += existing_summary->getPropertyDouble("avg"); -//! total_avg_denominator += existing_summary->getPropertyDouble("count"); -//! } -//! avg = (total_avg_numerator / total_avg_denominator); -//! } -//! return avg; -//! } -//! -//! -//! This could be registered with the schema's TableSummaries -//! object like so: -//! -//! simdb::TableSummaries summary; -//! summary.define("avg", &calcAverage) -//! .define("count", [](const std::vector & vals, -//! const ObjectProxy * cur_summary) { -//! double count = vals.size(); -//! if (cur_summary) { -//! count += cur_summary->getPropertyDouble("count"); -//! } -//! return count; -//! }); -//! -using SummaryFunction = std::function & vals)>; - -//! Map of table summary calculation functions by summary -//! function name. For instance: -//! -//! {{ "min", [](const std::vector & vals) { -//! return vals.empty() ? NAN : -//! *std::min_element(vals.begin(), vals.end()); -//! }, -//! { "max", [](const std::vector & vals) { -//! return vals.empty() ? NAN : -//! *std::max_element(vals.begin(), vals.end()); -//! }} -//! -using NamedSummaryFunctions = std::map; - -} - diff --git a/sparta/simdb/include/simdb/utils/BlobHelpers.hpp b/sparta/simdb/include/simdb/utils/BlobHelpers.hpp deleted file mode 100644 index 962d9c1378..0000000000 --- a/sparta/simdb/include/simdb/utils/BlobHelpers.hpp +++ /dev/null @@ -1,84 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include - -/*! - * \file BlobHelpers.h - * \brief This file contains utilities related to working - * with SimDB blobs, which are retrieved from the database - * as std::vector - */ - -namespace simdb { - -//! Lightweight "alias" to reinterpret_cast a std::vector -//! to a std::vector. This does not copy any underlying data. -template -struct VectorAlias { - //! Construct with a reference to your raw char vector - explicit VectorAlias(const std::vector & src_data) : - src_(src_data) - { - static_assert( - std::is_scalar::value and - std::is_arithmetic::value, - "simdb::VectorAlias utility is only supported to 'cast' a " - "vector to a vector of numeric scalars. \n" - "Examples (assume 'raw' is a vector): \n" - " simdb::VectorAlias alias(raw); \n" - " simdb::VectorAlias alias2(raw); \n"); - - static_assert( - sizeof(*this) == sizeof(void*), - "simdb::VectorAlias should have the size of a void pointer"); - } - - //! Comparison with std::vector - bool operator==(const std::vector & rhs) const { - const auto & lhs = *this; - if (lhs.size() != rhs.size()) { - return false; - } - - for (size_t idx = 0; idx < rhs.size(); ++idx) { - if (lhs[idx] != rhs[idx]) { - return false; - } - } - return true; - } - - //! Return the number of points in the "aliased" vector. Say - //! the original vector had 40 elements in it and this - //! alias is for vector - this size() method would - //! return 5. There would be five elements if this same raw - //! vector were interpreted as a vector of doubles. - size_t size() const { - return src_.size() / sizeof(DestinationT); - } - - //! Same as vector::empty() for any type T - bool empty() const { - return src_.empty(); - } - - //! Data access without bounds checking - const DestinationT & operator[](const size_t idx) const { - return *reinterpret_cast( - &src_[idx*sizeof(DestinationT)]); - } - - //! Data access with bounds checking - const DestinationT & at(const size_t idx) const { - return *reinterpret_cast( - &src_.at(idx*sizeof(DestinationT))); - } - -private: - const std::vector & src_; -}; - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/utils/CompatUtils.hpp b/sparta/simdb/include/simdb/utils/CompatUtils.hpp deleted file mode 100644 index ec7bfc721b..0000000000 --- a/sparta/simdb/include/simdb/utils/CompatUtils.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include - -namespace simdb { -namespace utils { - -//! \brief Replacement for std::is_pod, which was deprecated in C++20 - -template -struct is_pod { - static constexpr bool value = std::is_trivial::value - && std::is_standard_layout::value; -}; - -} // namespace utils -} // namespace simdb diff --git a/sparta/simdb/include/simdb/utils/MathUtils.hpp b/sparta/simdb/include/simdb/utils/MathUtils.hpp deleted file mode 100644 index fd64fe1c78..0000000000 --- a/sparta/simdb/include/simdb/utils/MathUtils.hpp +++ /dev/null @@ -1,62 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include - -namespace simdb { -namespace utils { - -//! \brief Comparison of two floating-point values with -//! a supplied tolerance. The tolerance value defaults -//! to machine epsilon. -template -typename std::enable_if< - std::is_floating_point::value, -bool>::type -approximatelyEqual(const T a, const T b, - const T epsilon = std::numeric_limits::epsilon()) -{ - const T fabs_a = std::fabs(a); - const T fabs_b = std::fabs(b); - const T fabs_diff = std::fabs(a - b); - - return fabs_diff <= ((fabs_a < fabs_b ? fabs_b : fabs_a) * epsilon); -} - -//! Static/global random number generator -struct RandNumGen { - static std::mt19937 & get() { - static std::mt19937 rng(time(nullptr)); - return rng; - } -}; - -//! \brief Pick a random integral number -template -typename std::enable_if< - std::is_integral::value, -T>::type -chooseRand() -{ - std::uniform_int_distribution dist; - return dist(RandNumGen::get()); -} - -//! \brief Pick a random floating-point number -template -typename std::enable_if< - std::is_floating_point::value, -T>::type -chooseRand() -{ - std::normal_distribution dist(0, 1000); - return dist(RandNumGen::get()); -} - -} // namespace utils -} // namespace simdb diff --git a/sparta/simdb/include/simdb/utils/ObjectQuery.hpp b/sparta/simdb/include/simdb/utils/ObjectQuery.hpp deleted file mode 100644 index bb8fba1563..0000000000 --- a/sparta/simdb/include/simdb/utils/ObjectQuery.hpp +++ /dev/null @@ -1,807 +0,0 @@ -// -*- C++ -*- - -#pragma once - -/*! - * \file ObjectQuery.h - * \brief This utility makes it easy to put together - * database queries with or without constraints (WHERE - * clauses), without explicitly writing SQL commands. - * It also makes it easy to iterate over many records - * that match your constraints without having to bring - * them all into memory at once. - */ - -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/utils/Stringifiers.hpp" - -#include -#include -#include -#include -#include -#include - -//SQLite-specific headers -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include - -namespace simdb { - -//! ORDER BY ASC|DESC -enum class ColumnOrdering : int8_t { - asc, - desc, - default_ordering -}; - -constexpr ColumnOrdering ASC = ColumnOrdering::asc; -constexpr ColumnOrdering DESC = ColumnOrdering::desc; - -inline std::ostream & operator<<(std::ostream & os, - const ColumnOrdering order) -{ - switch (order) { - case ColumnOrdering::asc: { - os << "ASC"; - break; - } - case ColumnOrdering::desc: { - os << "DESC"; - break; - } - case ColumnOrdering::default_ordering: { - simdb_throw("not handled") - } - } - return os; -} - -//! ORDER BY clause -class OrderBy { -public: - OrderBy(const char * column_name, - const ColumnOrdering column_order = ColumnOrdering::default_ordering) : - col_name_(column_name), - col_ordering_(column_order) - {} - - OrderBy(const std::string & column_name, - const ColumnOrdering column_order = ColumnOrdering::default_ordering) : - col_name_(column_name), - col_ordering_(column_order) - {} - - OrderBy() = default; - OrderBy(const OrderBy &) = default; - - OrderBy& operator=(const OrderBy&) = default; - - operator std::string() const { - std::ostringstream oss; - oss << (*this); - return oss.str(); - } - -private: - std::string col_name_; - ColumnOrdering col_ordering_ = ColumnOrdering::default_ordering; - friend std::ostream & operator<<(std::ostream &, const OrderBy); -}; - -inline std::ostream & operator<<(std::ostream & os, const OrderBy order_by) -{ - if (order_by.col_name_.empty()) { - return os; - } - - os << " ORDER BY " - << order_by.col_name_ << " " - << order_by.col_ordering_ << " "; - - return os; -} - -/*! - * \brief This class is used together with ObjectQuery to be - * able to execute an SQL query once, and then simply iterate - * over the result set by calling the getNext() API in this class. - */ -class ResultIter -{ -public: - //! Advance this iterator to the next record in its underlying - //! set of records. Returns true on success, false otherwise. - //! If false, the SQL code will be written to the output variable - //! 'sqlite_return_code', if provided. See 'sqlite3.h' for more - //! details about what the specific error codes mean. This method - //! will typically return false only when there are no more records - //! to iterate over. - bool getNext(int * sqlite_return_code = nullptr) - { - //Step our prepared statement forward - const int step_result = sqlite3_step(prepared_stmt_); - - //SQLite returns SQLITE_ROW when we have another record - //that matches the original query. - if (step_result != SQLITE_ROW) { - //We are either out of matching records, or something - //else happened (Someone else has put a lock on our - //database? This may need more investigation. For now, - //we'll just return false and halt further iterations - //for this query). - if (sqlite_return_code) { - *sqlite_return_code = step_result; - } - sqlite3_reset(prepared_stmt_); - return false; - } - - //Go out and memcpy the current matching record's column - //values into the user's variables. - writeCurrentResultToPtrs_(); - - //In the event of a successful iterator advance, give - //the SQLITE_ROW result code to the caller, if they - //have asked for it. - if (sqlite_return_code) { - *sqlite_return_code = step_result; - } - - return true; - } - - ~ResultIter() - { - sqlite3_finalize(prepared_stmt_); - } - -private: - //! Pair of the void* and the data type of a variable we were - //! given to write into during each call to getNext() - typedef std::pair ResultColumn; - - //! Mapping of variable name to its ResultColumn - typedef std::map NamedResultColumns; - - //! Private constructor only ObjectQuery has access to - ResultIter(const NamedResultColumns & result_cols, - sqlite3_stmt * stmt) : - dest_ptrs_(result_cols), - prepared_stmt_(stmt) - { - assert(prepared_stmt_); - if (dest_ptrs_.empty()) { - std::cout << "simdb::ResultIter object created without " - "result iteration pointers. This is probably " - "a mistake. The query can be stepped forward " - "with calls to getNext(), but nobody is listening " - "for those result data values." << std::endl; - } - } - - //! During each call to getNext(), write the column values into - //! the variables that were given to ObjectQuery by the user/caller. - void writeCurrentResultToPtrs_() - { - auto iter = dest_ptrs_.begin(); - for (size_t col_idx = 0; col_idx < dest_ptrs_.size(); ++col_idx) { - void * dest = iter->second.first; - - //Now we have the void* of the variable that is supposed to - //be written into during each getNext() iteration. How many - //bytes and/or the method we use to populate that void* - //only depends on the data type of the column in question. - switch (iter->second.second) { - case ColumnDataType::fkey_t: - case ColumnDataType::char_t: - case ColumnDataType::int8_t: - case ColumnDataType::uint8_t: - case ColumnDataType::int16_t: - case ColumnDataType::uint16_t: - case ColumnDataType::int32_t: - case ColumnDataType::uint32_t: { - const int val = sqlite3_column_int(prepared_stmt_, static_cast(col_idx)); - if (!memcpyInteger32_(val, dest, iter->second.second)) { - throw DBException("Unable to convert integer value ") - << "to the requested type"; - } - break; - } - - case ColumnDataType::int64_t: - case ColumnDataType::uint64_t: { - const sqlite3_int64 val = sqlite3_column_int64(prepared_stmt_, static_cast(col_idx)); - if (!memcpyInteger64_(val, dest, iter->second.second)) { - throw DBException("Unable to convert integer value ") - << "to the requested type"; - } - break; - } - - case ColumnDataType::float_t: - case ColumnDataType::double_t: { - const double val = sqlite3_column_double(prepared_stmt_, static_cast(col_idx)); - if (!memcpyFloatingPoint_(val, dest, iter->second.second)) { - throw DBException("Unable to convert integer value ") - << "to the requested type"; - } - break; - } - - case ColumnDataType::string_t: { - std::string tmp = std::string(reinterpret_cast( - sqlite3_column_text(prepared_stmt_, static_cast(col_idx)))); - std::swap(tmp, *static_cast(dest)); - break; - } - - case ColumnDataType::blob_t: { - const void * blob_ptr = sqlite3_column_blob(prepared_stmt_, static_cast(col_idx)); - const size_t num_bytes = sqlite3_column_bytes(prepared_stmt_, static_cast(col_idx)); - std::vector & dest_vector = *(static_cast*>(dest)); - dest_vector.resize(num_bytes); - memcpy(&dest_vector[0], blob_ptr, num_bytes); - break; - } - - default: - throw DBException("Unrecognized column data type encountered"); - } - ++iter; - } - } - - //! When iterating over ObjectQuery result sets, we need - //! to copy an integer value that SQLite gave us from the - //! database into the variable given at the call site: - //! - //! ObjectQuery query(...); - //! - //! int8_t x; - //! uint16_t y; - //! query.writeResultIterationsTo("MyInt8", &x, "MyUint16", &y); - //! ... - //! - //! This method performs the memcpy into variables like - //! x and y above. - bool memcpyInteger32_( - const int val, void * dest, - const ColumnDataType dest_type) const - { - using dt = ColumnDataType; - - switch (dest_type) { - case dt::int8_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(int8_t)); - break; - } - - case dt::uint8_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(uint8_t)); - break; - } - - case dt::int16_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(int16_t)); - break; - } - - case dt::uint16_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(uint16_t)); - break; - } - - case dt::int32_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(int32_t)); - break; - } - - case dt::uint32_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(uint32_t)); - break; - } - - case dt::char_t: { - const auto casted = static_cast(val); - memcpy(dest, &casted, sizeof(char)); - break; - } - - default: - return false; - } - - return true; - } - - //! Similar to memcpyInteger32, but this is for 64-bit - //! integer data types. - bool memcpyInteger64_( - const int64_t val, void * dest, - const ColumnDataType dest_type) const - { - using dt = ColumnDataType; - - if (dest_type == dt::int64_t || dest_type == dt::uint64_t) { - memcpy(dest, &val, sizeof(int64_t)); - return true; - } - return false; - } - - //! Similar to memcpyInteger32, but this is for floats - //! and doubles. - bool memcpyFloatingPoint_( - const double val, void * dest, - const ColumnDataType dest_type) const - { - using dt = ColumnDataType; - - if (dest_type == dt::float_t) { - const float casted = static_cast(val); - memcpy(dest, &casted, sizeof(float)); - return true; - } else if (dest_type == dt::double_t) { - memcpy(dest, &val, sizeof(double)); - return true; - } - return false; - } - - NamedResultColumns dest_ptrs_; - sqlite3_stmt * prepared_stmt_ = nullptr; - - //This class goes hand in hand with ObjectQuery. - //Nobody else can create one of these. - friend class ObjectQuery; -}; - -/* - * \brief This class lets you put together SELECT statements - * without having to explicitly write SQL commands yourself. - * It supports WHERE clauses of the form: - * - * Col1 constraint Val1 AND Col2 constraint Val2 AND... - * - * See the top of this file for the constraints enum. Some - * limitations of this class currently include: - * - * - No support for WHERE clauses that include logical *OR* - * - No support for WHERE clauses against blob columns - * - * Here are examples / pseudo-sql statements that are not - * supported, per the above limitations: - * - * SELECT * FROM Employees WHERE LastName='Smith' OR Position='Mgr' - * SELECT * FROM Employees WHERE EmployeeID=[1,5,3,5,2,8] - * - * The first is no good since it uses logical OR, and the - * second is no good since it has a column constraint with - * a blob value data type. - * - * One note on the logical OR limitation. If you need a - * query that looks something like this: - * - * SELECT * FROM Employees WHERE EmployeeID=104 OR EmployeeID=398 - * SELECT * FROM Employees WHERE LastName='Smith' OR LastName='Thompson' - * - * You can achieve this with the 'in_set' constraint like so: - * - * ObjectQuery query1(obj_mgr, "Employees"); - * query1.addConstraints( - * "EmployeeID", simdb::constraints::in_set, {104,398}); - * ... - * - * ObjectQuery query2(obj_mgr, "Employees"); - * query2.addConstraints( - * "LastName", simdb::constraints::in_set, {"Smith","Thompson"}); - * ... - */ -class ObjectQuery -{ -public: - ObjectQuery(const ObjectManager & obj_mgr, - const std::string & tbl_name) : - obj_mgr_(obj_mgr), - tbl_name_(tbl_name) - {} - - //! Single-contraint query. Also used as the base function - //! for the variadic function below. Specific to integral - //! and string target values. - template - typename std::enable_if< - not std::is_floating_point::value, - void>::type - addConstraints(const char * col_name, - const constraints constraint, - const ColumnT & col_val) - { - //Chain multiple constraints with logical AND - if (has_query_constraints_) { - query_constraints_oss_ << " AND "; - } - has_query_constraints_ = true; - - //Note that the stringify() method will take - //care of any special formatting this data type - //might need in order to form a valid SQL string, - //such as enclosing string constraint values in - //single quotes. - query_constraints_oss_ << col_name - << constraint - << stringify(col_val); - } - - //! Single-contraint query. Also used as the base function - //! for the variadic function below. Specific to floating- - //! point target values. - template - typename std::enable_if< - std::is_floating_point::value, - void>::type - addConstraints(const char * col_name, - const constraints constraint, - const ColumnT col_val) - { - //Chain multiple constraints with logical AND - if (has_query_constraints_) { - query_constraints_oss_ << " AND "; - } - has_query_constraints_ = true; - - //For queries that want to compare floating-point - //column values against a target value to match - //*exactly*, we will allow a tolerance of machine - //epsilon to consider the comparison a match. We - //perform the comparison ourselves in the withinTol() - //function we registered with SQLite. - if (constraint == constraints::equal) { - query_constraints_oss_ - << "withinTol(" << col_name << "," - << std::setprecision(std::numeric_limits::max_digits10) - << col_val << "," << std::numeric_limits::epsilon() << ")"; - } else { - //Note that the stringify() method will take - //care of any special formatting this data type - //might need in order to form a valid SQL string, - //such as enclosing string constraint values in - //single quotes. - query_constraints_oss_ << col_name - << constraint - << stringify(col_val); - } - } - - //! Initializer list constraints. Supports queries like: - //! - //! query.addConstraints( - //! "EmployeeID", - //! constraints::in_set, - //! {100,106,107,598,678}); - template - void addConstraints(const char * col_name, - const constraints constraint, - std::initializer_list col_val) - { - //Chain multiple constraints with logical AND - if (has_query_constraints_) { - query_constraints_oss_ << " AND "; - } - has_query_constraints_ = true; - - //See note about stringify() in the above method - query_constraints_oss_ << col_name - << constraint - << stringify(col_val); - } - - //! Multi-constraint query. Call site might look like this: - //! - //! ObjectQuery query(obj_mgr, "ReportHeader"); - //! - //! query.addConstraints( - //! "WarmupInsts", constraints::greater_than, 12800, - //! "NumStatInsts", constraints::less_than_or_equal_to, 300, - //! "ReportName", constraints::in_set, {"mysim","sparta_core_example"}); - //! - template - void addConstraints(const char * col_name, - const constraints constraint, - const ColumnT & col_val, - Args &&... args) - { - //Process one constraint. - addConstraints(col_name, constraint, col_val); - - //Peel off the rest of the constraints and keep going. - addConstraints(std::forward(args)...); - } - - //! Single-column value memcpy. You will typically want to - //! read one or more column values for each of the records - //! that match your query. Call this method to have the - //! records' column values written directly into your own - //! variables. For example: - //! - //! ObjectQuery query(obj_mgr, "ReportHeader"); - //! query.addConstraints(...); //optional - //! - //! std::string report_name; - //! query.writeResultIterationsTo("ReportName", &report_name); - //! - //! std::unique_ptr db_iter = query.executeQuery(); - //! while (db_iter->getNext()) { - //! std::cout << "Found another record! Its report name is '" - //! << report_name << "'" << std::endl; - //! } - //! - //! This method is also used as the base function for the - //! variadic function below. - template - void writeResultIterationsTo(const char * col_name, - ColumnT * result_ptr) - { - static_assert( - !is_container::value || - (is_contiguous::value && - std::is_trivial::value_type>::value), - "ObjectQuery::writeResultIterationsTo(name, &var) can " - "be used to extract blobs from the database directly " - "into your own variable, but that variable must be a " - "std::vector, where the type T is scalar numeric."); - - result_iter_dest_ptrs_[col_name] = std::make_pair( - result_ptr, column_info::data_type()); - } - - //! Multi-column value memcpy. Call site might look like this: - //! - //! ObjectQuery query(obj_mgr, "ReportHeader"); - //! query.addConstraints(...); //optional - //! - //! std::string report_name, - //! uint64_t start_time; - //! uint64_t end_time; - //! - //! query.writeResultIterationsTo( - //! "ReportName", &report_name, - //! "StartTime", &start_time, - //! "EndTime", &end_time); - //! - //! std::unique_ptr db_iter = query.executeQuery(); - //! while (db_iter->getNext()) { - //! std::cout << "Found another record! Its report name is '" - //! << report_name << "', its start time is " - //! << start_time << ", and its end time is " - //! << end_time << std::endl; - //! } - template - void writeResultIterationsTo(const char * col_name, - ColumnT * result_ptr, - Args &&... args) - { - writeResultIterationsTo(col_name, result_ptr); - writeResultIterationsTo(std::forward(args)...); - } - - //! Optionally apply an ORDER BY clause to your query - void orderBy(OrderBy order_by) - { - std::swap(order_by_, order_by); - } - - //! Optionally apply a LIMIT clause to your query - void setLimit(const uint32_t limit) - { - if (limit == 0) { - std::cout << "Encountered ObjectQuery::setLimit() call with " - << "LIMIT 0. This will be ignored." << std::endl; - return; - } - limit_ = limit; - } - - //! Call this method to see how many records match your - //! current constraints: - //! - //! auto query(obj_mgr, "ReportHeader"); - //! query.addConstraints("ReportName", constraints::equal, "foo.json"); - //! std::cout << "There are " << query.countMatches() << " reports " - //! << "with the name 'foo.json'"; - //! - //! query.addConstraints("StartTime", constraints::greater, 5000); - //! std::cout << "And of those, " << query.countMatches() << " have " - //! << "a start time greater than 5000."; - size_t countMatches() { - auto cur_has_query_constraints = has_query_constraints_; - auto cur_query_constraints_oss_str = query_constraints_oss_.str(); - auto cur_result_iter_dest_ptrs = result_iter_dest_ptrs_; - auto cur_limit = limit_; - auto cur_order_by = order_by_; - - if (!has_query_constraints_) { - addConstraints("Id", constraints::not_equal, 0); - } - - DatabaseID unused_id; - if (result_iter_dest_ptrs_.empty()) { - writeResultIterationsTo("Id", &unused_id); - } - - size_t count = 0; - auto result_iter = executeQuery(); - while (result_iter->getNext()) { - ++count; - } - - has_query_constraints_ = cur_has_query_constraints; - query_constraints_oss_ << cur_query_constraints_oss_str; - result_iter_dest_ptrs_ = cur_result_iter_dest_ptrs; - limit_ = cur_limit; - order_by_ = cur_order_by; - - return count; - } - - //! Once your constraints (WHERE, addConstraint()) and your - //! destination variables (SELECT, writeResultIterationsTo()) - //! have been set, finalize the query with a call to this method. - //! The returned object is "bound" to the SQL query and your - //! destination variable(s). Use the ResultIter object to loop - //! over all records that matched your query. - std::unique_ptr executeQuery() - { - if (result_iter_dest_ptrs_.empty()) { - //We should consider warning. This is an empty SQL query - //without any SELECT clause. - return nullptr; - } - - //ObjectQuery is not supported by all SimDB implementations - //at this time. Return null if this is the case. - if (!obj_mgr_.getDbConn()->supportsObjectQuery_()) { - return nullptr; - } - - auto int2ascii = [](const uint32_t i) { - std::ostringstream oss; - oss << i; - return oss.str(); - }; - - const std::string tbl_name = obj_mgr_.getQualifiedTableName(tbl_name_, "Stats"); - - //Build up the final SQL command - const std::string command = - "SELECT " + getResultIterColumnNames_() + - " FROM " + tbl_name + - (has_query_constraints_ ? - (" WHERE " + query_constraints_oss_.str()) : "") + - std::string(order_by_) + - (limit_ > 0 ? (" LIMIT " + int2ascii(limit_)) : ""); - - const DbConnProxy * db_proxy = obj_mgr_.getDbConn(); - - //Create the prepared statement - sqlite3_stmt * stmt_retrieve = nullptr; - void * statement = nullptr; - - db_proxy->prepareStatement_(command, &statement); - assert(statement != nullptr); - stmt_retrieve = static_cast(statement); - - //Prepared statement should be good to go - assert(stmt_retrieve != nullptr); - - std::unique_ptr result(new ResultIter( - result_iter_dest_ptrs_, stmt_retrieve)); - - //Clear out our internals before returning. This needs - //further design/discussion about what we do with these - //ObjectQuery's that have already been used. They may - //be useful to keep around for performance gains in - //a use case like this: - // - // ObjectQuery query(obj_mgr, "Employees"); - // std::string first_name; - // int age; - // - // query.writeResultIterationsTo( - // "FirstName", &first_name, - // "Age", &age); - // - // //Say we want to find all first names and ages for - // //those employees with last name 'Smith'. - // query.addConstraint("LastName", constraints::equal, "Smith"); - // auto db_iter = query.executeQuery(); - // while (db_iter->getNext()) { - // //Process the employees with last name 'Smith' - // } - // - // //Now do the same thing, but for all employees with - // //last name 'Thompson'. Nothing else has changed - // //regarding our destination variables 'first_name' - // //and 'age', so it would be nice to be able to reuse - // //the ObjectQuery and keep going: - // query.resetConstraint("LastName", constraints::equal, "Thompson"); - // db_iter = query.executeQuery(); - // while (db_iter->getNext()) { - // //Process the employees with last name 'Thompson' - // } - // - //For right now, reset all ObjectQuery internals until - //this part of the design / DB requirements are better - //understood. - has_query_constraints_ = false; - query_constraints_oss_.str(std::string()); - query_constraints_oss_.clear(); - result_iter_dest_ptrs_.clear(); - - OrderBy empty_order_by; - orderBy(empty_order_by); - limit_ = 0; - - return result; - } - -private: - //Put together the SELECT clause for this query. Examples: - // - // SELECT * FROM Customers - // SELECT LastName,FirstName FROM Employees - // SELECT PhoneNumber FROM Contacts - // ... - std::string getResultIterColumnNames_() - { - //We should have short-circuited this function call - //if we didn't have any SELECT column(s) - if (result_iter_dest_ptrs_.empty()) { - throw DBException("ObjectQuery::executeQuery() called ") - << "before the query was ready. This is a bug."; - } - - //Single-column SELECT's are returned as-is - auto iter = result_iter_dest_ptrs_.begin(); - if (result_iter_dest_ptrs_.size() == 1) { - return iter->first; - } - - //Multi-column SELECT's are comma-separated - std::ostringstream oss; - for (size_t idx = 0; idx < result_iter_dest_ptrs_.size()-1; ++idx) { - oss << iter->first << ","; - ++iter; - } - oss << iter->first; - return oss.str(); - } - - const ObjectManager & obj_mgr_; - const std::string tbl_name_; - bool has_query_constraints_ = false; - std::ostringstream query_constraints_oss_; - OrderBy order_by_; - uint32_t limit_ = 0; - - //Mapping from a column name to a pair - // - The ResultColumn pair holds the address of the user's - // object (&std::string, &int, etc.) as well as the column - // data type enum (String, Blob, etc.) and we use both of - // these pieces of info to write a record's column values - // into the caller's own variables. - std::map result_iter_dest_ptrs_; -}; - -} // namespace simdb diff --git a/sparta/simdb/include/simdb/utils/StringUtils.hpp b/sparta/simdb/include/simdb/utils/StringUtils.hpp deleted file mode 100644 index 778916ed29..0000000000 --- a/sparta/simdb/include/simdb/utils/StringUtils.hpp +++ /dev/null @@ -1,206 +0,0 @@ -// -*- C++ -*- - -/*! - * \file StringUtils.h - * \brief String utilities used in SimDB - */ - -#pragma once - -#include "simdb/utils/uuids.hpp" - -#include -#include - -namespace simdb { -namespace utils { - -/*! - * \brief Generate a random string value. Uses the generateUUID() - * method. This is here for continuity with the other chooseRand() - * methods found in MathUtils.h - */ -template -typename std::enable_if< - std::is_same::value, -T>::type -chooseRand() -{ - return simdb::generateUUID(); -} - -/*! - * \brief Utility class which applies a user-provided functor to all - * chars of a std::string, so that users do not have to remember to - * always apply these algorithms manually themselves. - */ -template -class TransformedString -{ -public: - TransformedString() = default; - - TransformedString(const char * str) : - TransformedString(std::string(str)) - {} - - TransformedString(const std::string & str) : - str_(str) - { - applyTransform_(); - } - - TransformedString & operator=(const std::string & str) { - str_ = str; - applyTransform_(); - return *this; - } - - TransformedString(const TransformedString & rhs) : - str_(rhs.str_) - {} - - TransformedString & operator=(const TransformedString & rhs) { - str_ = rhs.str_; - return *this; - } - - inline bool operator==(const TransformedString & rhs) const { - return str_ == rhs.str_; - } - - inline bool operator!=(const TransformedString & rhs) const { - return str_ != rhs.str_; - } - - inline bool operator==(const std::string & rhs) const { - return str_ == rhs; - } - - inline bool operator!=(const std::string & rhs) const { - return str_ != rhs; - } - - inline bool operator==(const char * rhs) const { - return str_ == rhs; - } - - inline bool operator!=(const char * rhs) const { - return str_ != rhs; - } - - inline TransformedString & operator+=(const char ch) { - TransformedString rhs(std::string(1, ch)); - str_ += rhs.str_; - return *this; - } - - inline TransformedString & operator+=(const std::string & str) { - TransformedString rhs(str); - str_ += rhs.str_; - return *this; - } - - bool empty() const { - return str_.empty(); - } - - size_t size() const { - return str_.size(); - } - - void clear() { - str_.clear(); - } - - inline const std::string & getString() const { - return str_; - } - - inline operator std::string() const { - return str_; - } - -private: - void applyTransform_() { - std::transform(str_.begin(), str_.end(), str_.begin(), transformer_); - } - - std::string str_; - AppliedTransform transformer_; -}; - -// Get the lower/UPPER underlying transformed string, and print -// the transformed string to an ostream. - -// Functor for ::tolower -struct make_lowercase { - int operator()(const int ch) const { - return std::tolower(ch); - } -}; - -// Functor for ::toupper -struct make_uppercase { - int operator()(const int ch) const { - return std::toupper(ch); - } -}; - -// always lowercase string data type -typedef TransformedString lowercase_string; - -// ALWAYS UPPERCASE STRING DATA TYPE -typedef TransformedString UPPERCASE_STRING; - -// Get the lowercase underlying transformed string, and print -// the transformed string to an ostream. -inline std::ostream & operator<<(std::ostream & os, - const lowercase_string & s) -{ - os << s.getString(); - return os; -} - -// Get the UPPERCASE underlying transformed string, and print -// the transformed string to an ostream. -inline std::ostream & operator<<(std::ostream & os, - const UPPERCASE_STRING & s) -{ - os << s.getString(); - return os; -} - -// Less than comparison, so you can put these data types -// into ordered containers like std::set's -template -inline bool operator<(const TransformedString & one, - const TransformedString & two) -{ - return one.getString() < two.getString(); -} - -// Comparisons against std::string where the utils object is the rhs: -// -// std::string s1(...); -// lowercase_string s2(...); -// if (s1 == s2) { -// ... -// } -template -inline bool operator==(const std::string & one, - const TransformedString & two) -{ - return one == two.getString(); -} - -template -inline bool operator!=(const std::string & one, - const TransformedString & two) -{ - return one != two.getString(); -} - -} // namespace utils -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/utils/Stringifiers.hpp b/sparta/simdb/include/simdb/utils/Stringifiers.hpp deleted file mode 100644 index 62138eb498..0000000000 --- a/sparta/simdb/include/simdb/utils/Stringifiers.hpp +++ /dev/null @@ -1,103 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/GeneralMetaStructs.hpp" - -#include -#include -#include - -namespace simdb { - -//! Stringify scalar numbers for SQL statement creation -template -typename std::enable_if< - !std::is_same::type, const char*>::value and - std::is_trivial::value, std::string>::type -stringify(const ColumnT val) -{ - std::ostringstream oss; - constexpr auto digits10 = std::numeric_limits::max_digits10; - oss << std::setprecision(digits10); - oss << val; - return oss.str(); -} - -//! Stringify std::string's for SQL statement creation -template -typename std::enable_if::value, std::string>::type -stringify(const ColumnT & val) -{ - std::ostringstream oss; - oss << "'" << val << "'"; - return oss.str(); -} - -//! Stringify const char*'s for SQL statement creation -template -typename std::enable_if< - std::is_same::type, const char*>::value, std::string>::type -stringify(const ColumnT val) -{ - std::ostringstream oss; - oss << "'" << val << "'"; - return oss.str(); -} - -//! Stringify STL containers -template -typename std::enable_if::value, std::string>::type -stringify(const ColumnT & val) -{ - auto begin = val.begin(); - auto end = val.end(); - const auto num_els = std::distance(begin, end); - - if (num_els == 0) { - return ""; - } - - if (num_els == 1) { - return "(" + stringify(*begin) + ")"; - } - - std::ostringstream oss; - oss << "("; - for (long idx = 0; idx < num_els-1; ++idx) { - oss << stringify(*begin) << ","; - ++begin; - } - oss << stringify(*begin) << ")"; - return oss.str(); -} - -//! Stringifier std::initializer_list -template -typename std::enable_if::value, std::string>::type -stringify(ColumnT val) -{ - auto begin = val.begin(); - auto end = val.end(); - const size_t num_els = std::distance(begin, end); - - if (num_els == 0) { - return ""; - } - - if (num_els == 1) { - return "(" + stringify(*begin) + ")"; - } - - std::ostringstream oss; - oss << "("; - for (size_t idx = 0; idx < num_els-1; ++idx) { - oss << stringify(*begin) << ","; - ++begin; - } - oss << stringify(*begin) << ")"; - return oss.str(); -} - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb/utils/uuids.hpp b/sparta/simdb/include/simdb/utils/uuids.hpp deleted file mode 100644 index 92bb91662b..0000000000 --- a/sparta/simdb/include/simdb/utils/uuids.hpp +++ /dev/null @@ -1,46 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include - -#include -#include -#include - -namespace simdb { - -//! \note There is a known issue with the Boost -//! UUID library that causes valgrind to fail. -//! Some Boost versions intentionally read from -//! uninitialized memory in order to increase -//! randomness of their UUID algorithms. This -//! occurs in the default constructor of: -//! -//! boost::uuids::basic_random_generator::ctor() -//! -//! The Boost documentation notes related to -//! boost::uuid and valgrind suggest to get -//! around the valgrind failure by either -//! suppressing these valgrind errors, or -//! by using a non-default constructor for -//! the basic_random_generator class. -inline std::string generateUUID() -{ - std::mt19937 ran; - auto seed = std::chrono::high_resolution_clock::now(). - time_since_epoch().count(); - - ran.seed(static_cast(seed)); - - boost::uuids::basic_random_generator gen(&ran); - boost::uuids::uuid uuid = gen(); - - std::ostringstream oss; - oss << uuid; - return oss.str(); -} - -} // namespace simdb - diff --git a/sparta/simdb/include/simdb_fwd.hpp b/sparta/simdb/include/simdb_fwd.hpp deleted file mode 100644 index 1ce3df2519..0000000000 --- a/sparta/simdb/include/simdb_fwd.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/DatabaseTypedefs.hpp" - -/*! - * \brief Forward declarations of SimDB classes. - * Declared/defined here so downstream projects - * can freely include it in any of their header - * files without worrying about impacting compile - * times. - */ - -namespace simdb { - -//! Forward declarations -------------------------- -class AsyncTaskController; -class AsyncTaskEval; -class Column; -class ColumnValueBase; -class DatabaseNamespace; -class DatabaseRoot; -class DbConnProxy; -class ObjectProxy; -class ObjectQuery; -class ObjectManager; -class ObjectRef; -class ResultIter; -class Schema; -class SQLiteConnProxy; -class Table; -class TableProxy; -class TableRef; -class TableSummaries; -class TimerThread; - -template -class ConcurrentQueue; - -} - diff --git a/sparta/simdb/src/HDF5Connection.cpp b/sparta/simdb/src/HDF5Connection.cpp deleted file mode 100644 index 2b26f4d6ab..0000000000 --- a/sparta/simdb/src/HDF5Connection.cpp +++ /dev/null @@ -1,1099 +0,0 @@ -// -*- C++ -*- - -#include "simdb/ObjectManager.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/schema/ColumnMetaStructs.hpp" -#include "simdb/Errors.hpp" - -//HDF5-specific headers -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" -#include "simdb/impl/hdf5/DataTypeUtils.hpp" -#include "simdb/impl/hdf5/Resources.hpp" -#include "hdf5.h" - -//Standard headers -#include -#include -#include -#include -#include - -#define HDF5_MAX_NAME 1024 - -namespace simdb { - -/*! - * \class HDF5DatasetIds - * - * \brief Class which holds onto HDF5 identifiers, - * and closes them / their associated resource from - * the destructor. Basically a smart pointer for - * HDF5 ids. - */ -class HDF5DatasetIds -{ -public: - //! \brief Construct with HDF5 dataset and data - //! type IDs - //! - //! \param dset Dataset ID. H5Dclose() will be - //! called on this identifier on destruction. - //! - //! \param dtype Data type ID. H5Tclose() will - //! be called on this identifier on destruction. - HDF5DatasetIds(const hid_t dset, - const hid_t dtype) : - dset_(dset), - dtype_(dtype) - {} - - HDF5DatasetIds(const HDF5DatasetIds &) = delete; - HDF5DatasetIds & operator=(const HDF5DatasetIds &) = delete; - - hid_t getDatasetId() const { - return dset_; - } - - hid_t getDataTypeId() const { - return dtype_; - } - -private: - H5DResource dset_; - H5TResource dtype_; -}; - -/*! - * \class HDF5FileScanner - * - * \brief Given a handle to an HDF5 file, scan the file - * contents for all tables (datasets) and columns (dataset - * elements/fields). This is used to recreate the original - * HDF5 schema when connectToExistingDatabase() is called - * on an HDF5 file outside of a running simulation. - */ -class HDF5FileScanner -{ -public: - //! \brief Reconstruct HDF5 dataset / data type objects - //! from an existing HDF5 database file - //! - //! \param hfile File ID obtained via H5Fcreate() or - //! H5Fopen() - //! - //! \return Reconstructed schema object containing the - //! same tables (datasets) and columns (member fields) as - //! found in the file. - Schema scanSchema(const hid_t hfile) - { - Schema schema; - H5GResource grp = H5Gopen(hfile, "/", H5P_DEFAULT); - scanGroup_(grp, schema); - return schema; - } - - //! \brief Get a mapping of the table names found in the - //! file, to their associated dataset and data type IDs. - //! - //! \return Map of table names to an HDF5DatasetIDs object, - //! which can be used to get the dataset (table) ID and - //! the compound/struct data type ID of that table. - const std::map> & getDatasetIds() - { - return dataset_ids_; - } - -private: - //! \brief Parse dataset information about an HDF5 group - void scanGroup_(const hid_t gid, Schema & schema) - { - char memb_name[HDF5_MAX_NAME]; - - hsize_t nobj; - H5Gget_num_objs(gid, &nobj); - for (size_t idx = 0; idx < nobj; ++idx) { - auto len = H5Gget_objname_by_idx( - gid, idx, memb_name, HDF5_MAX_NAME); - - const std::string memb_name_string(memb_name, len); - - auto otype = H5Gget_objtype_by_idx(gid, idx); - if (otype == H5G_DATASET) { - auto & table = schema.addTable(memb_name_string); - auto dsid = H5Dopen(gid, memb_name, H5P_DEFAULT); - scanDataset_(dsid, table); - } - } - } - - //! \brief Parse data type information about an HDF5 dataset - void scanDataset_(const hid_t dsid, Table & table) - { - const hid_t tid = H5Dget_type(dsid); - const H5T_class_t t_class = H5Tget_class(tid); - if (t_class == H5T_COMPOUND) { - scanCompoundDatatype_(tid, table); - dataset_ids_[table.getName()] = std::make_shared(dsid, tid); - } else { - H5Dclose(dsid); - H5Tclose(tid); - } - } - - //! \brief Parse field information about an HDF5 compound - //! data type - void scanCompoundDatatype_(const hid_t tid, Table & table) - { - const int nfields = H5Tget_nmembers(tid); - for (int idx = 0; idx < nfields; ++idx) { - const std::string memb_name(reinterpret_cast( - H5Tget_member_name(tid, idx))); - - H5TResource ftype = H5Tget_member_type(tid, idx); - table.addColumn(memb_name, getPODColumnDTypeFromHDF5(ftype)); - } - } - - std::map> dataset_ids_; -}; - -/*! - * \class HDF5Dataset - * - * \brief Handle requests from the HDF5ConnProxy class - * to create and populate HDF5 datasets from SimDB schemas - * and record raw values. - */ -class HDF5Dataset -{ -public: - //! \brief Construct a dataset with the desired table name - explicit HDF5Dataset(const std::string & table_name) : - table_name_(table_name) - { - if (table_name_.empty()) { - throw DBException("Empty table name given to HDF5 dataset"); - } - } - - //! \brief There are TableRef APIs which instantiate - //! records using non-contiguous record values paired - //! up with their column names, as well as APIs which - //! accept column values *only*. The latter is more - //! performant, but you can only use it when your table / - //! dataset has nothing but fixed-size POD's for its - //! columns / fields. - //! - //! This method here lets us know if we can "interpret - //! the table records as structs". If yes, we can ask - //! our Column objects directly what their byte offset - //! is. Non-contiguous records use Columns that do not - //! have a known byte offset, since they do not have - //! the same reference point like POD fields in a C - //! struct do. - void interpretAsStruct() - { - is_struct_table_ = true; - } - - //! \brief As the schema is getting realized, this - //! method gets called once for each table column. - void addColumnToDataset(const Column & col) - { - columns_.emplace_back(col); - } - - //! \brief Set up the variables needed in order to - //! read or write an existing HDF5 file. - void recreateDatasetFromFile(const std::shared_ptr & ids, - const std::string & table_name) - { - if (ids == nullptr) { - return; - } - dataset_ids_ = ids; - dataset_name_ = table_name; - - constexpr hsize_t ndims = 2; - static hsize_t memspace_dims[ndims] = {1, 1}; - memspace_ = H5Screate_simple(ndims, memspace_dims, nullptr); - - H5SResource space = H5Dget_space(dataset_ids_->getDatasetId()); - num_writes_ = H5Sget_select_npoints(space); - } - - //! \brief Given an HDF5 file ID and table name, turn - //! our schema metadata into a realized, ready-to-go - //! HDF5 Dataset. We'll use the dataset later to write - //! raw bytes into the HDF5 file through the C library. - void createDatasetInFile(const hid_t hfile, - const std::string & table_name, - CompressionType compression) - { - static const hsize_t ndims = 1; - hsize_t dims[ndims] = {0}; - static hsize_t maxdims[ndims] = {H5S_UNLIMITED}; - H5SResource filespace = H5Screate_simple(ndims, dims, maxdims); - - H5PResource plist = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_layout(plist, H5D_CHUNKED); - - if (compression != CompressionType::NONE && - !H5Zfilter_avail(H5Z_FILTER_DEFLATE)) - { - std::cout << " [simdb] HDF5 compression requested, " - << "but gzip is not available" << std::endl; - compression = CompressionType::NONE; - } - - //The HDF5 library lets you set the compression - //level on a scale from 0-9: - // - // 0 -- 1 -- 2 -- 3 -- 4 -- 5 -- 6 -- 7 -- 8 -- 9 - //Compression: (none) ... (some) ........................ (max) - //Speed: (fast) .... (slower) ............... (slowest) - switch (compression) { - case CompressionType::NONE: { - H5Pset_deflate(plist, 0); - break; - } - case CompressionType::DEFAULT_COMPRESSION: { - H5Pset_deflate(plist, 5); - break; - } - case CompressionType::BEST_COMPRESSION_RATIO: { - H5Pset_deflate(plist, 9); - break; - } - case CompressionType::BEST_COMPRESSION_SPEED: { - H5Pset_deflate(plist, 1); - break; - } - } - - //TODO: It is not really possible to determine a one-size- - //fits-all chunk size that is performant for every use case. - //This should eventually be surfaced as a tunable parameter. - hsize_t chunkdims[ndims] = {1000}; - H5Pset_chunk(plist, ndims, chunkdims); - - //Build the compound data type field by field. We first - //need to calculate the total number of bytes in one of - //these compound elements (structs). This is the data - //size in bytes that will be written to the HDF5 file - //during each call to the writeRawBytes() method. - const size_t record_num_bytes = getOneRecordNumBytes_(); - auto dtid = H5Tcreate(H5T_COMPOUND, record_num_bytes); - - //The dtid variable is the data type ID for the compound - //data type (struct, or one record/row in our table). - addColumnsToCompoundDataType_(dtid); - - auto dset = H5Dcreate( - hfile, table_name.c_str(), dtid, filespace, - H5P_DEFAULT, plist, H5P_DEFAULT); - - if (dset < 0) { - throw DBException("Unable to create dataset for HDF5 table: ") - << table_name; - } - dataset_ids_ = std::make_shared(dset, dtid); - dataset_name_ = table_name; - - static hsize_t memspace_dims[ndims] = {1}; - memspace_ = H5Screate_simple(ndims, memspace_dims, nullptr); - } - - //! \brief This dataset object currently only supports fixed- - //! size columns (POD's) so we are free to write raw memory - //! into the HDF5 library directly. This method is called when - //! the TableRef we are associated with gets a request to - //! create a record with provided column values. - //! - //! \return The total number of records we have written into - //! this specific dataset. - size_t writeRawBytes(const void * raw_bytes_ptr, - const size_t num_raw_bytes) - { - const hid_t dset = dataset_ids_->getDatasetId(); - if (dset == 0) { - throw DBException( - "Method cannot be called. Dataset does not exist."); - } - - hsize_t dims[2] = {num_writes_ + 1, 1}; - H5Dset_extent(dset, dims); - - H5SResource filespace = H5Dget_space(dset); - hsize_t start[2] = {num_writes_, 0}; - hsize_t count[2] = {1, 1}; - H5Sselect_hyperslab(filespace, H5S_SELECT_SET, - start, nullptr, - count, nullptr); - - const hid_t dtype = dataset_ids_->getDataTypeId(); - - //The HDF5ConnProxy class currently only supports writing - //fixed-size records (either a single fixed-size POD type, - //or a struct whose fields are all fixed-size POD types). - if (H5Tget_size(dtype) != num_raw_bytes) { - auto err = DBException("Invalid call to write HDF5 data. Attempt to ") - << "write " << num_raw_bytes << " bytes' worth of raw data into " - << "an HDF5 dataset that is " << H5Tget_size(dtype) << " bytes " - << "in size."; - if (!dataset_name_.empty()) { - err << " Occurred for dataset '" << dataset_name_ << "'."; - } - err << "\n"; - throw (err); - } - - H5Dwrite(dset, dtype, memspace_, filespace, H5P_DEFAULT, raw_bytes_ptr); - - ++num_writes_; - return num_writes_; - } - - //! \brief Read raw data out of the HDF5 file belonging - //! to this dataset - //! - //! \param prop_name Column / field name of the requested value - //! - //! \param db_id Unique database ID of the record in this dataset - //! (table). Equivalent to SQL's rowid. - //! - //! \param dest_ptr Preallocated memory the raw bytes from the - //! database should be written into - //! - //! \param num_bytes Number of bytes of preallocated memory - //! the dest_ptr parameter is pointing to - //! - //! \return Number of bytes read - size_t readRawBytes( - const std::string & prop_name, - const DatabaseID db_id, - void * dest_ptr, - const size_t num_bytes) const - { - if (db_id <= 0) { - return 0; - } - - const hid_t dset = dataset_ids_->getDatasetId(); - const hid_t dtype = dataset_ids_->getDataTypeId(); - - const int field_idx = H5Tget_member_index(dtype, prop_name.c_str()); - if (field_idx < 0) { - std::cout << " [simdb] Property named '" - << prop_name << "' not found in HDF5 dataset '" - << table_name_ << "'" << std::endl; - return 0; - } - - H5TResource field_dtype = H5Tget_member_type(dtype, field_idx); - if (H5Tget_size(field_dtype) != num_bytes) { - std::cout << " [simdb] Incorrect number of bytes " - << "requested from HDF5 dataset '" - << table_name_ << "'" << std::endl; - return 0; - } - - H5SResource filespace = H5Dget_space(dset); - H5Sselect_all(filespace); - - const hsize_t start[2] = {static_cast(db_id - 1), 0}; - const hsize_t count[2] = {1, 1}; - H5Sselect_hyperslab(filespace, H5S_SELECT_SET, - start, nullptr, - count, nullptr); - - const size_t compound_size = H5Tget_size(dtype); - if (compound_size == 0) { - return 0; - } - - const hsize_t memdims[2] = {1, 1}; - H5SResource memspace = H5Screate_simple(1, memdims, nullptr); - - const size_t field_offset = H5Tget_member_offset(dtype, field_idx); - assert(field_offset < compound_size); - - std::vector raw_chars(compound_size); - H5Dread(dset, dtype, memspace, filespace, - H5P_DEFAULT, &raw_chars[0]); - - memcpy(dest_ptr, &raw_chars[field_offset], num_bytes); - return num_bytes; - } - - //! \brief Return the number of elements in this dataset. - //! This can be called on an "active" HDF5 connection - //! during simulation, or on an "inactive" connection - //! outside of a simulation. - size_t getNumElements() const - { - return num_writes_; - } - -private: - //! Calculate the number of bytes in one row of - //! this dataset table. - size_t getOneRecordNumBytes_() const - { - if (is_struct_table_) { - return getOneRecordNumBytesForStructTable_(); - } else { - return getOneRecordNumBytesForNonContiguousTable_(); - } - } - - //! Calculate the number of bytes in one row of - //! this dataset table. Here, a "struct table" is - //! one which was defined in the schema using the - //! addTable() / addField() methods. Tables defined - //! like this are free to use the more performant - //! TableRef::createObjectFromStruct() API, which - //! takes a literal struct of POD's and writes - //! the struct to file directly just reading bytes - //! from the caller's struct: - //! - //! struct Foo { - //! //field 1 - //! //... - //! //field 100 - //! }; - //! Foo f; - //! - //! sim_db_->getTable("Foo")->createObjectFromStruct(f); - size_t getOneRecordNumBytesForStructTable_() const - { - assert(is_struct_table_); - - const auto & col = columns_.back(); - - size_t byte_offset = col.getByteOffset(); - - byte_offset += getFixedNumBytesForColumnDType( - col.getDataType(), - col.getDimensions()); - - return byte_offset; - } - - //! Calculate the number of bytes in one row of - //! this dataset table. Here, a "non-contiguous" - //! table is one which was defined in the schema - //! using the addTable() / addColumn() methods. - //! Tables built with these APIs must use the - //! TableRef::createObject(), createObjectWithArgs(), - //! and/or createObjectWithVals() APIs. - size_t getOneRecordNumBytesForNonContiguousTable_() const - { - assert(!is_struct_table_); - - size_t record_num_bytes = 0; - for (const auto & col : columns_) { - record_num_bytes += getFixedNumBytesForColumnDType( - col.getDataType(), - columns_.back().getDimensions()); - } - - return record_num_bytes; - } - - //! Append a field to this dataset with the given ID. - void addColumnsToCompoundDataType_( - const hid_t compound_dtid) - { - if (is_struct_table_) { - addColumnsToCompoundDataTypeForStructTable_(compound_dtid); - } else { - addColumnsToCompoundDataTypeForNonContiguousTable_(compound_dtid); - } - } - - //! Append a field to this dataset with the given ID, - //! for datasets that are populated using literal C - //! structs as the input data source. - void addColumnsToCompoundDataTypeForStructTable_( - const hid_t compound_dtid) - { - assert(is_struct_table_); - - for (const auto & col : columns_) { - const size_t el_offset = col.getByteOffset(); - auto field_dtype = getScopedDTypeForHDF5(col); - - H5Tinsert(compound_dtid, col.getName().c_str(), - el_offset, field_dtype.getDataTypeId()); - } - } - - //! Append a field to this dataset with the given ID, - //! for datasets that are populated using separate - //! variables, even if those variables are all fixed- - //! size data types. - void addColumnsToCompoundDataTypeForNonContiguousTable_( - const hid_t compound_dtid) - { - assert(!is_struct_table_); - - size_t el_offset = 0; - for (const auto & col : columns_) { - auto field_dtype = getScopedDTypeForHDF5(col); - - H5Tinsert(compound_dtid, col.getName().c_str(), - el_offset, field_dtype.getDataTypeId()); - - el_offset += getFixedNumBytesForColumnDType(col.getDataType()); - } - } - - H5SResource memspace_; - size_t num_writes_ = 0; - std::vector columns_; - bool is_struct_table_ = false; - std::string table_name_; - std::shared_ptr dataset_ids_; - std::string dataset_name_; -}; - -//! \brief Utility method which looks for a file relative -//! to the working directory. Tries to find the file with -//! and without the provided directory. -//! -//! \param db_dir Directory path to the HDF5 file -//! -//! \param db_file HDF5 file name, including the ".h5" -//! extension -//! -//! \return Returns the full filename if the file was found, -//! returns an empty string if not. -std::string resolveDbFilename( - const std::string & db_dir, - const std::string & db_file) -{ - std::ifstream fin(db_dir + "/" + db_file); - if (fin) { - return db_dir + "/" + db_file; - } - - fin.open(db_file); - if (fin) { - return db_file; - } - return ""; -} - -/*! - * \class HDF5ConnProxy::Impl - * - * \brief HDF5 implementation class for SimDB - */ -class HDF5ConnProxy::Impl -{ -public: - //! \brief During database schema creation, tables - //! may be created in a "non-contiguous data" way, - //! like so: - //! - //! simdb::Schema schema; - //! - //! schema.addTable("MyNonContig") - //! .addColumn("X", dt::string_t) - //! .addColumn("Y", dt::int64_t); - //! - //! This is the table format that SQLiteConnProxy - //! uses, but HDF5 tables can have individual columns - //! put together to be physically contiguous, like a - //! C struct. Imagine the following two call sites - //! that want to write this simple struct into a - //! table: - //! - //! struct MyContig { - //! int16_t A = rand(); - //! int16_t B = rand(); - //! float C = (float)rand() / 10000 * 3.14; - //! }; - //! - //! The SQLite-esque way of doing it would be to - //! define the table as non-contiguous, and write - //! the values into the TableRef APIs in separate - //! variables, like this: - //! - //! schema.addTable("MyContig") - //! .addColumn("A", dt::int16_t) - //! .addColumn("B", dt::int16_t) - //! .addColumn("C", dt::float_t); - //! - //! void writeRow(ObjectManager & db, const MyContig & mc) - //! { - //! db.getTable("MyContig")->createObjectWithArgs( - //! "A", mc.A, "B", mc.B, "C", mc.C); - //! } - //! - //! Another way to do the same thing (though with - //! better performance) would be to define the table - //! like it is a C struct with fields: - //! - //! schema.addTable("MyContig") - //! .addField("A", dt::int16_t, FOFFSET(MyContig,A)) - //! .addField("B", dt::int16_t, FOFFSET(MyContig,B)) - //! .addField("C", dt::float_t, FOFFSET(MyContig,C)); - //! - //! This "struct table" could be written to like this: - //! - //! void writeRow(ObjectManager & db, const MyContig & mc) - //! { - //! db.getTable("MyContig")->createObjectWithVals( - //! mc.A, mc.B, mc.C); - //! } - //! - //! The lack of "A"/"B"/"C" specifiers like you see in the first - //! createObjectWithArgs() call is allowed because C structs of - //! POD's are a fixed number of bytes, and you can't rearrange - //! their fields from one binary write to the next, so we can - //! get a performance boost by writing the POD values directly, - //! without any column names to go with the values at the call - //! site. One last API which we could also use looks like this: - //! - //! void writeRow(ObjectManager & db, const MyContig & mc) - //! { - //! db.getTable("MyContig")->createObjectFromStruct(mc); - //! } - //! - //! The TableRef::createObjectFromStruct() API should only be - //! called for tables that were defined using the addField() - //! API to build struct tables. - //! - //! Unlike SQLite, we can support all three of these use cases - //! for HDF5. This method here lets the schema creation classes - //! tell us which tables can be interpreted as C structures. - //! - //! \param struct_tables Set of table names referring to schema - //! tables that were constructed with the addField() API's to - //! signify contiguous, fixed-size fields like you would have - //! in a struct of POD's. - void setStructTables(std::unordered_set && struct_tables) - { - struct_tables_ = std::move(struct_tables); - } - - //! \brief Turn a Schema object into an actual database - //! connection - //! - //! \param schema Schema object to realize - //! - //! \param obj_mgr SimDB ObjectManager that this DbConnProxy - //! object belongs to - void realizeSchema( - const Schema & schema, - const ObjectManager &) - { - static const std::map< - std::string, - std::shared_ptr - > empty_table_ids; - - realizeSchema_(schema, empty_table_ids); - } - - //! \return The HDF5 file identifier. Similar to FILE*. - hid_t getFileId() const - { - return hfile_; - } - - //! \brief Get the full database filename being used. This - //! includes the database path, stem, and extension. Returns - //! empty if no connection is open. - //! - //! \return Full filename, such as "/tmp/abcd-1234.h5" - const std::string & getDatabaseFullFilename() const - { - return db_full_filename_; - } - - //! \brief Try to open a connection to an existing database file - //! - //! \param db_file HDF5 file name, including the ".h5" extension - //! - //! \return Return true on successful connection, false otherwise - bool connectToExistingDatabase(const std::string & db_file) - { - if (openDbFile(".", db_file, false).empty() || !hfile_.good()) { - return false; - } - - HDF5FileScanner scanner; - const Schema schema = scanner.scanSchema(hfile_); - const auto & dataset_ids = scanner.getDatasetIds(); - realizeSchema_(schema, dataset_ids); - - return true; - } - - //! \return Returns true if this database is open and ready to - //! take read and write requests, false otherwise - bool isValid() const - { - return hfile_.good(); - } - - //! \brief Given a table (dataset) and a column (field) for - //! a particular record (element index / linear offset), read - //! one field's value from the database - //! - //! \param table_name Name of the table containing data - //! we want to read - //! - //! \param prop_name Name of the specific property (column / - //! field) in that table being read - //! - //! \param db_id Unique database ID for the requested - //! record. Equivalent to rowid in SQL. - //! - //! \param dest_ptr Preallocated memory the raw bytes from - //! the database should be written into - //! - //! \param num_bytes Number of bytes of preallocated memory - //! the dest_ptr is pointing to - //! - //! \return Number of bytes read - size_t readRawBytes( - const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - void * dest_ptr, - const size_t num_bytes) const - { - auto iter = datasets_.find(table_name); - if (iter != datasets_.end()) { - return iter->second->readRawBytes( - prop_name, db_id, dest_ptr, num_bytes); - } - return 0; - } - - //! \brief First-time database file open - //! - //! \param db_dir Directory path to the HDF5 file - //! - //! \param db_file HDF5 file name, including the ".h5" - //! extension - //! - //! \param create_file Flag which tells us if we should - //! create a brand new file, or open an existing file - //! - //! \return The full filename of the opened HDF5 file - std::string openDbFile( - const std::string & db_dir, - const std::string & db_file, - const bool create_file) - { - db_full_filename_ = resolveDbFilename(db_dir, db_file); - - bool open_existing = true; - if (db_full_filename_.empty()) { - if (create_file) { - db_full_filename_ = db_dir + "/" + db_file; - open_existing = false; - } else { - throw DBException("Could not find database file: '") - << db_dir << "/" << db_file; - } - } - - if (open_existing) { - hfile_ = H5Fopen(db_full_filename_.c_str(), - H5F_ACC_RDWR, - H5P_DEFAULT); - } else { - hfile_ = H5Fcreate(db_full_filename_.c_str(), - H5F_ACC_TRUNC, - H5P_DEFAULT, - H5P_DEFAULT); - } - - assert(!db_full_filename_.empty()); - return db_full_filename_; - } - - //! \brief We maintain our own unique IDs for rows written - //! into HDF5 tables, and they are zero-based, incrementing - //! by one with each write into a particular table. We can - //! answer the question "hasObject()" by comparing the ID - //! given to us with the total number of elements in the - //! dataset. - //! - //! \param table_name Table we are looking into for the - //! database ID in question - //! - //! \param db_id Unique database ID for the given table - //! - //! \return True if this database ID was found in the given - //! table, false otherwise - bool hasObject( - const std::string & table_name, - const DatabaseID db_id) const - { - auto dset_iter = datasets_.find(table_name); - if (dset_iter == datasets_.end()) { - return false; - } - - const size_t nelems = dset_iter->second->getNumElements(); - return (static_cast(db_id) > 0 && - static_cast(db_id) <= nelems); - } - - //! \brief Respond when our FixedSizeObjectFactory is invoked. - //! Create a new object with the provided raw bytes. Since the - //! table is fixed in its column(s) width, the raw bytes array - //! passed in contains a fixed, known number of bytes that the - //! HDF5 library can read from. This byte array has all of the - //! new record's column value(s) all packed together. - //! - //! \param table_name Name of the table we want to create an - //! object for (table row) - //! - //! \param raw_bytes Flat byte array containing all of the - //! columns' values for the new record - //! - //! \return Database ID of the newly created record - DatabaseID createFixedSizeObject( - const std::string & table_name, - const void * raw_bytes_ptr, - const size_t num_raw_bytes) - { - auto dset_iter = datasets_.find(table_name); - if (dset_iter == datasets_.end()) { - throw DBException("Could not find table '") << table_name << "'"; - } - - if (raw_bytes_ptr == nullptr) { - throw DBException("Cannot create a fixed-sized " - "HDF5 object with no data"); - } - - return dset_iter->second->writeRawBytes(raw_bytes_ptr, num_raw_bytes); - } - -private: - //! Turn schema/table/row metadata into a realized HDF5 file, - //! complete with the dataset objects we'll need in order to - //! write simulation data. - void realizeSchema_( - const Schema & schema, - const std::map //Dataset IDs - > & dset_ids) - { - for (const auto & table : schema) { - std::unique_ptr dset(new HDF5Dataset(table.getName())); - if (struct_tables_.find(table.getName()) != struct_tables_.end()) { - dset->interpretAsStruct(); - } - - for (const auto & column : table) { - dset->addColumnToDataset(*column); - } - - auto ids_iter = dset_ids.find(table.getName()); - if (ids_iter == dset_ids.end()) { - dset->createDatasetInFile(getFileId(), - table.getName(), - table.getCompression()); - } else if (ids_iter->second != nullptr) { - dset->recreateDatasetFromFile(ids_iter->second, table.getName()); - } - - datasets_[table.getName()] = std::move(dset); - } - } - - std::unordered_map> datasets_; - std::unordered_set struct_tables_; - H5FResource hfile_; - std::string db_full_filename_; -}; - -HDF5ConnProxy::HDF5ConnProxy() : impl_(new HDF5ConnProxy::Impl) -{ -} - -HDF5ConnProxy::~HDF5ConnProxy() -{ -} - -//! \brief This validate method gets called when a schema is -//! given to an ObjectManager to use with an HDF5 connection. -//! -//! \param schema Schema object to validate -void HDF5ConnProxy::validateSchema(const Schema & schema) const -{ - std::unordered_set struct_tables; - - for (const auto & tbl : schema) { - const auto num_columns = std::distance(tbl.begin(), tbl.end()); - if (num_columns == 0) { - continue; - } - - bool first_col_evaluated = false; - bool first_col_is_struct_field = false; - - for (const auto & col : tbl) { - const size_t prod = std::accumulate( - col->getDimensions().begin(), - col->getDimensions().end(), - 1, std::multiplies()); - - if (prod == 0) { - throw DBException("Invalid dimensions encountered in ") - << "HDF5 schema (0 is not allowed). Found in table " - << tbl.getName() << ":" << getColumnDTypeStr(*col); - } else if (col->getDataType() == ColumnDataType::blob_t) { - throw DBException("Invalid data type encountered in ") - << "HDF5 schema. Blob data types are not supported. " - << "Found in table " << tbl.getName() << ":" - << getColumnDTypeStr(*col); - } else if (col->getDataType() == ColumnDataType::string_t) { - throw DBException("Invalid data type encountered in ") - << "HDF5 schema. String data types are not supported. " - << "Found in table " << tbl.getName() << ":" - << getColumnDTypeStr(*col); - } - - if (first_col_evaluated) { - if (col->hasByteOffset() != first_col_is_struct_field) { - throw DBException("Table encountered which has column(s) ") - << "set as a field of a struct, and column(s) which are " - << "not defined as part of a struct"; - } - } else { - first_col_is_struct_field = col->hasByteOffset(); - first_col_evaluated = true; - } - } - - if (first_col_is_struct_field) { - struct_tables.insert(tbl.getName()); - } - } - - impl_->setStructTables(std::move(struct_tables)); -} - -void HDF5ConnProxy::realizeSchema( - const Schema & schema, - const ObjectManager & obj_mgr) -{ - impl_->realizeSchema(schema, obj_mgr); -} - -bool HDF5ConnProxy::connectToExistingDatabase(const std::string & db_file) -{ - return impl_->connectToExistingDatabase(db_file); -} - -std::string HDF5ConnProxy::getDatabaseFullFilename() const -{ - return impl_->getDatabaseFullFilename(); -} - -bool HDF5ConnProxy::isValid() const -{ - return impl_->isValid(); -} - -std::string HDF5ConnProxy::openDbFile_( - const std::string & db_dir, - const std::string & db_file, - const bool create_file) -{ - return impl_->openDbFile(db_dir, db_file, create_file); -} - -bool HDF5ConnProxy::hasObjectImpl_( - const std::string & table_name, - const DatabaseID db_id) const -{ - return impl_->hasObject(table_name, db_id); -} - -size_t HDF5ConnProxy::readRawBytes( - const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - void * dest_ptr, - const size_t num_bytes) const -{ - return impl_->readRawBytes( - table_name, prop_name, db_id, dest_ptr, num_bytes); -} - -AnySizeObjectFactory HDF5ConnProxy::getObjectFactoryForTable( - const std::string &) const -{ - return +[](DbConnProxy * db_proxy, - const std::string & table_name, - const ColumnValues & values) - { - //Take the incoming column values, put them into - //a contiguous vector of raw bytes, and call the - //"fixed-size" factory method to create the object - //with these column values. - std::vector raw_bytes; - - for (const auto & col : values) { - const size_t cur_num_bytes = raw_bytes.size(); - - const size_t elm_num_bytes = - getFixedNumBytesForColumnDType(col.getDataType()); - - const size_t new_num_bytes = cur_num_bytes + elm_num_bytes; - - raw_bytes.resize(new_num_bytes); - memcpy(&raw_bytes[cur_num_bytes], - col.getDataPtr(), - elm_num_bytes); - } - - return static_cast(db_proxy)-> - createFixedSizeObject(table_name, &raw_bytes[0], raw_bytes.size()); - }; -} - -FixedSizeObjectFactory HDF5ConnProxy::getFixedSizeObjectFactoryForTable( - const std::string &) const -{ - return +[](DbConnProxy * db_proxy, - const std::string & table_name, - const void * raw_bytes_ptr, - const size_t num_raw_bytes) - { - return static_cast(db_proxy)-> - createFixedSizeObject(table_name, raw_bytes_ptr, num_raw_bytes); - }; -} - -DatabaseID HDF5ConnProxy::createObject( - const std::string &, - const ColumnValues &) -{ - //Until the HDF5 SimDB implementation supports column - //data types that are variable in length (such as strings), - //this method should never be getting called. - throw DBException("Not implemented"); -} - -DatabaseID HDF5ConnProxy::createFixedSizeObject( - const std::string & table_name, - const void * raw_bytes_ptr, - const size_t num_raw_bytes) -{ - return impl_->createFixedSizeObject(table_name, raw_bytes_ptr, num_raw_bytes); -} - -} // namespace simdb diff --git a/sparta/simdb/src/ObjectManager.cpp b/sparta/simdb/src/ObjectManager.cpp deleted file mode 100644 index e3878ca5a4..0000000000 --- a/sparta/simdb/src/ObjectManager.cpp +++ /dev/null @@ -1,868 +0,0 @@ -// -*- C++ -*- - -//SimDB headers -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/TableProxy.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/utils/uuids.hpp" -#include "simdb/Errors.hpp" -#include "simdb/async/AsyncTaskEval.hpp" - -//SQLite-specific headers -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include "simdb/impl/sqlite/Schema.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include - -//HDF5-specific headers -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" - -//Standard headers -#include -#include - -namespace simdb { - -/*! - * \brief Take all of the fully qualified table names an - * ObjectManager has, split them around the table namespace - * delimiter, and return a mapping from unqualified table - * name to its associated namespace(s), and vice versa. - */ -static void parseTableNamespaces( - const std::unordered_set & full_table_names, - std::unordered_map> & tables_by_namespace, - std::unordered_map> & namespaces_by_table) -{ - for (const auto & full_table_name : full_table_names) { - auto delim_idx = full_table_name.find(std::string(1, Table::NS_DELIM)); - if (delim_idx != std::string::npos && delim_idx + 1 < full_table_name.size()) { - const std::string namespace_name = full_table_name.substr(0, delim_idx); - const std::string unqualified_table_name = full_table_name.substr(delim_idx + 1); - tables_by_namespace[namespace_name].insert(unqualified_table_name); - namespaces_by_table[unqualified_table_name].insert(namespace_name); - } - } -} - -//! RAII used for beginTransaction()/commitTransaction() -//! calls into the DbConnProxy class. -struct ScopedTransaction { - ScopedTransaction(const DbConnProxy * db_proxy, - ObjectManager::TransactionFunc & transaction, - bool & in_transaction_flag) : - db_proxy_(db_proxy), - transaction_(transaction), - in_transaction_flag_(in_transaction_flag) - { - in_transaction_flag_ = true; - db_proxy_->beginAtomicTransaction(); - transaction_(); - } - - ~ScopedTransaction() - { - db_proxy_->commitAtomicTransaction(); - in_transaction_flag_ = false; - } - -private: - //Open database connection - const DbConnProxy * db_proxy_ = nullptr; - - //The caller's function they want inside BEGIN/COMMIT - ObjectManager::TransactionFunc & transaction_; - - //The caller's "in transaction flag" - in case they - //need to know whether *their code* is already in - //an ongoing transaction: - // - // void MyObj::callSomeSQL(DbConnProxy * db_proxy) { - // if (!already_in_transaction_) { - // ScopedTransaction(db_proxy, - // [&](){ eval_sql(db_proxy, "INSERT INTO Customers ..."); }, - // already_in_transaction_); - // - // //Now call another method which MIGHT - // //call this "callSomeSQL()" method again: - // callFooBarFunction_(); - // } else { - // eval_sql(db_proxy, "INSERT INTO Customers ..."); - // } - // } - // - //The use of this flag lets functions like MyObj::callSomeSQL() - //be safely called recursively. Without it, "BEGIN TRANSACTION" - //could get called a second time like this: - // - // BEGIN TRANSACTION - // INSERT INTO Customers ... - // BEGIN TRANSACTION <-- SQLite will error! - // (was expecting COMMIT TRANSACTION before - // seeing this again) - bool & in_transaction_flag_; -}; - -//Helper to get rid of 'unused variable' errors and keep the code tidy -#define LOCAL_TRANSACTION(db_proxy, transaction, transaction_flag) \ - ScopedTransaction raii(db_proxy, transaction, transaction_flag); \ - (void) raii; - -//! Database files are currently given a random file name, like: -//! 345l34-gu345lkj-234lsdf-kjh892y.db -//! -//! Users only have control over the directory where the database -//! should live, but not the file name. -static std::string generateRandomDatabaseFilename( - const std::string & extension) -{ - return generateUUID() + extension; -} - -ObjectManager::ObjectManager(const std::string & db_dir) : - db_dir_(db_dir), - task_queue_(new AsyncTaskEval), - warning_log_(db_dir + "/database.warn") -{ -} - -void ObjectManager::addToTaskController( - AsyncTaskController * controller) -{ - task_queue_->setSimulationDatabase(this); - task_queue_->addToTaskController(controller); - task_controller_ = controller; -} - -void ObjectManager::captureTableSummaries() -{ - if (db_proxy_) { - safeTransaction([&]() { - const auto & summary_source_tables = schema_.summary_query_info_structs_.source_tables; - for (const auto & summary_table : summary_source_tables) { - const std::string & source_table_name = summary_table.table_name; - if (!schema_.shouldSummarizeTable_(source_table_name)) { - continue; - } - getTable(source_table_name)->captureSummary(); - } - }); - } -} - -ObjectManager::~ObjectManager() -{ -} - -void ObjectManager::openDatabaseWithoutSchema_() -{ - assertNoDatabaseConnectionOpen_(); - const std::string extension = db_proxy_->getDatabaseFileExtension(); - const std::string db_file = generateRandomDatabaseFilename(extension); - openDbFile_(db_file, true); -} - -void ObjectManager::assertNoDatabaseConnectionOpen_() const -{ - if (db_proxy_ == nullptr) { - return; - } - - //For now, we only allow one ObjectManager owning - //one SimDB connection. This method is called in - //several places where we need to make sure a user - //isn't accidentally trying to open a new connection - //when we already have one opened. - if (db_proxy_->isValid()) { - throw DBException( - "A database connection has already been " - "made for this ObjectManager"); - } -} - -//Ask the proxy object to give us the table names in -//the database. These are cached in memory after that -//for performance reasons. -void ObjectManager::getDatabaseTableNames_() const -{ - if (!table_names_.empty()) { - return; - } - db_proxy_->getTableNames(table_names_); - - if (table_names_.empty()) { - table_names_ = default_table_names_; - default_table_names_.clear(); - } -} - -bool ObjectManager::connectToExistingDatabase(const std::string & db_file) -{ - assertNoDatabaseConnectionOpen_(); - - bool connected = false; - auto slash = db_file.find_last_of("."); - - if (slash < db_file.size()) { - //Use the file extension to take a best guess on the - //database format. The DbConnProxy subclasses will - //verify the file for us. - const std::string extension = db_file.substr(slash); - if (extension == ".db") { - db_proxy_.reset(new SQLiteConnProxy); - connected = db_proxy_->connectToExistingDatabase(db_file); - } - if (!connected && extension == ".h5") { - db_proxy_.reset(new HDF5ConnProxy); - connected = db_proxy_->connectToExistingDatabase(db_file); - } - if (!connected && extension == ".todo") { - //todo (other implementations get added here) - } - } else { - //There is no file extension. Go through each database - //implementation that SimDB supports until we find one - //that verifies the file format. - db_proxy_.reset(new SQLiteConnProxy); - connected = db_proxy_->connectToExistingDatabase(db_file); - - if (!connected) { - db_proxy_.reset(new HDF5ConnProxy); - connected = db_proxy_->connectToExistingDatabase(db_file); - } - if (!connected) { - //todo (other implementations get added here) - } - } - - if (!connected) { - db_proxy_.reset(); - db_full_filename_.clear(); - return false; - } - - db_full_filename_ = db_proxy_->getDatabaseFullFilename(); - return true; -} - -const std::string & ObjectManager::getDatabaseFile() const -{ - return !db_full_filename_.empty() ? db_full_filename_ : db_dir_; -} - -//! This mutex is also guarding the 'is_in_transaction_' variable, -//! which is not protected against recursive calls in the same -//! way that ScopedTransaction is. This needs to be a recursive -//! mutex since safeTransaction() often gets called recursively. -//! -//! As long as this mutex is here and everyone is going through -//! safeTransaction() to make database calls, we will put as much -//! of the database work that we can through the worker thread / -//! simdb::AsyncTaskEval so the vast majority of the database calls -//! don't have to wait on this mutex. The only forced synchronous -//! flush we need today is at simulation end, and to do that we -//! will just put an interrupt task in the queue and wait for the -//! interrupt to be issued... after all of our pending database -//! queries / inserts have already run. -static std::recursive_mutex obj_mgr_transaction_mutex; - -void ObjectManager::safeTransaction(TransactionFunc transaction) const -{ - const DbConnProxy * db_proxy = getDbConn(); - - //There are "normal" or "acceptable" SQLite errors that - //we trap: SQLITE_BUSY (the database file is locked), and - //SQLITE_LOCKED (a table in the database is locked). These - //can occur when SQLite is used in concurrent systems, and - //are not necessarily "real" errors. - // - //If these *specific* types of errors occur, we will catch - //them and keep retrying the transaction until successful. - //This is part of what is meant by a "safe" transaction. - //Database transactions will not fail due to concurrent - //access errors that are not always obvious from a SPARTA - //user/developer's perspective. - - while (true) { - try { - //More thought needs to go into thread safety of the - //database writes/reads. Let's be super lazy and grab - //a mutex right here for the time being. - std::lock_guard lock(obj_mgr_transaction_mutex); - - //Check to see if we are already in a transaction, in which - //case we simply call the transaction function. We cannot - //call "BEGIN TRANSACTION" recursively. - if (is_in_transaction_ || !db_proxy->supportsAtomicTransactions()) { - transaction(); - } else { - LOCAL_TRANSACTION(db_proxy, transaction, is_in_transaction_) - } - - //We got this far without an exception, which means - //that the proxy's commitAtomicTransaction() method - //has been called (if it supports atomic transactions). - break; - - //Retry transaction due to database access errors - } catch (const DBAccessException & ex) { - warning_log_ << ex.what() << "\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(25)); - continue; - } - - //Note that other std::exceptions are still being thrown, - //and may abort the simulation - } -} - -bool ObjectManager::createDatabaseFromSchema( - Schema & schema, - std::unique_ptr db_proxy) -{ - if (db_proxy == nullptr) { - return false; - } - - schema.finalizeSchema_(); - db_proxy->validateSchema(schema); - db_proxy_ = std::move(db_proxy); - schema_ = schema; - - for (const auto & table : schema) { - default_table_names_.insert(table.getName()); - if (table.isFixedSize()) { - fixed_size_tables_.insert(table.getName()); - } - } - - openDatabaseWithoutSchema_(); - db_proxy_->realizeSchema(schema, *this); - - if (db_proxy_->isValid()) { - getAndStoreDatabaseID_(); - return true; - } - return false; -} - -bool ObjectManager::appendSchema(Schema & schema) -{ - if (db_proxy_ == nullptr) { - return false; - } else if (!db_proxy_->isValid()) { - throw DBException("Attempt to append schema tables to ") - << "an ObjectManager that does not have a valid " - << "database connection"; - } - - schema.finalizeSchema_(); - db_proxy_->validateSchema(schema); - - for (const auto & table : schema) { - const std::string table_name = table.getName(); - if (!table_names_.empty()) { - table_names_.insert(table_name); - } - default_table_names_.insert(table_name); - if (table.isFixedSize()) { - fixed_size_tables_.insert(table_name); - } - } - - db_proxy_->realizeSchema(schema, *this); - return true; -} - -std::string ObjectManager::getStatsTableName_( - const std::string & table_name) const -{ - auto qualified_table_name = getQualifiedTableName(table_name); - if (qualified_table_name.empty()) { - qualified_table_name = getQualifiedTableName(table_name, "Stats"); - } - return qualified_table_name; -} - -std::string ObjectManager::getQualifiedTableName( - const std::string & table_name, - const utils::lowercase_string & namespace_hint) const -{ - { - auto outer_iter = cached_qualified_table_names_.find(table_name); - if (outer_iter != cached_qualified_table_names_.end()) { - auto inner_iter = outer_iter->second.find(namespace_hint); - if (inner_iter != outer_iter->second.end()) { - return inner_iter->second; - } - } - } - - auto cache_qualified_table_name = [&](const std::string & qualified) { - auto & ret = cached_qualified_table_names_[table_name][namespace_hint]; - ret = qualified; - return ret; - }; - - const auto & table_names = getTableNames(); - const std::string & table_name_as_is = table_name; - if (table_names.count(table_name_as_is)) { - return cache_qualified_table_name(table_name_as_is); - } - - std::unordered_map> tables_by_namespace; - std::unordered_map> namespaces_by_table; - parseTableNamespaces(table_names, tables_by_namespace, namespaces_by_table); - - if (namespace_hint.empty()) { - auto iter = namespaces_by_table.find(table_name); - if (iter == namespaces_by_table.end()) { - return ""; - } - if (iter->second.size() == 1) { - auto possible = *(iter->second.begin()) + Table::NS_DELIM + table_name; - if (table_names.count(possible)) { - return cache_qualified_table_name(possible); - } - } - return ""; - } - - std::string hint = namespace_hint; - hint += std::string(1, Table::NS_DELIM); - hint += table_name; - if (table_names.count(hint)) { - return hint; - } - - return ""; -} - -std::unique_ptr ObjectManager::ObjectDatabase::getTable( - const std::string & table_name) const -{ - if (db_namespace_.empty()) { - return sim_db_->getTable(table_name); - } - - return sim_db_->getTable( - db_namespace_ + Table::NS_DELIM + table_name); -} - -TableProxy * ObjectManager::ObjectDatabase::getConditionalTable( - const std::string & table_name) const -{ - const std::string name = - db_namespace_.empty() ? table_name : - db_namespace_ + Table::NS_DELIM + table_name; - - auto iter = table_proxies_.find(name); - if (iter != table_proxies_.end()) { - return iter->second.get(); - } - - std::shared_ptr proxy(new TableProxy( - name, *sim_db_, db_namespace_obj_)); - table_proxies_[name] = proxy; - return proxy.get(); -} - -void ObjectManager::ObjectDatabase::grantAccess() { - for (auto & proxy : table_proxies_) { - proxy.second->grantAccess(); - } - access_granted_ = true; -} - -void ObjectManager::ObjectDatabase::revokeAccess() { - for (auto & proxy : table_proxies_) { - proxy.second->revokeAccess(); - } - access_granted_ = false; -} - -std::unique_ptr ObjectManager::getTable_( - const std::string & table_name) const -{ - if (table_name.empty()) { - return nullptr; - } - - //Ask the database for its table names, and - //cache them in memory - getDatabaseTableNames_(); - - //Return null if this is not a table in this database - auto iter = table_names_.find(table_name); - if (iter == table_names_.end()) { - return nullptr; - } - - //Table name is valid. Return a wrapper around this table. - AnySizeObjectFactory any_size_factory; - FixedSizeObjectFactory fixed_size_factory; - - auto fixed_iter = fixed_size_tables_.find(table_name); - if (fixed_iter != fixed_size_tables_.end()) { - auto fixed_size_factory_iter = fixed_size_record_factories_.find(table_name); - if (fixed_size_factory_iter == fixed_size_record_factories_.end()) { - fixed_size_factory = db_proxy_-> - getFixedSizeObjectFactoryForTable(table_name); - - if (!fixed_size_factory) { - fixed_size_tables_.erase(table_name); - } else { - fixed_size_record_factories_[table_name] = fixed_size_factory; - } - } else { - fixed_size_factory = fixed_size_factory_iter->second; - } - } - - auto any_iter = any_size_record_factories_.find(table_name); - if (any_iter == any_size_record_factories_.end()) { - any_size_factory = db_proxy_-> - getObjectFactoryForTable(table_name); - - if (any_size_factory) { - any_size_record_factories_[table_name] = any_size_factory; - } - } else { - any_size_factory = any_iter->second; - } - - NamedSummaryFunctions summary_fcns; - std::vector col_metadata; - for (const auto & tbl : schema_.summary_query_info_structs_.source_tables) { - if (tbl.table_name == table_name) { - col_metadata = tbl.table_columns; - summary_fcns = schema_.summary_query_info_structs_.summary_fcns; - break; - } - } - - return std::unique_ptr(new TableRef( - table_name, *this, db_proxy_, - col_metadata, - summary_fcns, - any_size_factory, - fixed_size_factory)); -} - -const std::unordered_set & - ObjectManager::ObjectDatabase::getTableNames() const -{ - if (!table_names_.empty()) { - return table_names_; - } - - if (db_namespace_.empty()) { - table_names_.clear(); - return table_names_; - } - - table_names_.clear(); - auto table_names = sim_db_->getTableNames_(); - const std::string target_str = db_namespace_ + Table::NS_DELIM; - - for (const auto & table_name : table_names) { - if (table_name.find(target_str) == 0) { - table_names_.insert(table_name.substr(target_str.size())); - } - } - return table_names_; -} - -const std::unordered_set & ObjectManager::getTableNames_() const -{ - getDatabaseTableNames_(); - return table_names_; -} - -void ObjectManager::getAndStoreDatabaseID_() -{ - if (uuid_ > 0) { - return; - } - if (db_proxy_ == nullptr) { - return; - } - - if (!db_proxy_->isValid()) { - throw DBException("There is no database connection yet. ") - << "The ObjectManager::getAndStoreDatabaseID_() method " - << "cannot be called."; - } - - safeTransaction([&]() { - //TODO: For custom-defined schemas, this table probably - //will not exist. We should think about whether we can - //safely add this table to these custom schemas. For now, - //this UUID is only being used for SI/report-related - //database work, i.e. using the default provided schema. - //We should be able to safely warn and early return. - auto obj_mgr_table_name = getQualifiedTableName( - "ObjectManagersInDatabase", "Stats"); - auto obj_mgr_uuids_tbl = getTable_(obj_mgr_table_name); - if (obj_mgr_uuids_tbl == nullptr) { - if (warnings_enabled_) { - std::cout << "Custom SimDB schema detected. You will not " - << "be able to make use of the ObjectManager::getId() " - << "method for anything useful; all ObjectManager " - << "connections made to this schema will return " - << "0 if getId() is called." - << std::endl; - } - return; - } - - simdb::ObjectQuery query(*this, "ObjectManagersInDatabase"); - - int32_t obj_mgr_id = 0; - query.writeResultIterationsTo("ObjMgrID", &obj_mgr_id); - - //We are looking for the max ObjMgrID in this database, - //and we'll take the ID that is 1 greater than it. - query.orderBy(OrderBy("ObjMgrID", DESC)); - query.setLimit(1); - auto result_iter = query.executeQuery(); - result_iter->getNext(); - - //Just increment the maximum existing UUID by 1 and add - //an entry to this table accordingly. - uuid_ = obj_mgr_id + 1; - - obj_mgr_uuids_tbl->createObjectWithArgs("ObjMgrID", uuid_); - }); -} - -std::unique_ptr ObjectManager::ObjectDatabase::findObject( - const std::string & table_name, - const DatabaseID db_id) const -{ - if (db_namespace_.empty()) { - return sim_db_->findObject(table_name, db_id); - } - return sim_db_->findObject( - db_namespace_ + Table::NS_DELIM + table_name, db_id); -} - -std::unique_ptr ObjectManager::findObject_( - const std::string & table_name, - const DatabaseID db_id) const -{ - if (db_proxy_ == nullptr) { - return nullptr; - } - - if (!db_proxy_->supportsObjectQuery_()) { - if (db_proxy_->hasObject_(table_name, db_id)) { - return std::unique_ptr( - new ObjectRef(*this, table_name, db_id)); - } else { - return nullptr; - } - } - - //We *could* first check if the 'table_name' is even - //in our set of known tables. We could return null in - //that case. But an object should really be unfound - //if the *database ID* was not found, NOT because the - //table name wasn't even legit. Let's not take the - //small performance hit of the map/set lookup, and - //just let SQLite hard error if the table name is - //bad. This is probably a bug anyway. - - //Try to find the record in that table whose 'Id' - //(primary key) is the one we're looking for. - ObjectQuery query(*this, table_name); - query.addConstraints("Id", constraints::equal, db_id); - - //This is only considered a "found" record if we found - //exactly one record with this Id. Since this is a primary - //key, we could also assert that it is either 0 (not found) - //or 1 (found). - std::unique_ptr obj_ref; - if (query.countMatches() == 1) { - obj_ref.reset(new ObjectRef(*this, table_name, db_id)); - } - return obj_ref; -} - -void ObjectManager::ObjectDatabase::findObjects( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const -{ - if (db_namespace_.empty()) { - sim_db_->findObjects(table_name, db_ids, obj_refs); - } else { - sim_db_->findObjects( - db_namespace_ + Table::NS_DELIM + table_name, db_ids, obj_refs); - } -} - -void ObjectManager::findObjects_( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const -{ - obj_refs.clear(); - - if (db_proxy_ == nullptr) { - return; - } - - ObjectQuery query(*this, table_name); - if (!db_ids.empty()) { - query.addConstraints("Id", constraints::in_set, db_ids); - } - - DatabaseID found_id; - query.writeResultIterationsTo("Id", &found_id); - - std::set found_ids; - auto result_iter = query.executeQuery(); - while (result_iter->getNext()) { - found_ids.insert(found_id); - } - - if (!db_ids.empty()) { - obj_refs.reserve(db_ids.size()); - for (const auto db_id : db_ids) { - if (found_ids.count(db_id) > 0) { - obj_refs.emplace_back(new ObjectRef(*this, table_name, db_id)); - } else { - obj_refs.emplace_back(nullptr); - } - } - } else { - obj_refs.reserve(found_ids.size()); - for (const auto db_id : found_ids) { - obj_refs.emplace_back(new ObjectRef(*this, table_name, db_id)); - } - } -} - -std::unique_ptr ObjectManager::ObjectDatabase:: - createObjectQueryForTable(const std::string & table_name) const -{ - auto sim_db = getObjectManager(); - if (!sim_db) { - return nullptr; - } - - auto qualified_table_name = sim_db-> - getQualifiedTableName(table_name, db_namespace_); - - if (qualified_table_name.empty()) { - return nullptr; - } - - return std::unique_ptr(new ObjectQuery( - *sim_db, qualified_table_name)); -} - -AsyncTaskEval* ObjectManager::ObjectDatabase:: - getTaskQueue() const -{ - auto sim_db = getObjectManager(); - if (!sim_db) { - return nullptr; - } - return sim_db->getTaskQueue(); -} - -const DbConnProxy * ObjectManager::getDbConn() const -{ - return db_proxy_.get(); -} - -bool ObjectManager::openDbFile_(const std::string & db_file, - const bool create_file) -{ - if (db_proxy_ == nullptr) { - return false; - } - - const std::string db_proxy_filename = - db_proxy_->openDbFile_(db_dir_, db_file, create_file); - - if (!db_proxy_filename.empty()) { - //File opened without issue. Store the full DB filename. - db_full_filename_ = db_proxy_filename; - return true; - } - return false; -} - -//! ------ DEPRECATED. ----------------------------------------- -//! For backwards compatibility only. -//! May be removed in a future release. -//! ------------------------------------------------------------ -std::unique_ptr ObjectManager::getTable( - const std::string & table_name) const -{ - std::string qualified_table_name = - getQualifiedTableName(table_name); - - if (qualified_table_name.empty()) { - qualified_table_name = getQualifiedTableName(table_name, "Stats"); - } - if (qualified_table_name.empty()) { - return nullptr; - } - return getTable_(qualified_table_name); -} - -//! -------------------- (... deprecated ...) -------------------- -const std::unordered_set & ObjectManager::getTableNames() const -{ - return getTableNames_(); -} - -//! -------------------- (... deprecated ...) -------------------- -std::unique_ptr ObjectManager::findObject( - const std::string & table_name, - const DatabaseID db_id) const -{ - if (!db_proxy_->supportsObjectQuery_()) { - if (db_proxy_->hasObject_(table_name, db_id)) { - return std::unique_ptr( - new ObjectRef(*this, table_name, db_id)); - } - return nullptr; - } - - auto table = getTable_(table_name); - if (!table) { - table = getTable_(getStatsTableName_(table_name)); - if (!table) { - return nullptr; - } - return findObject_(getStatsTableName_(table_name), db_id); - } - return findObject_(table_name, db_id); -} - -//! -------------------- (... deprecated ...) -------------------- -void ObjectManager::findObjects( - const std::string & table_name, - const std::vector & db_ids, - std::vector> & obj_refs) const -{ - auto table = getTable_(table_name); - if (!table) { - table = getTable_(getStatsTableName_(table_name)); - if (!table) { - return; - } - findObjects_(getStatsTableName_(table_name), db_ids, obj_refs); - } - findObjects_(table_name, db_ids, obj_refs); -} -//! ---------------------- (end DEPRECATED) ---------------------- - -} // namespace simdb diff --git a/sparta/simdb/src/ObjectRef.cpp b/sparta/simdb/src/ObjectRef.cpp deleted file mode 100644 index 0cd16cea21..0000000000 --- a/sparta/simdb/src/ObjectRef.cpp +++ /dev/null @@ -1,397 +0,0 @@ -// -*- C++ -*- - -#include "simdb/ObjectRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/utils/CompatUtils.hpp" - -//SQLite-specific headers -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include - -namespace simdb { - -//! Local helper which sets a record's scalar property. -//! All numeric properties get set through this method. -template -typename std::enable_if< - std::is_fundamental::value, -void>::type -LOCAL_setScalarProperty(const std::string & table_name, - const std::string & prop_name, - const PropertyDataT prop_value, - const DatabaseID db_id, - const ObjectManager & obj_mgr) -{ - auto table = obj_mgr.getTable(table_name); - - if (!table->updateRowValues(prop_name.c_str(), prop_value). - forRecordsWhere("Id", constraints::equal, db_id)) - { - throw DBException("Unable to write database property '") - << table_name << "::" << prop_name << "' for record " - << "with Id " << db_id << ". Error occurred in database " - << "file '" << obj_mgr.getDatabaseFile() << "'."; - } -} - -//! Local helper which sets a record's scalar property. -//! Specialization for std::string so we can pass it -//! in by const ref. -template -typename std::enable_if< - std::is_same::value, -void>::type -LOCAL_setScalarProperty(const std::string & table_name, - const std::string & prop_name, - const PropertyDataT & prop_value, - const DatabaseID db_id, - const ObjectManager & obj_mgr) -{ - auto table = obj_mgr.getTable(table_name); - - if (!table->updateRowValues(prop_name.c_str(), prop_value). - forRecordsWhere("Id", constraints::equal, db_id)) - { - throw DBException("Unable to write database property '") - << table_name << "::" << prop_name << "' for record " - << "with Id " << db_id << ". Error occurred in database " - << "file '" << obj_mgr.getDatabaseFile() << "'."; - } -} - -#define VERIFY_OBJECT_QUERY_SUPPORT \ - if (!obj_mgr_.getDbConn()->supportsObjectQuery_()) { \ - throw DBException("ObjectQuery is not supported."); \ - } - -#define HAS_OBJECT_QUERY_SUPPORT \ - obj_mgr_.getDbConn()->supportsObjectQuery_() - -//! Local helper which gets a record's scalar property. -template -PropertyDataT LOCAL_getScalarProperty(const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - const ObjectManager & obj_mgr) -{ - ObjectQuery query(obj_mgr, table_name); - - PropertyDataT prop_value{}; - query.writeResultIterationsTo(prop_name.c_str(), &prop_value); - query.addConstraints("Id", constraints::equal, db_id); - - if (!query.executeQuery()->getNext()) - { - throw DBException("Unable to read database property '") - << table_name << "::" << prop_name << "' for record " - << "with Id " << db_id << ". Error occurred in database " - << "file '" << obj_mgr.getDatabaseFile() << "'."; - } - - return prop_value; -} - -//! Local helper which gets a record's scalar property -//! without using ObjectQuery. -template -typename std::enable_if< - utils::is_pod::value, -PropertyDataT>::type -LOCAL_getScalarPropertyNoObjectQuery( - const std::string & table_name, - const std::string & prop_name, - const DatabaseID db_id, - const ObjectManager & obj_mgr) -{ - const DbConnProxy * db_proxy = obj_mgr.getDbConn(); - - PropertyDataT prop_value; - const size_t bytes_read = db_proxy->readRawBytes( - table_name, prop_name, db_id, - &prop_value, sizeof(PropertyDataT)); - - if (bytes_read != sizeof(PropertyDataT)) { - throw DBException("DbConnProxy::readRawBytes() failed"); - } - return prop_value; -} - -const ObjectManager & ObjectRef::getObjectManager() const -{ - return obj_mgr_; -} - -DatabaseID ObjectRef::getId() const -{ - return db_id_; -} - -void ObjectRef::setPropertyInt8( - const std::string & prop_name, - const int8_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyUInt8( - const std::string & prop_name, - const uint8_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyInt16( - const std::string & prop_name, - const int16_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyUInt16( - const std::string & prop_name, - const uint16_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyInt32( - const std::string & prop_name, - const int32_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyUInt32( - const std::string & prop_name, - const uint32_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyInt64( - const std::string & prop_name, - const int64_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyUInt64( - const std::string & prop_name, - const uint64_t prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyString( - const std::string & prop_name, - const std::string & prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyChar( - const std::string & prop_name, - const char prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyFloat( - const std::string & prop_name, - const float prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyDouble( - const std::string & prop_name, - const double prop_value) -{ - LOCAL_setScalarProperty( - table_name_, prop_name, prop_value, db_id_, obj_mgr_); -} - -void ObjectRef::setPropertyBlob( - const std::string & prop_name, - const Blob & prop_value) -{ - auto table = obj_mgr_.getTable(table_name_); - - if (!table->updateRowValues(prop_name.c_str(), prop_value). - forRecordsWhere("Id", constraints::equal, db_id_)) - { - throw DBException("Unable to write database property '") - << table_name_ << "::" << prop_name << "' for record " - << "with Id " << db_id_ << ". Error occurred in database " - << "file '" << obj_mgr_.getDatabaseFile() << "'."; - } -} - -int8_t ObjectRef::getPropertyInt8(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -uint8_t ObjectRef::getPropertyUInt8(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -int16_t ObjectRef::getPropertyInt16(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -uint16_t ObjectRef::getPropertyUInt16(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -int32_t ObjectRef::getPropertyInt32(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -uint32_t ObjectRef::getPropertyUInt32(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -int64_t ObjectRef::getPropertyInt64(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -uint64_t ObjectRef::getPropertyUInt64(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -std::string ObjectRef::getPropertyString(const std::string & prop_name) const -{ - VERIFY_OBJECT_QUERY_SUPPORT - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); -} - -char ObjectRef::getPropertyChar(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -float ObjectRef::getPropertyFloat(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -double ObjectRef::getPropertyDouble(const std::string & prop_name) const -{ - if (HAS_OBJECT_QUERY_SUPPORT) { - return LOCAL_getScalarProperty( - table_name_, prop_name, db_id_, obj_mgr_); - } - return LOCAL_getScalarPropertyNoObjectQuery( - table_name_, prop_name, db_id_, obj_mgr_); -} - -void ObjectRef::prepGetPropertyBlob_( - const std::string & prop_name, - void ** statement, - Blob & blob_descriptor) const -{ - VERIFY_OBJECT_QUERY_SUPPORT - sqlite3_stmt * stmt_retrieve; - - std::ostringstream oss; - oss << " SELECT " << prop_name << " FROM " << table_name_ - << " WHERE Id = " << db_id_; - const std::string command = oss.str(); - - //Create the prepared statement for this blob - obj_mgr_.getDbConn()->prepareStatement_(command, statement); - assert(statement != nullptr); - stmt_retrieve = static_cast(*statement); - - //Execute the prepared statement - if (sqlite3_step(stmt_retrieve) != SQLITE_ROW) { - throw DBException("Error getting property '") << prop_name << "' for SQL table '" - << table_name_ << "'"; - } - - //Notice the output argument for this function. We aren't - //actually getting the blob values right here, we are just - //getting the blob descriptor (void*, and number of bytes). - blob_descriptor.data_ptr = sqlite3_column_blob(stmt_retrieve, 0); - blob_descriptor.num_bytes = static_cast(sqlite3_column_bytes(stmt_retrieve, 0)); -} - -void ObjectRef::finalizeGetPropertyBlob_(void * statement) const -{ - //Destroy the prepared statement - we are done with it. - sqlite3_finalize(static_cast(statement)); -} - -} // namespace simdb diff --git a/sparta/simdb/src/SQLiteConnection.cpp b/sparta/simdb/src/SQLiteConnection.cpp deleted file mode 100644 index 4866801ce2..0000000000 --- a/sparta/simdb/src/SQLiteConnection.cpp +++ /dev/null @@ -1,1065 +0,0 @@ -// -*- C++ -*- - -#include "simdb/ObjectManager.hpp" -#include "simdb/utils/MathUtils.hpp" -#include "simdb/Errors.hpp" - -//SQLite-specific headers -#include "simdb/impl/sqlite/Schema.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include - -//Standard headers -#include -#include - -namespace simdb { - -//! Local utility to turn any 8, 16, or 32 bit integer -//! column value into an int32_t -int getColumnValueAsInt32(const ColumnValueBase & col) -{ - using dt = ColumnDataType; - - switch (col.getDataType()) { - case dt::char_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::int8_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::uint8_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::int16_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::uint16_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::int32_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::uint32_t: { - const auto val = col.getAs(); - return static_cast(val); - } - default: - throw DBException("Invalid call to getColumnValueAsInt32() ") - << "- the ColumnValueBase object passed in has a value " - << "that cannot be cast to 32-bit int."; - } -} - -//! Local utility to turn any 64 bit integer column value -//! into an int64_t -sqlite3_int64 getColumnValueAsInt64(const ColumnValueBase & col) -{ - using dt = ColumnDataType; - - switch (col.getDataType()) { - case dt::int64_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::uint64_t: { - const auto val = col.getAs(); - return static_cast(val); - } - default: - throw DBException("Invalid call to getColumnValueAsInt64() ") - << "- the ColumnValueBase object passed in has a value " - << "that cannot be cast to 64-bit int."; - } -} - -//! Local utility to turn any floating point column value -//! into a double -double getColumnValueAsDouble(const ColumnValueBase & col) -{ - using dt = ColumnDataType; - - switch (col.getDataType()) { - case dt::float_t: { - const auto val = col.getAs(); - return static_cast(val); - } - case dt::double_t: { - const auto val = col.getAs(); - return static_cast(val); - } - default: - throw DBException("Invalid call to getColumnValueAsDouble() ") - << "- the ColumnValueBase object passed in has a value " - << "that cannot be cast to double."; - } -} - -//! Local helper method used by INSERT and UPDATE code in -//! SQLiteConnProxy code below. -void LOCAL_finalizeInsertOrUpdateStatement( - sqlite3_stmt * prepared_stmt, - const ColumnValues & col_values) -{ - int rc = SQLITE_OK; - auto check_sql = [&rc]() { - if (rc != SQLITE_OK) { - throw DBException("An error was encountered while ") - << "a TableRef object was writing to the database. " - << "The sqlite error code was " << rc << "."; - } - }; - auto check_done = [&rc]() { - if (rc != SQLITE_OK && rc != SQLITE_DONE) { - throw DBException("An error was encountered while ") - << "a TableRef object was writing to the database. " - << "The sqlite error code was " << rc << "."; - } - }; - - //Scoped object which finalizes a SQLite statement - struct OnCreateOrUpdateExit { - OnCreateOrUpdateExit(sqlite3_stmt * prepared_stmt) : - stmt_(prepared_stmt) - {} - ~OnCreateOrUpdateExit() { - if (stmt_) { - sqlite3_finalize(stmt_); - } - } - private: - sqlite3_stmt * stmt_ = nullptr; - }; - - OnCreateOrUpdateExit scoped_exit(prepared_stmt); - (void) scoped_exit; - - using dt = ColumnDataType; - - //Bind the TableRef's column values to the prepared statement. - for (int idx = 0; idx < (int)col_values.size(); ++idx) { - const int sql_col_idx = idx + 1; - const ColumnValueBase & col = col_values[idx]; - - switch (col.getDataType()) { - case dt::fkey_t: - case dt::char_t: - case dt::int8_t: - case dt::uint8_t: - case dt::int16_t: - case dt::uint16_t: - case dt::int32_t: - case dt::uint32_t: { - const int val = getColumnValueAsInt32(col); - rc = sqlite3_bind_int(prepared_stmt, sql_col_idx, val); - break; - } - - case dt::int64_t: - case dt::uint64_t: { - const sqlite3_int64 val = getColumnValueAsInt64(col); - rc = sqlite3_bind_int64(prepared_stmt, sql_col_idx, val); - break; - } - - case dt::float_t: - case dt::double_t: { - const double val = getColumnValueAsDouble(col); - rc = sqlite3_bind_double(prepared_stmt, sql_col_idx, val); - break; - } - - case dt::string_t: { - const char * val = col.getAs(); - rc = sqlite3_bind_text(prepared_stmt, sql_col_idx, val, -1, 0); - break; - } - - case ColumnDataType::blob_t: { - const Blob blob_descriptor = col.getAs(); - rc = sqlite3_bind_blob(prepared_stmt, sql_col_idx, - blob_descriptor.data_ptr, - (int)blob_descriptor.num_bytes, - 0); - break; - } - - default: - throw DBException("Unrecognized column data type encountered"); - } - check_sql(); - } - - rc = sqlite3_step(prepared_stmt); - check_done(); -} - -//Loop over a Table's columns one by one, and create -//a SQL statement that can be used with CREATE TABLE. -//Column names, data types, and value defaults are -//used here. Example SQL might look like this: -// -// First TEXT, Last TEXT, Age INT, Balance FLOAT DEFAULT 50.00 -// ------------- -// (default $50.00 balance!) -// -std::string getColumnsSqlCommand(const Table & table) -{ - std::ostringstream oss; - for (const auto & column : table) { - oss << column->getName() << " " << column->getDataType(); - if (column->hasDefaultValue()) { - oss << " DEFAULT " << column->getDefaultValueAsString(); - } - oss << ", "; - } - std::string command = oss.str(); - - //Trim the trailing ", " - if (command.back() == ' ') { command.pop_back(); } - if (command.back() == ',') { command.pop_back(); } - - return command; -} - -//! Execute a SQL statement against an open database connection. -//! The optional callback arguments are only used for SELECT -//! statements (eval_sql_select). -//! -//! Here is the documentation from the SQLite library regarding -//! the callback arguments: -//! -//! user_callback -//! - An optional callback that is invoked once for each row of -//! any query results produced by the SQL statements -//! -//! callback_obj -//! - First argument to 'user_callback'. It is the 'this' pointer -//! of the object that implements the callback function. -//! -//! For example: -//! -//! struct MyCallbackHandler { -//! int callback(int argc, char ** argv, char ** col_names) { -//! std::cout << "I just found a SELECT match. Here are the values:\n"; -//! for (int col_idx = 0; col_idx < argc; ++col_idx) { -//! std::cout << "\tColumn '" << col_names[col_idx] -//! << "' has value '" << argv[col_idx] << "'\n"; -//! } -//! return SQLITE_OK; -//! } -//! }; -//! -//! MyCallbackHandler callback_obj; -//! -//! //Assuming we have a sqlite3* connection "db_conn" -//! LOCAL_eval_sql(db_conn, -//! "SELECT * FROM Customers WHERE Last='Smith'", -//! +[](void * callback_obj, int argc, char ** argv, char ** col_names) { -//! return static_cast(callback_obj)->callback( -//! argc, argv, col_names); -//! }, -//! &callback_obj); -void LOCAL_eval_sql(sqlite3 * db_conn, - const std::string & command, - sqlite_select_callback user_callback = nullptr, - void * callback_obj = nullptr) -{ - char * err = 0; - const int res = sqlite3_exec(db_conn, - command.c_str(), - user_callback, - callback_obj, - &err); - if (res != SQLITE_OK) { - switch (res) { - case SQLITE_BUSY: - throw SqlFileLockedException(); - case SQLITE_LOCKED: - throw SqlTableLockedException(); - default: - break; - } - - std::string err_str; - if (err) { - //If our char* has an error message in it, - //we will add it to the exception. - err_str = err; - sqlite3_free(err); - } else { - //Otherwise, just add the SQLite error code. - //Users can look up the meaning of the code - //in sqlite3.h - std::ostringstream oss; - oss << res << " (see sqlite3.h for error code definitions)"; - err_str = oss.str(); - } - err_str += " (failed SQL command was '" + command + "')"; - throw DBException(err_str); - } -} - -//! Execute a SQL statement on an ObjectManager's connection proxy. -void eval_sql(const SQLiteConnProxy * db_proxy, const std::string & command) -{ - if (db_proxy) { - db_proxy->eval(command); - } -} - -//! Execute a SELECT SQL statement on an open database connection -void eval_sql_select(const SQLiteConnProxy * db_proxy, - const std::string & command, - sqlite_select_callback select_callback, - void * callback_obj) -{ - if (db_proxy) { - db_proxy->evalSelect(command, select_callback, callback_obj); - } -} - -//! Callback which gets invoked during SELECT queries that involve -//! floating point comparisons with a supplied tolerance. -void isWithinTolerance(sqlite3_context * context, - int, sqlite3_value ** argv) -{ - const double column_value = sqlite3_value_double(argv[0]); - const double target_value = sqlite3_value_double(argv[1]); - const double tolerance = sqlite3_value_double(argv[2]); - - if (utils::approximatelyEqual(column_value, target_value, tolerance)) { - sqlite3_result_int(context, 1); - } else { - sqlite3_result_int(context, 0); - } -} - -//! Implementation class for the SQL database connection. -class SQLiteConnProxy::Impl -{ -public: - std::string openDbFile(const std::string & db_dir, - const std::string & db_file, - const bool create_file) - { - db_full_filename_ = resolveDbFilename_(db_dir, db_file); - if (db_full_filename_.empty()) { - if (create_file) { - db_full_filename_ = db_dir + "/" + db_file; - } else { - throw DBException("Could not find database file: '") - << db_dir << "/" << db_file; - } - } - - const int db_open_flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE; - sqlite3 * sqlite_conn = nullptr; - auto err_code = sqlite3_open_v2(db_full_filename_.c_str(), - &sqlite_conn, db_open_flags, 0); - - //Inability to even open the database file may mean that - //we don't have write permissions in this directory or - //something like that. We should throw until we understand - //better how else we can get bad file opens. - if (err_code != SQLITE_OK) { - throw DBException( - "Unable to connect to the database file: ") << db_file; - } - - //SQLite isn't the only implementation that SimDB supports. - //The sqlite3_open_v2() function can still return a non-null - //handle for a file that is NOT even SQLite. Let's try to make - //a simple database query to verify the file is actually SQLite. - if (!validateConnectionIsSQLite_(sqlite_conn)) { - sqlite3_close(sqlite_conn); - sqlite_conn = nullptr; - } - - db_conn_ = sqlite_conn; - if (db_conn_) { - sqlite3_create_function(db_conn_, "withinTol", 3, - SQLITE_UTF8, nullptr, - &isWithinTolerance, - nullptr, nullptr); - } - return (db_conn_ != nullptr ? db_full_filename_ : ""); - } - - void validateSchema(const Schema & schema) const - { - auto get_dims_str = [](const std::vector & dims) { - if (dims.empty()) { - return std::string(); - } - - std::ostringstream oss; - oss << "{"; - if (dims.size() == 1) { - oss << dims[0]; - } else { - for (size_t idx = 0; idx < dims.size() - 1; ++idx) { - oss << dims[idx] << ","; - } - oss << dims.back(); - } - oss << "}"; - return oss.str(); - }; - - std::ostringstream oss; - - for (const auto & table : schema) { - for (const auto & column : table) { - const std::vector & dims = column->getDimensions(); - const size_t dims_product = std::accumulate( - dims.begin(), dims.end(), 1, - std::multiplies()); - - if (dims_product > 1) { - oss << " [simdb] SQLite schema error: Table '" - << table.getName() << "', Column '" << column->getName() - << "' has data type " << column->getDataType() << get_dims_str(dims) - << ". Non-scalar ints/floats/strings are not supported by " - << "SQLite. Use a blob column data type instead.\n\n"; - } else if (dims_product == 0) { - oss << " [simdb] SQLite schema error: Table '" - << table.getName() << "', Column '" << column->getName() - << "' has data type " << column->getDataType() << ", but " - << "its dimensions are " << get_dims_str(dims) << ". The " - << "dimensions vector should not have any zeros in it.\n\n"; - } - } - } - - std::string err = oss.str(); - if (!err.empty()) { - err = "SQLite could not validate the schema. " - "These errors were produced:\n\n" + err; - throw DBException(err); - } - } - - void realizeSchema(const Schema & schema, - const ObjectManager & obj_mgr) - { - obj_mgr.safeTransaction([&]() { - for (const auto & table : schema) { - //First create the table and its columns - std::ostringstream oss; - oss << "CREATE TABLE " << table.getName() << "("; - - //All tables have an auto-incrementing primary key - oss << "Id INTEGER PRIMARY KEY AUTOINCREMENT"; - if (!table.hasColumns()) { - //A table without any columns would be somewhat - //odd, but that's what the user's schema specified. - //It is not invalid SQL, so we do not throw. - oss << ");"; - } else { - //Fill in the rest of the CREATE TABLE command: - //CREATE TABLE Id INTEGER PRIMARY KEY AUTOINCREMENT First TEXT, ... - // --------------- - oss << ", " << getColumnsSqlCommand(table) << ");"; - } - - //Create the table in the database - evalSql(oss.str()); - - //Now create any table indexes, for example: - // CREATE INDEX customer_fullname ON Customers (First,Last) - // CREATE INDEX county_population ON Counties (CountyName,Population) - // ... - makeIndexesForTable_(table); - } - }); - } - - bool connectToExistingDatabase(const std::string & db_file) - { - return (!openDbFile(".", db_file, false).empty()); - } - - std::string getDatabaseFullFilename() const - { - return db_full_filename_; - } - - bool isValid() const - { - return (db_conn_ != nullptr); - } - - void getTableNames( - std::unordered_set & table_names) - { - //Helper object that will get called once for each - //matching record in the SELECT statement. - struct TableNames { - TableNames(std::unordered_set & names) : - tbl_names(names) - {} - - int addTableName(int argc, char ** argv, char **) { - //We got another table name. Add it to the set. - //*BUT* skip over any tables that are prefixed - //with "sqlite_". Those are all reserved for the - //library, and aren't really ours. - assert(argc == 1); - if (std::string(argv[0]).find("sqlite_") != 0) { - tbl_names.insert(argv[0]); - } - return 0; - } - std::unordered_set & tbl_names; - }; - - TableNames select_callback_obj(table_names); - - //The TableNames object above has a reference to the table_names - //output argument. Running the following SELECT statement will - //populate this variable. - evalSqlSelect("SELECT name FROM sqlite_master WHERE type='table'", - +[](void * callback_obj, int argc, char ** argv, char ** col_names) { - return static_cast(callback_obj)-> - addTableName(argc, argv, col_names); - }, - &select_callback_obj); - } - - void evalSql(const std::string & command) const - { - eval_(command, nullptr, nullptr); - } - - void evalSqlSelect(const std::string & command, - int (*callback)(void *, int, char **, char **), - void * callback_obj) const - { - eval_(command, callback, callback_obj); - } - - void prepareStatement(const std::string & command, - sqlite3_stmt ** statement) const - { - if (db_conn_ == nullptr) { - return; - } - if (statement == nullptr) { - return; - } - if (sqlite3_prepare_v2(db_conn_, command.c_str(), -1, statement, 0)) { - throw DBException("Malformed SQL command: '") << command << "'"; - } - } - - void openBlob(const std::string & table_name, - const std::string & column_name, - const int row_id, - sqlite3_blob ** blob) const - { - if (db_conn_ == nullptr) { - return; - } - if (blob == nullptr) { - return; - } - if (sqlite3_blob_open(db_conn_, "main", table_name.c_str(), - column_name.c_str(), row_id, 1, blob)) - { - throw DBException("Error encountered while opening database blob. Occurred in ") - << "table '" << table_name << "', column '" << column_name << "'."; - } - } - - int getLastActionNumRecordChanges() const - { - if (db_conn_ == nullptr) { - return 0; - } - return sqlite3_changes(db_conn_); - } - - DatabaseID createObject(const std::string & table_name, - const ColumnValues & values) - { - if (values.empty()) { - evalSql("INSERT INTO " + table_name + " DEFAULT VALUES"); - } else { - const std::string command = - prepareSqlInsertStatement_(table_name, values); - sqlite3_stmt * prepared_stmt = nullptr; - prepareStatement(command, &prepared_stmt); - assert(prepared_stmt != nullptr); - - //Execute the prepared statement - LOCAL_finalizeInsertOrUpdateStatement(prepared_stmt, values); - } - return getLastInsertRowId_(); - } - - ~Impl() - { - if (db_conn_) { - sqlite3_close(db_conn_); - } - } - -private: - //See if there is an existing file by the name - //and return it. If not, return just if it exists. - //Return "" if no such file could be found. - std::string resolveDbFilename_( - const std::string & db_dir, - const std::string & db_file) const - { - std::ifstream fin(db_dir + "/" + db_file); - if (fin) { - return db_dir + "/" + db_file; - } - - fin.open(db_file); - if (fin) { - return db_file; - } - return ""; - } - - //For the CREATE INDEX statements, this helper makes a comma- - //separated string of Column names like "First,Last" - std::string makePropertyIndexesStr_(const Column & column) - { - std::ostringstream oss; - for (const auto & indexed_property : column.getIndexedProperties()) { - oss << indexed_property->getName() << ","; - } - std::string indexes_str = oss.str(); - if (indexes_str.back() == ',') { - indexes_str.pop_back(); - } - return indexes_str; - } - - //Execute index creation statements like: - // - // "CREATE INDEX Customers_Last ON Customers(Last)" - // ^^ indexes Customers table by Last column only - // - // "CREATE INDEX Customers_Last ON Customers(First,Last)" - // ^^ multi-column index on the Customers table by First+Last columns - // - void makeIndexesForColumnInTable_(const Table & table, - const Column & column) - { - std::ostringstream oss; - oss << " CREATE INDEX " << table.getName() << "_" << column.getName() - << " ON " << table.getName() - << " (" << makePropertyIndexesStr_(column) << ")"; - evalSql(oss.str()); - } - - //Create indexes for a given Table, depending on how the - //user set up the Column indexes (indexed by itself, vs. - //indexed together with other columns) - void makeIndexesForTable_(const Table & table) - { - if (!table.hasColumns()) { - return; - } - - for (const auto & column : table) { - if (column->isIndexed()) { - makeIndexesForColumnInTable_(table, *column); - } - } - } - - //Attempt to run an SQL command against our open connection. - //A file may have been given to us that was actually a different - //database format, such as HDF5. - bool validateConnectionIsSQLite_(sqlite3 * db_conn) const - { - struct FindHelper { - bool did_find_any = false; - }; - - FindHelper finder; - const std::string command = - "SELECT name FROM sqlite_master WHERE type='table'"; - - try { - LOCAL_eval_sql(db_conn, command, - +[](void * callback_obj, int, char **, char **) { - static_cast(callback_obj)->did_find_any = true; - return SQLITE_OK; - }, - &finder); - return true; - } catch (...) { - } - return false; - } - - //All SQL commands (both reads and writes) end up here. The only - //difference between a read and a write is if the two callback - //inputs are null or not. - void eval_(const std::string & command, - int (*callback)(void *, int, char **, char **), - void * callback_obj) const - { - //This proxy is intended to be used for safety checks - //to ensure no disallowed statements are executed against - //the database, such as "DROP TABLE Timeseries". - // - //Statement verification should go here as needed before - //calling the LOCAL_eval_sql function. - LOCAL_eval_sql(db_conn_, command, callback, callback_obj); - } - - //Put together an INSERT statement for this table's - //current column values. - std::string prepareSqlInsertStatement_( - const std::string & table_name, - const ColumnValues & col_values) const - { - //Build the prepared SQL statement. This will put - //placeholders ("?") for all the column values, - //which we'll bind to shortly. - // - //The resulting SQL command looks something like this: - // - // INSERT INTO Customers values (?,?,?) - // - std::string stmt; - std::ostringstream oss; - oss << "INSERT INTO " << table_name << " ("; - if (col_values.size() == 1) { - oss << col_values[0].getColumnName() << ")"; - } else { - for (size_t idx = 0; idx < col_values.size() - 1; ++idx) { - oss << col_values[idx].getColumnName() << ","; - } - oss << col_values.back().getColumnName() << ")"; - } - - oss << " values ("; - if (col_values.size() == 1) { - oss << "?"; - } else { - for (size_t idx = 0; idx < col_values.size() - 1; ++idx) { - oss << "?,"; - } - oss << "?"; - } - oss << ")"; - stmt = oss.str(); - return stmt; - } - - //Return the database ID of the last record INSERT - int getLastInsertRowId_() const - { - if (db_conn_ == nullptr) { - return 0; - } - return sqlite3_last_insert_rowid(db_conn_); - } - - //Physical database connection - sqlite3 * db_conn_ = nullptr; - - //Filename of the database in use - std::string db_full_filename_; -}; - -SQLiteConnProxy::SQLiteConnProxy() : impl_(new SQLiteConnProxy::Impl) -{ -} - -void SQLiteConnProxy::validateSchema(const Schema & schema) const -{ - impl_->validateSchema(schema); -} - -void SQLiteConnProxy::realizeSchema( - const Schema & schema, - const ObjectManager & obj_mgr) -{ - impl_->realizeSchema(schema, obj_mgr); -} - -bool SQLiteConnProxy::connectToExistingDatabase(const std::string & db_file) -{ - return impl_->connectToExistingDatabase(db_file); -} - -std::string SQLiteConnProxy::getDatabaseFullFilename() const -{ - return impl_->getDatabaseFullFilename(); -} - -bool SQLiteConnProxy::isValid() const -{ - return impl_->isValid(); -} - -void SQLiteConnProxy::getTableNames( - std::unordered_set & table_names) -{ - impl_->getTableNames(table_names); -} - -void SQLiteConnProxy::beginAtomicTransaction() const -{ - impl_->evalSql("BEGIN TRANSACTION"); -} - -void SQLiteConnProxy::commitAtomicTransaction() const -{ - impl_->evalSql("COMMIT TRANSACTION"); -} - -void stringifyColumnValue(const ColumnValueBase & col, - std::ostream & os) -{ - using dt = ColumnDataType; - - auto print_char_if_has_set_constraint = [&](const char ch) { - if (!col.hasConstraint()) { - return; - } - - switch (col.getConstraint()) { - case constraints::in_set: - case constraints::not_in_set: { - os << ch; - break; - } - default: - break; - } - }; - - print_char_if_has_set_constraint('('); - - auto print_col_value_at_idx = [&](const size_t idx) { - switch (col.getDataType()) { - case dt::char_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::int8_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::uint8_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::int16_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::uint16_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::int32_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::uint32_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::int64_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::uint64_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::float_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::double_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::string_t: { - os << stringify(col.getAs(idx)); - break; - } - case dt::fkey_t: { - os << stringify(col.getAs(idx)); - break; - } - default: - throw DBException("ColumnValueBase cannot be stringified"); - } - }; - - if (col.getNumValues() == 1) { - print_col_value_at_idx(0); - } else { - for (size_t idx = 0; idx < col.getNumValues() - 1; ++idx) { - print_col_value_at_idx(idx); - os << ","; - } - print_col_value_at_idx(col.getNumValues() - 1); - } - - print_char_if_has_set_constraint(')'); -} - -//! Turn a ColumnValueBase object's value into a clause -//! that looks something like this: -//! -//! WHERE LastName='Smith' -//! -//! This is used when building constrained UPDATE and -//! DELETE statements. -std::string createWhereClause(const ColumnValueBase & col) -{ - std::ostringstream oss; - oss << col.getColumnName() << col.getConstraint(); - stringifyColumnValue(col, oss); - return oss.str(); -} - -void SQLiteConnProxy::performDeletion( - const std::string & table_name, - const ColumnValues & where_clauses) const -{ - std::ostringstream oss; - oss << "DELETE FROM " << table_name; - if (!where_clauses.empty()) { - oss << " WHERE "; - if (where_clauses.size() == 1) { - oss << createWhereClause(where_clauses[0]); - } else { - for (size_t idx = 0; idx < where_clauses.size() - 1; ++idx) { - oss << createWhereClause(where_clauses[idx]) << " AND "; - } - oss << createWhereClause(where_clauses.back()); - } - } - - const std::string command = oss.str(); - impl_->evalSql(command); -} - -size_t SQLiteConnProxy::performUpdate( - const std::string & table_name, - const ColumnValues & col_values, - const ColumnValues & where_clauses) const -{ - //Build the prepared SQL statement. This will put - //placeholders ("?") for all the column values, - //which we'll bind to shortly. - // - //The resulting SQL command looks something like this: - // - // UPDATE Customers SET AccountActive=? - // WHERE Name='Smith' - std::ostringstream oss; - oss << "UPDATE " << table_name << " SET "; - if (col_values.size() == 1) { - oss << col_values[0].getColumnName() << "=?"; - } else { - for (size_t idx = 0; idx < col_values.size() - 1; ++idx) { - oss << col_values[idx].getColumnName() << "=?,"; - } - oss << col_values.back().getColumnName() << "=?"; - } - - std::string command; - if (where_clauses.empty()) { - command = oss.str(); - } else { - oss << " WHERE "; - if (where_clauses.size() == 1) { - oss << createWhereClause(where_clauses[0]); - } else { - for (size_t idx = 0; idx < where_clauses.size() - 1; ++idx) { - oss << createWhereClause(where_clauses[idx]) << " AND "; - } - oss << createWhereClause(where_clauses.back()); - } - command = oss.str(); - } - - //Execute the prepared statement - sqlite3_stmt * prepared_stmt = nullptr; - impl_->prepareStatement(command, &prepared_stmt); - assert(prepared_stmt != nullptr); - - LOCAL_finalizeInsertOrUpdateStatement(prepared_stmt, col_values); - return impl_->getLastActionNumRecordChanges(); -} - -AnySizeObjectFactory SQLiteConnProxy::getObjectFactoryForTable( - const std::string &) const -{ - return +[](DbConnProxy * db_proxy, - const std::string & table_name, - const ColumnValues & obj_values) - { - return static_cast(db_proxy)-> - createObject(table_name, obj_values); - }; -} - -DatabaseID SQLiteConnProxy::createObject( - const std::string & table_name, - const ColumnValues & values) -{ - return impl_->createObject(table_name, values); -} - -std::string SQLiteConnProxy::openDbFile_( - const std::string & db_dir, - const std::string & db_file, - const bool create_file) -{ - return impl_->openDbFile(db_dir, db_file, create_file); -} - -void SQLiteConnProxy::eval(const std::string & command) const -{ - impl_->evalSql(command); -} - -void SQLiteConnProxy::evalSelect( - const std::string & command, - int (*callback)(void *, int, char **, char **), - void * callback_obj) const -{ - impl_->evalSqlSelect(command, callback, callback_obj); -} - -void SQLiteConnProxy::prepareStatement_( - const std::string & command, - void ** statement) const -{ - sqlite3_stmt * stmt = nullptr; - impl_->prepareStatement(command, &stmt); - if (stmt) { - *statement = stmt; - } -} - -SQLiteConnProxy::~SQLiteConnProxy() -{ -} - -} // namespace simdb diff --git a/sparta/simdb/src/TableRef.cpp b/sparta/simdb/src/TableRef.cpp deleted file mode 100644 index b4edf2fa9d..0000000000 --- a/sparta/simdb/src/TableRef.cpp +++ /dev/null @@ -1,398 +0,0 @@ -// -*- C++ -*- - -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/utils/ObjectQuery.hpp" - -namespace simdb { - -//! Scoped object which ensures the TableRef's member -//! variables related to object inserts and updates -//! are cleared out at the end of the methods which -//! put one of these on the stack. -class OnCreateOrUpdateExit { -public: - OnCreateOrUpdateExit( - bool & is_in_update_statement, - std::unique_ptr & record_finder, - std::vector & raw_bytes, - ColumnValueContainer & col_values) : - is_in_update_statement_(is_in_update_statement), - record_finder_(record_finder), - raw_bytes_(raw_bytes), - col_values_(col_values) - {} - - ~OnCreateOrUpdateExit() { - is_in_update_statement_ = false; - record_finder_.reset(); - raw_bytes_.clear(); - col_values_.clear(); - } - -private: - bool & is_in_update_statement_; - std::unique_ptr & record_finder_; - std::vector & raw_bytes_; - ColumnValueContainer & col_values_; -}; - -std::unique_ptr TableRef::createDefaultObject_() -{ - std::unique_ptr obj_ref; - DatabaseID db_id = 0; - - obj_mgr_.safeTransaction([&]() { - //Get ready for the database statement to be run. - //Protect the TableRef member variables from exceptions. - OnCreateOrUpdateExit scoped_exit( - is_in_update_statement_, - record_finder_for_update_, - raw_bytes_for_obj_create_, - col_values_); - - (void) scoped_exit; - - if (!raw_bytes_for_obj_create_.empty()) { - db_id = fixed_size_record_factory_( - db_proxy_.get(), - table_name_, - &raw_bytes_for_obj_create_[0], - raw_bytes_for_obj_create_.size()); - - return; - } - - ColumnValues null_values; - db_id = any_size_record_factory_( - db_proxy_.get(), table_name_, null_values); - }); - - const bool return_object = - explicit_return_object_ == ExplicitReturnObject::DEFAULT || - explicit_return_object_ == ExplicitReturnObject::ALWAYS_RETURN; - - if (db_id > 0 && return_object) { - obj_ref.reset(new ObjectRef(obj_mgr_, table_name_, db_id)); - } - return obj_ref; -} - -//! Zero-argument object creation. All columns will take -//! their default values if any were specified in this -//! table's schema definition. -std::unique_ptr TableRef::createObject() -{ - return createDefaultObject_(); -} - -std::unique_ptr TableRef::finalizeCreationStatement_() -{ - //Defer to the zero-argument createObject() API - //if we do not have any column values up front. - //This could happen with a call site like this: - // - // std::vector empty_vec; - // auto obj = table->createObjectWithArgs( - // "MyBlob", - // empty_vec); - if (col_values_.empty()) { - return createDefaultObject_(); - } - - DatabaseID db_id = 0; - obj_mgr_.safeTransaction([&]() { - //Get ready for the database statement to be run. - //Protect the TableRef member variables from exceptions. - OnCreateOrUpdateExit scoped_exit( - is_in_update_statement_, - record_finder_for_update_, - raw_bytes_for_obj_create_, - col_values_); - - (void) scoped_exit; - - //Make sure the TableRef method updateRowValues() was - //used like this: - // - // table->updateRowValues("MyInt", 100). - // forRecordsWhere("MyInt", simdb::constraints::less, 85); - // - //And not like this: - // - // auto & updater = table->updateRowValues("MyInt", 100); - // table->createObjectWithArgs("MyFloat", 3.14); - // updater.forRecordsWhere("MyInt", simdb::constraints::less, 85); - // - //The scoped OnCreateOrUpdateExit object above will prevent - //resource leaks if we throw. - if (is_in_update_statement_) { - throw DBException( - "You cannot make calls to RecordFinder::forRecordsWhere() " - "at a different time (on a different line of code) than " - "calls to TableRef::updateRowValues()."); - } - - db_id = any_size_record_factory_( - db_proxy_.get(), table_name_, col_values_.getValues()); - }); - - const bool return_object = - explicit_return_object_ == ExplicitReturnObject::DEFAULT || - explicit_return_object_ == ExplicitReturnObject::ALWAYS_RETURN; - - if (!return_object) { - return nullptr; - } - - if (db_id <= 0) { - throw DBException("Invalid database ID encountered while ") - << "executing TableRef::createObjectWithArgs()"; - } - - return std::unique_ptr( - new ObjectRef(obj_mgr_, table_name_, db_id)); -} - -void TableRef::finalizeDeletionStatement_() -{ - struct OnDeletionExit { - OnDeletionExit(ColumnValueContainer & where_clauses) : - where_clauses_(where_clauses) - {} - ~OnDeletionExit() { - where_clauses_.clear(); - } - private: - ColumnValueContainer & where_clauses_; - }; - - obj_mgr_.safeTransaction([&]() { - OnDeletionExit scoped_exit(delete_where_clauses_); - (void) scoped_exit; - - db_proxy_->performDeletion( - table_name_, delete_where_clauses_.getValues()); - }); -} - -size_t TableRef::finalizeUpdateStatement_() -{ - if (record_finder_for_update_ == nullptr) { - return 0; - } - if (col_values_.empty()) { - return 0; - } - - size_t num_records_updated = 0; - - obj_mgr_.safeTransaction([&]() { - //Get ready for the database statement to be run. - //Protect the TableRef member variables from exceptions. - OnCreateOrUpdateExit scoped_exit( - is_in_update_statement_, - record_finder_for_update_, - raw_bytes_for_obj_create_, - col_values_); - - (void) scoped_exit; - - const auto & where_clauses = - record_finder_for_update_->update_where_clauses_; - - num_records_updated = db_proxy_->performUpdate( - table_name_, col_values_.getValues(), where_clauses.getValues()); - }); - - return num_records_updated; -} - -struct ColumnValueCaster { - ColumnValueCaster(const ColumnDataType dtype) : - dtype_(dtype) - { - bytes_.resize(getFixedNumBytesForColumnDType(dtype_)); - value_ = &bytes_[0]; - } - - void * getValuePtr() { - return value_; - } - - double getDoubleValue() const { - using dt = ColumnDataType; - - switch (dtype_) { - case dt::char_t: { - const char casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::int8_t: { - const int8_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::uint8_t: { - const uint8_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::int16_t: { - const int16_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::uint16_t: { - const uint16_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::int32_t: { - const int32_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::uint32_t: { - const uint32_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::int64_t: { - const int64_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::uint64_t: { - const uint64_t casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::float_t: { - const float casted = *reinterpret_cast(value_); - return static_cast(casted); - } - case dt::double_t: { - return *static_cast(value_); - } - default: - return 0; - } - } - - ColumnDataType getDataType() const { - return dtype_; - } - - ColumnValueCaster(const ColumnValueCaster &) = delete; - ColumnValueCaster(ColumnValueCaster &&) = delete; - -private: - const ColumnDataType dtype_; - void * value_ = nullptr; - std::vector bytes_; -}; - -bool TableRef::captureSummary() -{ - ObjectQuery query(obj_mgr_, table_name_); - std::vector source_column_values; - - std::unique_ptr summary_table_ref = - obj_mgr_.getTable(table_name_ + "_Summary"); - - if (!summary_table_ref) { - return false; - } - - std::unique_ptr summary_table_record; - - for (const auto & col : col_descriptors_) { - ColumnValueCaster caster(col.second); - - using dt = ColumnDataType; - - switch (caster.getDataType()) { - case dt::char_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::int8_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::uint8_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::int16_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::uint16_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::int32_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::uint32_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::int64_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::uint64_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::float_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - case dt::double_t: { - query.writeResultIterationsTo( - col.first.c_str(), static_cast(caster.getValuePtr())); - break; - } - default: - continue; - } - - source_column_values.clear(); - const size_t matches = query.countMatches(); - if (matches == 0) { - continue; - } - source_column_values.reserve(matches); - - if (summary_table_record == nullptr) { - summary_table_record = summary_table_ref->createObject(); - } - - auto result_iter = query.executeQuery(); - while (result_iter->getNext()) { - source_column_values.emplace_back(caster.getDoubleValue()); - } - - for (const auto & summary_fcn : summary_fcns_) { - const std::string summary_table_column_name = col.first + "_" + summary_fcn.first; - const double summarized_value = summary_fcn.second(source_column_values); - - summary_table_record->setPropertyDouble( - summary_table_column_name, - summarized_value); - } - } - - return summary_table_record != nullptr; -} - -} // namespace simdb diff --git a/sparta/simdb/src/simdb.cpp b/sparta/simdb/src/simdb.cpp deleted file mode 100644 index 81dd9c3e7e..0000000000 --- a/sparta/simdb/src/simdb.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// -*- C++ -*- - -#include "simdb/async/TimerThread.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/schema/DatabaseRoot.hpp" - -/*! - * \brief Static initializations in the SimDB module. - */ - -namespace simdb -{ - std::atomic TimerThread::current_num_task_threads_{0}; - bool TimerThread::stress_testing_ = false; - std::map DatabaseRoot::db_types_by_namespace_; - std::map> DatabaseRoot::schema_builders_by_namespace_; - std::map DatabaseRoot::proxy_creators_by_db_type_; -} diff --git a/sparta/simdb/test/CMakeLists.txt b/sparta/simdb/test/CMakeLists.txt deleted file mode 100644 index a6e8a92ce5..0000000000 --- a/sparta/simdb/test/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -project(SIMDB_TESTS) - -# Enable testing and configures the test/DartConfiguration.tcl -include (CTest) - -# Setup options for valgrind testing. -set(VALGRIND_REGRESS_ENABLED TRUE) -set (VALGRIND_OPTS - "--error-exitcode=5" - "--leak-check=full" - "--show-reachable=yes" - "--undef-value-errors=yes" - "--suppressions=${SPARTA_BASE}/test/valgrind_leakcheck.supp" - "--soname-synonyms=somalloc=NONE" - ) - -set (VALGRIND_TEST_LABEL valgrind_test) - -# Add the custom regress/regress_valgrind targets. -add_custom_target (simdb_regress) -add_custom_target (simdb_regress_valgrind) - -# Since make does not pass the parallel jobs flag to ctest from the user, -# a fixed count will be set based on core count w/ a max 8 -include(ProcessorCount) -ProcessorCount(NUM_CORES) -if (NOT NUM_CORES EQUAL 0) - set(CTEST_BUILD_FLAGS -j${NUM_CORES}) - set(ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${NUM_CORES}) -endif() -message(STATUS "Found " ${NUM_CORES} " cores in machine (for ctest)") - -# NOTE: -# running ctest with --test-action test creates Testing//Test.xml -# that can be loaded into the CI test result tracker -add_custom_command(TARGET simdb_regress POST_BUILD COMMAND ctest -LE ${VALGRIND_TEST_LABEL} -j${NUM_CORES} --test-action test) -add_custom_command(TARGET simdb_regress_valgrind POST_BUILD COMMAND ctest -L ${VALGRIND_TEST_LABEL} -j${NUM_CORES} --test-action test) - -add_subdirectory(CoreDatabase) -add_subdirectory(HDF5Database) -add_subdirectory(SharedDB) -add_subdirectory(SQLiteDatabase) -add_subdirectory(Thread) -add_subdirectory(Utils) diff --git a/sparta/simdb/test/Colors.hpp b/sparta/simdb/test/Colors.hpp deleted file mode 100644 index b55c53fcfc..0000000000 --- a/sparta/simdb/test/Colors.hpp +++ /dev/null @@ -1,159 +0,0 @@ -// -*- C++ -*- - -/*! - * \brief Color codes / utilities for SimDB. - */ - -#pragma once - -#include "simdb/Errors.hpp" - -#include -#include - -//! Define some color code values that are used as the defaults -//! in the global default ColorScheme instance. These should never -//! actually be used manually. You should use the accessor methods of a -//! ColorScheme instance so you get support for easily disabling/enabling -//! the output of colors. -#define SIMDB_UNMANAGED_COLOR_NORMAL "\033[0;0m" -#define SIMDB_UNMANAGED_COLOR_BOLD "\033[0;1m" -#define SIMDB_UNMANAGED_COLOR_RED "\033[0;31m" -#define SIMDB_UNMANAGED_COLOR_GREEN "\033[0;32m" -#define SIMDB_UNMANAGED_COLOR_YELLOW "\033[0;33m" -#define SIMDB_UNMANAGED_COLOR_BLUE "\033[0;34m" -#define SIMDB_UNMANAGED_COLOR_MAGENTA "\033[0;35m" -#define SIMDB_UNMANAGED_COLOR_CYAN "\033[0;36m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_RED "\033[1;31m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_GREEN "\033[1;32m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_YELLOW "\033[1;33m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_BLUE "\033[1;34m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_MAGENTA "\033[1;35m" -#define SIMDB_UNMANAGED_COLOR_BRIGHT_CYAN "\033[1;36m" -#define SIMDB_UNMANAGED_COLOR_BG_RED "\033[0;41m" -#define SIMDB_UNMANAGED_COLOR_BG_GREEN "\033[0;42m" -#define SIMDB_UNMANAGED_COLOR_BG_YELLOW "\033[0;43m" -#define SIMDB_UNMANAGED_COLOR_BG_BLUE "\033[0;44m" -#define SIMDB_UNMANAGED_COLOR_BG_MAGENTA "\033[0;45m" -#define SIMDB_UNMANAGED_COLOR_BG_CYAN "\033[0;46m" - -//! Macros for accessing the colors through the default scheme. -#define SIMDB_CURRENT_COLOR_NORMAL simdb::color::ColorScheme::getDefaultScheme().color(Color::Normal) -#define SIMDB_CURRENT_COLOR_BRIGHT_NORMAL simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightNormal) -#define SIMDB_CURRENT_COLOR_BG_NORMAL simdb::color::ColorScheme::getDefaultScheme().color(Color::BgNormal) -#define SIMDB_CURRENT_COLOR_BOLD simdb::color::ColorScheme::getDefaultScheme().color(Color::Bold) -#define SIMDB_CURRENT_COLOR_BRIGHT_BOLD simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightBold) -#define SIMDB_CURRENT_COLOR_BG_BOLD simdb::color::ColorScheme::getDefaultScheme().color(Color::BgBold) -#define SIMDB_CURRENT_COLOR_RED simdb::color::ColorScheme::getDefaultScheme().color(Color::Red) -#define SIMDB_CURRENT_COLOR_BRIGHT_RED simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightRed) -#define SIMDB_CURRENT_COLOR_BG_RED simdb::color::ColorScheme::getDefaultScheme().color(Color::BgRed) -#define SIMDB_CURRENT_COLOR_GREEN simdb::color::ColorScheme::getDefaultScheme().color(Color::Green) -#define SIMDB_CURRENT_COLOR_BRIGHT_GREEN simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightGreen) -#define SIMDB_CURRENT_COLOR_BG_GREEN simdb::color::ColorScheme::getDefaultScheme().color(Color::BgGreen) -#define SIMDB_CURRENT_COLOR_YELLOW simdb::color::ColorScheme::getDefaultScheme().color(Color::Yellow) -#define SIMDB_CURRENT_COLOR_BRIGHT_YELLOW simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightYellow) -#define SIMDB_CURRENT_COLOR_BG_YELLOW simdb::color::ColorScheme::getDefaultScheme().color(Color::BgYellow) -#define SIMDB_CURRENT_COLOR_BLUE simdb::color::ColorScheme::getDefaultScheme().color(Color::Blue) -#define SIMDB_CURRENT_COLOR_BRIGHT_BLUE simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightBlue) -#define SIMDB_CURRENT_COLOR_BG_BLUE simdb::color::ColorScheme::getDefaultScheme().color(Color::BgBlue) -#define SIMDB_CURRENT_COLOR_MAGENTA simdb::color::ColorScheme::getDefaultScheme().color(Color::Magenta) -#define SIMDB_CURRENT_COLOR_BRIGHT_MAGENTA simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightMagenta) -#define SIMDB_CURRENT_COLOR_BG_MAGENTA simdb::color::ColorScheme::getDefaultScheme().color(Color::BgMagenta) -#define SIMDB_CURRENT_COLOR_CYAN simdb::color::ColorScheme::getDefaultScheme().color(Color::Cyan) -#define SIMDB_CURRENT_COLOR_BRIGHT_CYAN simdb::color::ColorScheme::getDefaultScheme().color(Color::BrightCyan) -#define SIMDB_CURRENT_COLOR_BG_CYAN simdb::color::ColorScheme::getDefaultScheme().color(Color::BgCyan) - -static constexpr const char * ALL_COLORS[] = { - SIMDB_UNMANAGED_COLOR_NORMAL, - SIMDB_UNMANAGED_COLOR_BOLD, - SIMDB_UNMANAGED_COLOR_RED, - SIMDB_UNMANAGED_COLOR_GREEN, - SIMDB_UNMANAGED_COLOR_YELLOW, - SIMDB_UNMANAGED_COLOR_BLUE, - SIMDB_UNMANAGED_COLOR_MAGENTA, - SIMDB_UNMANAGED_COLOR_CYAN, - SIMDB_UNMANAGED_COLOR_BRIGHT_RED, - SIMDB_UNMANAGED_COLOR_BRIGHT_GREEN, - SIMDB_UNMANAGED_COLOR_BRIGHT_YELLOW, - SIMDB_UNMANAGED_COLOR_BRIGHT_BLUE, - SIMDB_UNMANAGED_COLOR_BRIGHT_MAGENTA, - SIMDB_UNMANAGED_COLOR_BRIGHT_CYAN, - SIMDB_UNMANAGED_COLOR_BG_RED, - SIMDB_UNMANAGED_COLOR_BG_GREEN, - SIMDB_UNMANAGED_COLOR_BG_YELLOW, - SIMDB_UNMANAGED_COLOR_BG_BLUE, - SIMDB_UNMANAGED_COLOR_BG_MAGENTA, - SIMDB_UNMANAGED_COLOR_BG_CYAN -}; - -//! Define enums for accessing the different colors via a ColorScheme. -enum class Color { - Normal, Bold, Red, Green, Yellow, Blue, Magenta, Cyan, - BrightRed, BrightGreen, BrightYellow, BrightBlue, - BrightMagenta, BrightCyan, BgRed, BgGreen, BgYellow, BgBlue, - BgMagenta, BgCyan -}; - -#define SIMDB_CMDLINE_COLOR_NORMAL "" // SIMDB_UNMANAGED_COLOR_NORMAL -#define SIMDB_CMDLINE_COLOR_ERROR "" // SIMDB_UNMANAGED_COLOR_ERROR -#define SIMDB_CMDLINE_COLOR_WARNING "" // SIMDB_UNMANAGED_COLOR_YELLOW -#define SIMDB_CMDLINE_COLOR_GOOD "" // SIMDB_UNMANAGED_COLOR_GOOD - -namespace simdb { -namespace color { - - /** - * \class ColorScheme - * \brief Accessor methods for obtaining color code strings. - * \details The idea behind ColorScheme is to have the ability - * to disable terminal colors in the module with a simple flag. - */ - class ColorScheme - { - public: - static ColorScheme & getDefaultScheme() { - static ColorScheme scheme; - return scheme; - } - - ~ColorScheme() = default; - - /** - * \brief Enable or disable colors. - * \param enabled Flag denoting whether colors are enabled for - * error reporting in SimDB. - */ - void setIsEnabled(const bool enabled) { - enabled_ = enabled; - } - - //! The accessors that should always be used for colors. - const char * color(const Color c) const { - if (enabled_) { - using utype = typename std::underlying_type::type; - return all_colors_.at(static_cast(c)).c_str(); - } - - static const char * empty = ""; - return empty; - } - - private: - ColorScheme() - { - // Load all the colors. - for (const char * c : ALL_COLORS) { - all_colors_.emplace_back(c); - } - } - - //! Whether or not we are returning real colors. - bool enabled_ = true; - - //! A list of colors in order. - std::vector all_colors_; - }; - -} // namespace color -} // namespace simdb - diff --git a/sparta/simdb/test/CoreDatabase/CMakeLists.txt b/sparta/simdb/test/CoreDatabase/CMakeLists.txt deleted file mode 100644 index 11aac6180a..0000000000 --- a/sparta/simdb/test/CoreDatabase/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -project(SIMDB_CoreDatabase_test) - -add_executable(SIMDB_CoreDatabase_test CoreDatabase_test.cpp) -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_recursive_copy(SIMDB_CoreDatabase_test test_dbs) - -simdb_test(SIMDB_CoreDatabase_test SIMDB_CoreDatabase_test_RUN) diff --git a/sparta/simdb/test/CoreDatabase/CoreDatabase_test.cpp b/sparta/simdb/test/CoreDatabase/CoreDatabase_test.cpp deleted file mode 100644 index e39546f6eb..0000000000 --- a/sparta/simdb/test/CoreDatabase/CoreDatabase_test.cpp +++ /dev/null @@ -1,769 +0,0 @@ -/*! - * \file CoreDatabase_test.cpp - * - * \brief Database tests for SimDB functionality that is not - * specific to any particular database format (SQLite, HDF5, - * etc.) - */ - -#include "simdb/test/SimDBTester.hpp" - -//Core database headers -#include "simdb/schema/DatabaseRoot.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/utils/uuids.hpp" -#include "simdb/utils/MathUtils.hpp" -#include "simdb/utils/StringUtils.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/Errors.hpp" - -//SQLite-specific includes -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" - -//Standard headers -#include - -#define DB_DIR "test_dbs" - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -//! Schema builder for the Strings namespace. -void StringsSchemaBuilder(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("Strings") - .addColumn("First", dt::string_t) - .addColumn("Second", dt::string_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::string_t); -} - -void testNamespaceSchemas() -{ - PRINT_ENTER_TEST - - simdb::DatabaseRoot db_root(DB_DIR); - auto strings_namespace = db_root.getNamespace("Strings"); - EXPECT_NOTEQUAL(strings_namespace, nullptr); - - //Since we registered a schema builder for the Strings - //namespace, we should expect certain tables to be in - //the schema (autopopulated). - EXPECT_TRUE(strings_namespace->hasSchema()); - EXPECT_TRUE(strings_namespace->hasTableNamed("Metadata")); - - //Verify that table MoreMetadata is not in the schema, - //and then verify that we are able to add this table - //to the Strings namespace schema ourselves. - EXPECT_FALSE(strings_namespace->hasTableNamed("MoreMetadata")); - - strings_namespace->addToSchema([](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("MoreMetadata") - .addColumn("Name", dt::string_t) - .addColumn("Alias", dt::string_t); - }); - - EXPECT_TRUE(strings_namespace->hasTableNamed("MoreMetadata")); - - const simdb::Table * more_metadata_table = - strings_namespace->getTableNamed("MoreMetadata"); - - //Verify an exception is thrown if we attempt to add - //a table that already exists in this namespaces's - //schema. But it should only throw if the table we - //attempt to add has a different column configuration - //than the existing schema table of the same name. - EXPECT_THROW(strings_namespace->addToSchema([](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("MoreMetadata") - .addColumn("Name", dt::string_t) - .addColumn("Alais", dt::string_t); //Typo intentional - })); - - //Verify that an exception is NOT thrown if we attempt - //to add a table that already exists by the same name, - //but the column configuration of the table we try to - //add is identical to the table that is already there. - EXPECT_NOTHROW(strings_namespace->addToSchema([](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("MoreMetadata") - .addColumn("Name", dt::string_t) - .addColumn("Alias", dt::string_t); //No typo this time - })); - - //Double check that the namespace returns the same - //table pointer for "MoreMetadata", since we did not - //actually create another table by that name. It simply - //gets ignored. - EXPECT_EQUAL(strings_namespace->getTableNamed("MoreMetadata"), - more_metadata_table); - - //Verify that we can ask the namespace for one of its - //tables when we pass in the fully qualified table name. - //It is advised not to do this, but if the qualified table - //name matches exactly, DatabaseNamespace allows it. - const std::string valid_qualified_table_name = - std::string("Strings") + simdb::Table::NS_DELIM + "MoreMetadata"; - - EXPECT_EQUAL(strings_namespace->getTableNamed(valid_qualified_table_name), - more_metadata_table); - - //Verify that we can ask the namespace for one of its - //tables using a fully qualified table name, where the - //namespace is correct but the unqualified table name - //does not exist. It should not throw; it should return - //nullptr. - const std::string nonexistent_qualified_table_name = - std::string("Strings") + simdb::Table::NS_DELIM + "DoesNotExist"; - - EXPECT_EQUAL( - strings_namespace->getTableNamed(nonexistent_qualified_table_name), - nullptr); - - //Edge case: Use the correct namespace, but leave the - //unqualified table name blank. Should return null. - const std::string valid_ns_empty_table_name = - std::string("Strings") + simdb::Table::NS_DELIM; - - EXPECT_EQUAL( - strings_namespace->getTableNamed(valid_ns_empty_table_name), - nullptr); - - //Edge case: Pass in only the namespace delimiter. - //Should return null. - const std::string empty_ns_empty_table_name = - std::string(1, simdb::Table::NS_DELIM); - - EXPECT_EQUAL( - strings_namespace->getTableNamed(empty_ns_empty_table_name), - nullptr); - - //Edge case: Pass in an invalid namespace, and an - //empty unqualified table name. Should throw. - const std::string invalid_ns_empty_table_name = - std::string("Striings") + simdb::Table::NS_DELIM; - - EXPECT_THROW( - strings_namespace->getTableNamed(invalid_ns_empty_table_name)); - - //This test only worked on DatabaseNamespace schemas; - //no ObjectManager's / DbConnProxy's should have been - //created. - EXPECT_FALSE(strings_namespace->databaseConnectionEstablished()); -} - -void testNamespaceRecords() -{ - PRINT_ENTER_TEST - - simdb::DatabaseRoot db_root(DB_DIR); - auto numbers_namespace = db_root.getNamespace("Numbers"); - - EXPECT_NOTEQUAL(numbers_namespace, nullptr); - EXPECT_TRUE(numbers_namespace->hasSchema()); - - //Test data structure for the Numbers namespace. - struct Numbers { - struct Data { - int32_t first; - double second; - }; - Data data; - - struct Metadata { - std::string name; - int64_t value; - }; - Metadata metadata; - - struct MoreMetadata { - std::string name; - double value; - }; - MoreMetadata more_metadata; - - static Numbers createRandom() { - Numbers n; - n.data.first = simdb::utils::chooseRand(); - n.data.second = simdb::utils::chooseRand(); - n.metadata.name = simdb::utils::chooseRand(); - n.metadata.value = simdb::utils::chooseRand(); - n.more_metadata.name = simdb::utils::chooseRand(); - n.more_metadata.value = rand() * 3.14; - return n; - } - }; - - //Before we try to create any records, verify that no - //database connection has been made yet. - EXPECT_FALSE(numbers_namespace->databaseConnectionEstablished()); - - //Now ask for the ObjectDatabase from this namespace. - //This should trigger the physical database connection - //to be made. - auto numbers_db = numbers_namespace->getDatabase(); - EXPECT_TRUE(numbers_namespace->databaseConnectionEstablished()); - - //Create a record using the default Numbers schema. - auto record_values = Numbers::createRandom(); - auto numbers_tbl = numbers_db->getTable("Numbers"); - - auto numbers_row1 = numbers_tbl->createObjectWithArgs( - "First", record_values.data.first, - "Second", record_values.data.second); - - //Use the ObjectDatabase::findObject() method to ask - //the database for the ObjectRef wrapping the record - //we just created. - auto recovered_numbers_row1 = numbers_db->findObject( - "Numbers", numbers_row1->getId()); - - //Validate the record values. - EXPECT_EQUAL(recovered_numbers_row1->getPropertyInt32("First"), - record_values.data.first); - - EXPECT_EQUAL(recovered_numbers_row1->getPropertyDouble("Second"), - record_values.data.second); - - //Now add a new table that was not in the hard-coded / - //registered schema builder for this namespace. - numbers_namespace->addToSchema([](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("MoreMetadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::double_t); - }); - - //Create another record, this time for the MoreMetadata - //table we just added to the namespace schema. - record_values = Numbers::createRandom(); - auto more_metadata_tbl = numbers_db->getTable("MoreMetadata"); - - auto more_metadata_row1 = more_metadata_tbl->createObjectWithArgs( - "Name", record_values.more_metadata.name, - "Value", record_values.more_metadata.value); - - //Again, use the ObjectDatabase::findObject() method to - //ask for the ObjectRef which wraps this MoreMetadata - //record. - auto recovered_more_metadata_row1 = numbers_db->findObject( - "MoreMetadata", more_metadata_row1->getId()); - - //Validate the record values. - EXPECT_EQUAL(recovered_more_metadata_row1->getPropertyString("Name"), - record_values.more_metadata.name); - - EXPECT_EQUAL(recovered_more_metadata_row1->getPropertyDouble("Value"), - record_values.more_metadata.value); - - //Verify ObjectDatabase::findObjects() - create another - //MoreMetadata record first so we have multiple results - //from findObjects() we can verify. - std::vector find_objs_expected_ans = {record_values}; - record_values = Numbers::createRandom(); - find_objs_expected_ans.emplace_back(record_values); - - auto more_metadata_row2 = more_metadata_tbl->createObjectWithArgs( - "Name", record_values.more_metadata.name, - "Value", record_values.more_metadata.value); - - std::vector record_ids = { - more_metadata_row1->getId(), - more_metadata_row2->getId() - }; - - std::vector> recovered_more_metadata_rows; - - numbers_db->findObjects( - "MoreMetadata", record_ids, recovered_more_metadata_rows); - - EXPECT_EQUAL(recovered_more_metadata_rows.size(), - record_ids.size()); - - //Verify the first MoreMetadata record. - EXPECT_NOTEQUAL(recovered_more_metadata_rows[0].get(), nullptr); - - EXPECT_EQUAL(recovered_more_metadata_rows[0]->getPropertyString("Name"), - find_objs_expected_ans[0].more_metadata.name); - - EXPECT_EQUAL(recovered_more_metadata_rows[0]->getPropertyDouble("Value"), - find_objs_expected_ans[0].more_metadata.value); - - //Verify the second MoreMetadata record. - EXPECT_NOTEQUAL(recovered_more_metadata_rows[1].get(), nullptr); - - EXPECT_EQUAL(recovered_more_metadata_rows[1]->getPropertyString("Name"), - find_objs_expected_ans[1].more_metadata.name); - - EXPECT_EQUAL(recovered_more_metadata_rows[1]->getPropertyDouble("Value"), - find_objs_expected_ans[1].more_metadata.value); - - //Verify that we can use ObjectQuery to find records instead - //of just using findObject(s)() with database ID(s). - std::unique_ptr query = numbers_db-> - createObjectQueryForTable("MoreMetadata"); - - EXPECT_NOTEQUAL(query.get(), nullptr); - - //Set up the query to look for the second MoreMetadata - //record we just created above. - query->addConstraints( - "Name", simdb::constraints::equal, - record_values.more_metadata.name); - - std::string name_from_obj_query; - double value_from_obj_query; - - query->writeResultIterationsTo( - "Name", &name_from_obj_query, - "Value", &value_from_obj_query); - - auto result_iter = query->executeQuery(); - - //We should have found one record... - EXPECT_TRUE(result_iter->getNext()); - - //...and only one record. - EXPECT_FALSE(result_iter->getNext()); - - //Validate the record properties. - EXPECT_EQUAL(name_from_obj_query, record_values.more_metadata.name); - EXPECT_EQUAL(value_from_obj_query, record_values.more_metadata.value); -} - -void testNamespaceWithoutSchemaBuilder() -{ - PRINT_ENTER_TEST - - simdb::DatabaseRoot db_root(DB_DIR); - auto no_schema_namespace = db_root.getNamespace("NoSchemaBuilder"); - EXPECT_NOTEQUAL(no_schema_namespace, nullptr); - - EXPECT_FALSE(no_schema_namespace->databaseConnectionEstablished()); - - no_schema_namespace->addToSchema([](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("PalindromePhrases") - .addColumn("Fiz", dt::string_t) - .addColumn("Fuz", dt::double_t); - }); - - EXPECT_FALSE(no_schema_namespace->databaseConnectionEstablished()); - auto db = no_schema_namespace->getDatabase(); - EXPECT_TRUE(no_schema_namespace->databaseConnectionEstablished()); - - struct Data { - std::string fiz; - double fuz; - - static Data createRandom() { - Data d; - d.fiz = simdb::utils::chooseRand(); - d.fuz = simdb::utils::chooseRand(); - return d; - } - }; - - Data expected1 = Data::createRandom(); - Data expected2 = Data::createRandom(); - Data expected3 = Data::createRandom(); - Data expected4 = Data::createRandom(); - - //Overwrite the randomly generated 'fiz' values - //so we can get multiple records using ObjectQuery - //in a deterministic way. - expected1.fiz = "a toyota"; - expected2.fiz = "race fast"; - expected3.fiz = "safe car"; - expected4.fiz = "a toyota"; - - auto table = db->getTable("PalindromePhrases"); - - table->createObjectWithArgs( - "Fiz", expected1.fiz, - "Fuz", expected1.fuz); - - table->createObjectWithArgs( - "Fiz", expected2.fiz, - "Fuz", expected2.fuz); - - table->createObjectWithArgs( - "Fiz", expected3.fiz, - "Fuz", expected3.fuz); - - table->createObjectWithArgs( - "Fiz", expected4.fiz, - "Fuz", expected4.fuz); - - std::unique_ptr query = - db->createObjectQueryForTable("PalindromePhrases"); - - query->addConstraints( - "Fiz", simdb::constraints::equal, "a toyota"); - - std::string actual_fiz; - double actual_fuz; - - query->writeResultIterationsTo( - "Fiz", &actual_fiz, - "Fuz", &actual_fuz); - - auto result_iter = query->executeQuery(); - - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(actual_fiz, expected1.fiz); - EXPECT_EQUAL(actual_fuz, expected1.fuz); - - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(actual_fiz, expected4.fiz); - EXPECT_EQUAL(actual_fuz, expected4.fuz); - - //There should have been exactly two matches found. - EXPECT_FALSE(result_iter->getNext()); -} - -//Schema builder used for the "macros edge cases" unit test below. -void BuildFooSchema1(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("MyTable") - .addColumn("x", dt::string_t) - .addColumn("y", dt::double_t); -} - -//Schema builder used for the "macros edge cases" unit test below. -void BuildFooSchema2(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("MyTable") - .addColumn("x", dt::string_t) - .addColumn("y", dt::int32_t); -} - -//Schema builder used for the "macros edge cases" unit test below. -void BuildIdenticalFooSchema1(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("MyTable") - .addColumn("x", dt::string_t) - .addColumn("y", dt::double_t); -} - -//Connection proxy factory used for "macros edge cases" unit test below. -simdb::DbConnProxy * CreateFooProxy1() -{ - return new simdb::SQLiteConnProxy; -} - -//Connection proxy factory used for "macros edge cases" unit test below. -simdb::DbConnProxy * CreateFooProxy2() -{ - return new simdb::SQLiteConnProxy; -} - -//Connection proxy factory used for "macros edge cases" unit test below. -simdb::DbConnProxy * CreateFooProxy3() -{ - return new simdb::HDF5ConnProxy; -} - -//Test structure used in testRegistrationMacrosEdgeCases() below. -struct TestOverlap { - double a, b; - float c, d; - int16_t e, f; - std::string g, h; - - static TestOverlap makeRandom() { - TestOverlap test; - test.a = simdb::utils::chooseRand(); - test.b = simdb::utils::chooseRand(); - test.c = simdb::utils::chooseRand(); - test.d = simdb::utils::chooseRand(); - test.e = simdb::utils::chooseRand(); - test.f = simdb::utils::chooseRand(); - test.g = simdb::utils::chooseRand(); - test.h = simdb::utils::chooseRand(); - return test; - } - - void clear() { - a = b = c = d = e = f = 0; - g = h = ""; - } -}; - -//Equivalence check for EXPECT_EQUAL(TestOverlap, TestOverlap) -bool operator==(const TestOverlap & lhs, const TestOverlap & rhs) -{ - return lhs.a == rhs.a && - lhs.b == rhs.b && - lhs.c == rhs.c && - lhs.d == rhs.d && - lhs.e == rhs.e && - lhs.f == rhs.f && - lhs.g == rhs.g && - lhs.h == rhs.h; -} - -//Equivalence check for EXPECT_NOTEQUAL(TestOverlap, TestOverlap) -bool operator!=(const TestOverlap & lhs, const TestOverlap & rhs) -{ - return !(lhs == rhs); -} - -//Stream operator EXPECT_EQUAL(TestOverlap, TestOverlap) needs. -std::ostream & operator<<(std::ostream & os, const TestOverlap & val) -{ - os << " a (double): " << val.a << "\n" - << " b (double): " << val.b << "\n" - << " c (float): " << val.c << "\n" - << " d (float): " << val.d << "\n" - << " e (int16_t): " << val.e << "\n" - << " f (int16_t): " << val.f << "\n" - << " g (string): " << val.g << "\n" - << " h (string): " << val.h << "\n" - << std::endl; - - return os; -} - -void testRegistrationMacrosEdgeCases() -{ - PRINT_ENTER_TEST - - //Typical registration - same as other tests above. - EXPECT_NOTHROW(REGISTER_SIMDB_NAMESPACE(Strings, SQLite)); - - //Test case-insensitivity. Aside from that, double- - //registering the Strings namespace for the SQLite - //database type is ignored... - EXPECT_NOTHROW(REGISTER_SIMDB_NAMESPACE(StRiNgS, sqlITE)); - - //...BUT if we try to double-register the same namespace - //for HDF5, SimDB currently does not allow it. - EXPECT_THROW(REGISTER_SIMDB_NAMESPACE(Strings, HDF5)); - - //Typical registration - attach a default schema builder - //to the registered namespace. - EXPECT_NOTHROW(REGISTER_SIMDB_SCHEMA_BUILDER(Foo, BuildFooSchema1)); - - //Even though namespace Foo already has a schema builder, - //and this second builder we're trying to register here has - //a different table/column configuration than the schema - //already registered for namespace Foo, it still should not - //throw right away. If we try to access the namespace Foo - //however, *then* we expect it to throw. - EXPECT_NOTHROW(REGISTER_SIMDB_SCHEMA_BUILDER(Foo, BuildFooSchema2)); - - //Ignored registration - same namespace Foo, but this - //schema builder produces a table/column configuration - //which is identical to the schema already defined for - //namespace Foo. - EXPECT_NOTHROW(REGISTER_SIMDB_SCHEMA_BUILDER(Foo, [](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("MyTable") - .addColumn("x", dt::string_t) - .addColumn("y", dt::double_t); - })); - - //Proxy factory registration should never throw, but - //note that the first two factories are going to be - //ignored; only the third will take effect. - EXPECT_NOTHROW(REGISTER_SIMDB_PROXY_CREATE_FUNCTION(Foo, CreateFooProxy1)); - EXPECT_NOTHROW(REGISTER_SIMDB_PROXY_CREATE_FUNCTION(Foo, CreateFooProxy2)); - EXPECT_NOTHROW(REGISTER_SIMDB_PROXY_CREATE_FUNCTION(Foo, []() { - return new simdb::SQLiteConnProxy; - })); - - simdb::DatabaseRoot db_root(DB_DIR); - EXPECT_NOTHROW(REGISTER_SIMDB_NAMESPACE(Foo, SQLite)); - EXPECT_THROW((void)db_root.getNamespace("Foo")); - - //Let's register a few schema builders with SimDB. - //Each of these builders will add its own tables - //to the same namespace, and there will be some - //overlap in the table/column definitions. But - //none of the table configurations conflicts with - //other callbacks' table configurations, so SimDB - //should be able to combine them. - REGISTER_SIMDB_NAMESPACE(SchemaOverlap, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(SchemaOverlap, [](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - schema.addTable("Overlap1") - .addColumn("a", dt::double_t) - .addColumn("b", dt::double_t); - schema.addTable("Overlap2") - .addColumn("c", dt::float_t) - .addColumn("d", dt::float_t); - }); - REGISTER_SIMDB_SCHEMA_BUILDER(SchemaOverlap, [](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - schema.addTable("Overlap2") - .addColumn("c", dt::float_t) - .addColumn("d", dt::float_t); - schema.addTable("Overlap3") - .addColumn("e", dt::int16_t) - .addColumn("f", dt::int16_t); - }); - REGISTER_SIMDB_SCHEMA_BUILDER(SchemaOverlap, [](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - schema.addTable("Overlap3") - .addColumn("e", dt::int16_t) - .addColumn("f", dt::int16_t); - schema.addTable("Overlap4") - .addColumn("g", dt::string_t) - .addColumn("h", dt::string_t); - }); - - auto overlap_namespace = db_root.getNamespace("SchemaOverlap"); - EXPECT_NOTEQUAL(overlap_namespace, nullptr); - - auto overlap_table1 = overlap_namespace-> - getDatabase()->getTable("Overlap1"); - - auto overlap_table2 = overlap_namespace-> - getDatabase()->getTable("Overlap2"); - - auto overlap_table3 = overlap_namespace-> - getDatabase()->getTable("Overlap3"); - - auto overlap_table4 = overlap_namespace-> - getDatabase()->getTable("Overlap4"); - - std::vector overlap_values; - overlap_values.reserve(10); - for (size_t idx = 0; idx < overlap_values.capacity(); ++idx) { - overlap_values.emplace_back(TestOverlap::makeRandom()); - const TestOverlap & input_data = overlap_values.back(); - - overlap_table1->createObjectWithArgs( - "a", input_data.a, - "b", input_data.b); - - overlap_table2->createObjectWithArgs( - "c", input_data.c, - "d", input_data.d); - - overlap_table3->createObjectWithArgs( - "e", input_data.e, - "f", input_data.f); - - overlap_table4->createObjectWithArgs( - "g", input_data.g, - "h", input_data.h); - } - - auto overlap_db = overlap_namespace->getDatabase(); - - auto overlap_query1 = overlap_db->createObjectQueryForTable("Overlap1"); - EXPECT_EQUAL(overlap_query1->countMatches(), overlap_values.size()); - - auto overlap_query2 = overlap_db->createObjectQueryForTable("Overlap2"); - EXPECT_EQUAL(overlap_query2->countMatches(), overlap_values.size()); - - auto overlap_query3 = overlap_db->createObjectQueryForTable("Overlap3"); - EXPECT_EQUAL(overlap_query3->countMatches(), overlap_values.size()); - - auto overlap_query4 = overlap_db->createObjectQueryForTable("Overlap4"); - EXPECT_EQUAL(overlap_query4->countMatches(), overlap_values.size()); - - TestOverlap actual; - auto all_clear = [&]() { - actual.clear(); - }; - - overlap_query1->writeResultIterationsTo("a", &actual.a, "b", &actual.b); - overlap_query2->writeResultIterationsTo("c", &actual.c, "d", &actual.d); - overlap_query3->writeResultIterationsTo("e", &actual.e, "f", &actual.f); - overlap_query4->writeResultIterationsTo("g", &actual.g, "h", &actual.h); - all_clear(); - - size_t overlap_results_idx = 0; - auto overlap_results_iter1 = overlap_query1->executeQuery(); - auto overlap_results_iter2 = overlap_query2->executeQuery(); - auto overlap_results_iter3 = overlap_query3->executeQuery(); - auto overlap_results_iter4 = overlap_query4->executeQuery(); - auto all_get_next = [&]() { - return overlap_results_iter1->getNext() && - overlap_results_iter2->getNext() && - overlap_results_iter3->getNext() && - overlap_results_iter4->getNext(); - }; - - while (all_get_next()) { - const auto & expected = overlap_values.at(overlap_results_idx++); - EXPECT_EQUAL(actual, expected); - all_clear(); - } -} - -int main() -{ - //! At minimum, we must register our database namespaces - //! with an associated database type (SQLite, HDF5, etc.) - REGISTER_SIMDB_NAMESPACE(Strings, SQLite); - REGISTER_SIMDB_NAMESPACE(Numbers, SQLite); - - //! Schema definitions for each SimDB namespace can either - //! be registered with this macro, or inlined with a lambda - //! in user code. It also works with a combination of the - //! two: hard code all tables you always need for your - //! database namespace, put it in a schema builder callback, - //! and register it with this macro. You can request the - //! DatabaseNamespace object from the DatabaseRoot later - //! on and add extra tables if you need to. The schemas - //! will be combined under the hood. - REGISTER_SIMDB_SCHEMA_BUILDER(Strings, StringsSchemaBuilder); - REGISTER_SIMDB_SCHEMA_BUILDER(Numbers, [](simdb::Schema & schema) { - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addColumn("First", dt::int32_t) - .addColumn("Second", dt::double_t); - - schema.addTable("Metadata") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::int64_t); - }); - - //! In order to access the ObjectManager for the database - //! namespace object we'll create, we need to register a - //! factory method to create the appropriate DbConnProxy - //! subclass. - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(SQLite, []() { - return new simdb::SQLiteConnProxy; - }); - - //! Let's also register a SQLite namespace without any - //! schema build function to go with it. We will define - //! the schema ourselves with a call to the addToSchema() - //! method that SimDB provides. - REGISTER_SIMDB_NAMESPACE(NoSchemaBuilder, SQLite); - - testNamespaceSchemas(); - testNamespaceRecords(); - testNamespaceWithoutSchemaBuilder(); - testRegistrationMacrosEdgeCases(); -} diff --git a/sparta/simdb/test/CoreDatabase/test_dbs/placeholder.txt b/sparta/simdb/test/CoreDatabase/test_dbs/placeholder.txt deleted file mode 100644 index 09e11edc38..0000000000 --- a/sparta/simdb/test/CoreDatabase/test_dbs/placeholder.txt +++ /dev/null @@ -1,4 +0,0 @@ -This file serves as a placeholder to keep the 'test_dbs' -subdirectory in SimDB's test/Database folder in git. We -don't want to use things like boost::filesystem in SimDB -to create temp folders. diff --git a/sparta/simdb/test/HDF5Database/CMakeLists.txt b/sparta/simdb/test/HDF5Database/CMakeLists.txt deleted file mode 100644 index d50885c88e..0000000000 --- a/sparta/simdb/test/HDF5Database/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -project(SIMDB_HDF5Database_test) - -add_executable(SIMDB_HDF5Database_test HDF5Database_test.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_recursive_copy(SIMDB_HDF5Database_test test_dbs) - -# TODO: -# It seems that there are some known issues with HDF5/valgrind. -# https://support.hdfgroup.org/HDF5/faq/valgrind.html -# -# This needs investigation, and valgrind should be re-enabled -# for this test. -simdb_named_test_no_valgrind(SIMDB_HDF5Database_test SIMDB_HDF5Database_test) diff --git a/sparta/simdb/test/HDF5Database/HDF5Database_test.cpp b/sparta/simdb/test/HDF5Database/HDF5Database_test.cpp deleted file mode 100644 index 85f6372af8..0000000000 --- a/sparta/simdb/test/HDF5Database/HDF5Database_test.cpp +++ /dev/null @@ -1,625 +0,0 @@ -/*! - * \file HDF5Database_test.cpp - * - * \brief Tests functionality of SimDB's HDF5 implementation - */ - -#include "simdb/test/SimDBTester.hpp" - -//Core database headers -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/Errors.hpp" - -//HDF5-specific headers -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" - -//Standard headers -#include - -#define DB_DIR "test_dbs" -#define MatrixDblNumElems 2 -#define MatrixInt32NumRows 3 -#define MatrixInt32NumCols 2 - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -#define CREATE_HDF5_SCHEMA(obj_mgr, schema) \ - try { \ - obj_mgr.disableWarningMessages(); \ - const bool success = obj_mgr.createDatabaseFromSchema( \ - schema, std::unique_ptr(new simdb::HDF5ConnProxy)); \ - if (!success) { \ - throw; \ - } \ - } catch (...) { \ - throw simdb::DBException("Could not create HDF5 schema"); \ - } - -std::mt19937 rng; - -//! \brief Pick a random integral number -template -typename std::enable_if< - std::is_integral::value, -T>::type -chooseRand() -{ - std::uniform_int_distribution dist; - return dist(rng); -} - -//! \brief Pick a random floating-point number -template -typename std::enable_if< - std::is_floating_point::value, - T>::type -chooseRand() -{ - std::normal_distribution dist(0, 1000); - return dist(rng); -} - -//! \brief Fixed-size struct full of all the supported -//! POD data types in HDF5 SimDB. The word "compound" -//! is seen throughout this file, and it means the same -//! thing as "struct" - Compound is what HDF5 calls -//! structured data types. -struct CompoundPOD { - char ch; - int8_t i1; - uint8_t ui1; - int16_t i2; - uint16_t ui2; - int32_t i4; - uint32_t ui4; - int64_t i8; - uint64_t ui8; - float flt; - double dbl; -}; - -//! Create a randomized struct. Values are fed into -//! HDF5 record creation APIs, read back from disk, -//! and verified for accuracy. -CompoundPOD createRandomCompoundPOD() -{ - CompoundPOD comp; - comp.ch = (char) chooseRand(); - comp.i1 = (int8_t) chooseRand(); - comp.ui1 = (uint8_t)chooseRand(); - comp.i2 = chooseRand(); - comp.ui2 = chooseRand(); - comp.i4 = chooseRand(); - comp.ui4 = chooseRand(); - comp.i8 = chooseRand(); - comp.ui8 = chooseRand(); - comp.flt = chooseRand(); - comp.dbl = chooseRand(); - return comp; -} - -//! \brief Given an ObjectRef wrapping an HDF5 record -//! on disk, and the expected CompoundPOD values, compare -//! the record value for accuracy. -void verifyCompound(std::unique_ptr & row, - const CompoundPOD & comp) -{ - EXPECT_NOTEQUAL(row.get(), nullptr); - if (!row) { - return; - } - EXPECT_EQUAL(row->getPropertyChar("ch"), comp.ch); - EXPECT_EQUAL(row->getPropertyInt8("i1"), comp.i1); - EXPECT_EQUAL(row->getPropertyUInt8("ui1"), comp.ui1); - EXPECT_EQUAL(row->getPropertyInt16("i2"), comp.i2); - EXPECT_EQUAL(row->getPropertyUInt16("ui2"), comp.ui2); - EXPECT_EQUAL(row->getPropertyInt32("i4"), comp.i4); - EXPECT_EQUAL(row->getPropertyUInt32("ui4"), comp.ui4); - EXPECT_EQUAL(row->getPropertyInt64("i8"), comp.i8); - EXPECT_EQUAL(row->getPropertyUInt64("ui8"), comp.ui8); - EXPECT_EQUAL(row->getPropertyFloat("flt"), comp.flt); - EXPECT_EQUAL(row->getPropertyDouble("dbl"), comp.dbl); -} - -//! \brief Use the simdb::Table::addField() API to build -//! a fixed-size, compound data type table for testing -//! -//! \return Schema object containing a table suitable for -//! writing CompoundPOD structs to disk -simdb::Schema createSchemaForContiguousCompoundPOD( - const simdb::CompressionType compression) -{ - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("MyCompound", compression) - .addField("ch", dt::char_t, FOFFSET(CompoundPOD,ch)) - .addField("i1", dt::int8_t, FOFFSET(CompoundPOD,i1)) - .addField("ui1", dt::uint8_t, FOFFSET(CompoundPOD,ui1)) - .addField("i2", dt::int16_t, FOFFSET(CompoundPOD,i2)) - .addField("ui2", dt::uint16_t, FOFFSET(CompoundPOD,ui2)) - .addField("i4", dt::int32_t, FOFFSET(CompoundPOD,i4)) - .addField("ui4", dt::uint32_t, FOFFSET(CompoundPOD,ui4)) - .addField("i8", dt::int64_t, FOFFSET(CompoundPOD,i8)) - .addField("ui8", dt::uint64_t, FOFFSET(CompoundPOD,ui8)) - .addField("flt", dt::float_t, FOFFSET(CompoundPOD,flt)) - .addField("dbl", dt::double_t, FOFFSET(CompoundPOD,dbl)); - - return schema; -} - -//! \brief Use the simdb::Table::addColumn() API to build -//! a fixed-size, compound data type table for testing -//! -//! \return Schema object containing a table suitable for -//! writing CompoundPOD structs to disk -simdb::Schema createSchemaForNonContiguousCompoundPOD( - const simdb::CompressionType compression) -{ - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("MyCompound", compression) - .addColumn("ch", dt::char_t) - .addColumn("i1", dt::int8_t) - .addColumn("ui1", dt::uint8_t) - .addColumn("i2", dt::int16_t) - .addColumn("ui2", dt::uint16_t) - .addColumn("i4", dt::int32_t) - .addColumn("ui4", dt::uint32_t) - .addColumn("i8", dt::int64_t) - .addColumn("ui8", dt::uint64_t) - .addColumn("flt", dt::float_t) - .addColumn("dbl", dt::double_t); - - return schema; -} - -//! \brief Fixed-size struct full of all the supported -//! POD data types in HDF5 SimDB, including fields that -//! are non-scalar (but still fixed-size) POD's. -struct CompoundWithMatrixPOD -{ - char ch; - int8_t i1; - uint8_t ui1; - int16_t i2; - uint16_t ui2; - int32_t i4; - uint32_t ui4; - int64_t i8; - uint64_t ui8; - float flt; - double dbl; - double dblmat[MatrixDblNumElems]; - int32_t i4mat[MatrixInt32NumRows][MatrixInt32NumCols]; -}; - -//! Create a randomized struct. Values are fed into -//! HDF5 record creation APIs, read back from disk, -//! and verified for accuracy. -CompoundWithMatrixPOD createRandomCompoundWithMatrixPOD() -{ - CompoundWithMatrixPOD comp; - comp.ch = (char) chooseRand(); - comp.i1 = (int8_t) chooseRand(); - comp.ui1 = (uint8_t)chooseRand(); - comp.i2 = chooseRand(); - comp.ui2 = chooseRand(); - comp.i4 = chooseRand(); - comp.ui4 = chooseRand(); - comp.i8 = chooseRand(); - comp.ui8 = chooseRand(); - comp.flt = chooseRand(); - comp.dbl = chooseRand(); - for (int i = 0; i < MatrixDblNumElems; ++i) { - comp.dblmat[i] = 3.14 * rand(); - } - for (int i = 0; i < MatrixInt32NumRows; ++i) { - for (int j = 0; j < MatrixInt32NumCols; ++j) { - comp.i4mat[i][j] = 1.5245 * rand(); - } - } - return comp; -} - -//! \brief Given an ObjectRef wrapping an HDF5 record -//! on disk, and the expected CompoundPOD values, compare -//! the record value for accuracy. -void verifyCompoundMatrix(std::unique_ptr & row, - const CompoundWithMatrixPOD & comp) -{ - EXPECT_NOTEQUAL(row.get(), nullptr); - if (!row) { - return; - } - EXPECT_EQUAL(row->getPropertyChar("ch"), comp.ch); - EXPECT_EQUAL(row->getPropertyInt8("i1"), comp.i1); - EXPECT_EQUAL(row->getPropertyUInt8("ui1"), comp.ui1); - EXPECT_EQUAL(row->getPropertyInt16("i2"), comp.i2); - EXPECT_EQUAL(row->getPropertyUInt16("ui2"), comp.ui2); - EXPECT_EQUAL(row->getPropertyInt32("i4"), comp.i4); - EXPECT_EQUAL(row->getPropertyUInt32("ui4"), comp.ui4); - EXPECT_EQUAL(row->getPropertyInt64("i8"), comp.i8); - EXPECT_EQUAL(row->getPropertyUInt64("ui8"), comp.ui8); - EXPECT_EQUAL(row->getPropertyFloat("flt"), comp.flt); - EXPECT_EQUAL(row->getPropertyDouble("dbl"), comp.dbl); - - //TODO: Matrix data answer checking. We'll need a more - //user-friendly ObjectRef API for reading this data back - //in from disk. -} - -//! \brief Use the simdb::Table::addField() API to build -//! a fixed-size, compound data type table for testing -//! -//! \return Schema object containing a table suitable for -//! writing CompoundPOD structs to disk -simdb::Schema createSchemaForContiguousCompoundMatrixPOD() -{ - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("MyCompound") - .addField("ch", dt::char_t, FOFFSET(CompoundWithMatrixPOD,ch)) - .addField("i1", dt::int8_t, FOFFSET(CompoundWithMatrixPOD,i1)) - .addField("ui1", dt::uint8_t, FOFFSET(CompoundWithMatrixPOD,ui1)) - .addField("i2", dt::int16_t, FOFFSET(CompoundWithMatrixPOD,i2)) - .addField("ui2", dt::uint16_t, FOFFSET(CompoundWithMatrixPOD,ui2)) - .addField("i4", dt::int32_t, FOFFSET(CompoundWithMatrixPOD,i4)) - .addField("ui4", dt::uint32_t, FOFFSET(CompoundWithMatrixPOD,ui4)) - .addField("i8", dt::int64_t, FOFFSET(CompoundWithMatrixPOD,i8)) - .addField("ui8", dt::uint64_t, FOFFSET(CompoundWithMatrixPOD,ui8)) - .addField("flt", dt::float_t, FOFFSET(CompoundWithMatrixPOD,flt)) - .addField("dbl", dt::double_t, FOFFSET(CompoundWithMatrixPOD,dbl)) - .addField("dblmat", dt::double_t, FOFFSET(CompoundWithMatrixPOD,dblmat)) - ->setDimensions({MatrixDblNumElems}) - .addField("i4mat", dt::int32_t, FOFFSET(CompoundWithMatrixPOD,i4mat)) - ->setDimensions({MatrixInt32NumRows, MatrixInt32NumCols}); - - return schema; -} - -//! \brief Verify data accuracy in HDF5 database files when -//! using the Table::addColumn() API to build the schema, -//! and the TableRef::createObjectWithArgs() API to create -//! the records. -void testCompoundDataWritesWithArgs() -{ - PRINT_ENTER_TEST - - { - //Test without compression enabled - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::NONE); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectWithArgs( - "ch", c1.ch, "i1", c1.i1, "ui1", c1.ui1, - "i2", c1.i2, "ui2", c1.ui2, "i4", c1.i4, - "ui4", c1.ui4, "i8", c1.i8, "ui8", c1.ui8, - "flt", c1.flt, "dbl", c1.dbl); - - auto row2 = ctable->createObjectWithArgs( - "ch", c2.ch, "i1", c2.i1, "ui1", c2.ui1, - "i2", c2.i2, "ui2", c2.ui2, "i4", c2.i4, - "ui4", c2.ui4, "i8", c2.i8, "ui8", c2.ui8, - "flt", c2.flt, "dbl", c2.dbl); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - } - - { - //Now test with compression enabled - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::BEST_COMPRESSION_RATIO); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectWithArgs( - "ch", c1.ch, "i1", c1.i1, "ui1", c1.ui1, - "i2", c1.i2, "ui2", c1.ui2, "i4", c1.i4, - "ui4", c1.ui4, "i8", c1.i8, "ui8", c1.ui8, - "flt", c1.flt, "dbl", c1.dbl); - - auto row2 = ctable->createObjectWithArgs( - "ch", c2.ch, "i1", c2.i1, "ui1", c2.ui1, - "i2", c2.i2, "ui2", c2.ui2, "i4", c2.i4, - "ui4", c2.ui4, "i8", c2.i8, "ui8", c2.ui8, - "flt", c2.flt, "dbl", c2.dbl); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - } -} - -//! \brief Verify data accuracy in HDF5 database files when -//! using the Table::addColumn() API to build the schema, -//! and the TableRef::createObjectWithVals() API to create -//! the records. -void testCompoundDataWritesWithVals() -{ - PRINT_ENTER_TEST - - { - //Test without compression enabled - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::NONE); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectWithVals( - c1.ch, c1.i1, c1.ui1, - c1.i2, c1.ui2, c1.i4, - c1.ui4, c1.i8, c1.ui8, - c1.flt, c1.dbl); - - auto row2 = ctable->createObjectWithVals( - c2.ch, c2.i1, c2.ui1, - c2.i2, c2.ui2, c2.i4, - c2.ui4, c2.i8, c2.ui8, - c2.flt, c2.dbl); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - - //Now get a new TableRef tied to the same MyCompound - //table. We should be able to get records from the - //database file through either TableRef. - ctable = obj_mgr.getTable("MyCompound"); - auto c3 = createRandomCompoundPOD(); - - auto row3 = ctable->createObjectWithVals( - c3.ch, c3.i1, c3.ui1, - c3.i2, c3.ui2, c3.i4, - c3.ui4, c3.i8, c3.ui8, - c3.flt, c3.dbl); - - verifyCompound(row3, c3); - } - - { - //Now test with compression enabled - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::BEST_COMPRESSION_RATIO); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectWithVals( - c1.ch, c1.i1, c1.ui1, - c1.i2, c1.ui2, c1.i4, - c1.ui4, c1.i8, c1.ui8, - c1.flt, c1.dbl); - - auto row2 = ctable->createObjectWithVals( - c2.ch, c2.i1, c2.ui1, - c2.i2, c2.ui2, c2.i4, - c2.ui4, c2.i8, c2.ui8, - c2.flt, c2.dbl); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - - //Now get a new TableRef tied to the same MyCompound - //table. We should be able to get records from the - //database file through either TableRef. - ctable = obj_mgr.getTable("MyCompound"); - auto c3 = createRandomCompoundPOD(); - - auto row3 = ctable->createObjectWithVals( - c3.ch, c3.i1, c3.ui1, - c3.i2, c3.ui2, c3.i4, - c3.ui4, c3.i8, c3.ui8, - c3.flt, c3.dbl); - - verifyCompound(row3, c3); - } -} - -//! \brief Verify data accuracy in HDF5 database files when -//! using the Table::addField() API to build the schema, and -//! the TableRef::createObjectFromStruct() API to create the -//! records. -void testCompoundDataWritesFromStruct() -{ - PRINT_ENTER_TEST - - { - //Test without compression enabled - simdb::Schema schema = createSchemaForContiguousCompoundPOD( - simdb::CompressionType::NONE); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectFromStruct(c1); - auto row2 = ctable->createObjectFromStruct(c2); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - } - - { - //Now test with compression enabled - simdb::Schema schema = createSchemaForContiguousCompoundPOD( - simdb::CompressionType::BEST_COMPRESSION_RATIO); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - auto c2 = createRandomCompoundPOD(); - - auto row1 = ctable->createObjectFromStruct(c1); - auto row2 = ctable->createObjectFromStruct(c2); - - verifyCompound(row1, c1); - verifyCompound(row2, c2); - } -} - -//! \brief Verify data accuracy in HDF5 database files when -//! using the Table::addField() API to build the schema, and -//! the TableRef::createObjectFromStruct() API to create the -//! records. -void testCompoundMatrixDataWritesFromStruct() -{ - PRINT_ENTER_TEST - - simdb::Schema schema = createSchemaForContiguousCompoundMatrixPOD(); - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundWithMatrixPOD(); - auto c2 = createRandomCompoundWithMatrixPOD(); - - auto row1 = ctable->createObjectFromStruct(c1); - auto row2 = ctable->createObjectFromStruct(c2); - - verifyCompoundMatrix(row1, c1); - verifyCompoundMatrix(row2, c2); -} - -//! \brief Create a fixed-size HDF5 dataset, and attempt to -//! write records into it that are not the expected number -//! of bytes. Verify the exceptions are throw. -void testInvalidCompoundDataWritesWithVals() -{ - PRINT_ENTER_TEST - - //Test without compression enabled - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::NONE); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - - auto ctable = obj_mgr.getTable("MyCompound"); - auto c1 = createRandomCompoundPOD(); - - //Try to make an invalid call to createObjectWithArgs() - //again. Start with too few arguments. - EXPECT_THROW( - ctable->createObjectWithVals( - c1.ch, c1.i1, c1.ui1)); - - //Try to make an invalid call to createObjectWithArgs() - //again, this time calling with too many arguments. - EXPECT_THROW( - ctable->createObjectWithVals( - c1.ch, c1.i1, c1.ui1, - c1.i2, c1.ui2, c1.i4, - c1.ui4, c1.i8, c1.ui8, - c1.flt, c1.dbl, 1, 2, 3, 4, 5)); -} - -//! \brief Create an HDF5 database with some table records, -//! close the database and let the connection go out of scope, -//! then make a new connection to the same file and verify -//! the contents for accuracy. -void testDatabasePersistenceAcrossObjMgrs() -{ - PRINT_ENTER_TEST - - std::string db_file; - CompoundPOD baseline_struct1; - CompoundPOD baseline_struct2; - simdb::DatabaseID db_id1; - simdb::DatabaseID db_id2; - - { - simdb::Schema schema = createSchemaForNonContiguousCompoundPOD( - simdb::CompressionType::NONE); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_HDF5_SCHEMA(obj_mgr, schema); - db_file = obj_mgr.getDatabaseFile(); - - auto ctable = obj_mgr.getTable("MyCompound"); - baseline_struct1 = createRandomCompoundPOD(); - baseline_struct2 = createRandomCompoundPOD(); - auto & c1 = baseline_struct1; - auto & c2 = baseline_struct2; - - auto row1 = ctable->createObjectWithVals( - c1.ch, c1.i1, c1.ui1, - c1.i2, c1.ui2, c1.i4, - c1.ui4, c1.i8, c1.ui8, - c1.flt, c1.dbl); - - db_id1 = row1->getId(); - - auto row2 = ctable->createObjectWithVals( - c2.ch, c2.i1, c2.ui1, - c2.i2, c2.ui2, c2.i4, - c2.ui4, c2.i8, c2.ui8, - c2.flt, c2.dbl); - - db_id2 = row2->getId(); - } - - { - //The original ObjectManager / TableRef objects are gone, - //but we have the database filename and the database IDs - //of the records we just created. We should be able to - //get those records back using brand new ObjectManager's. - simdb::ObjectManager obj_mgr("."); - EXPECT_TRUE(obj_mgr.connectToExistingDatabase(db_file)); - - auto row1 = obj_mgr.findObject("MyCompound", db_id1); - auto row2 = obj_mgr.findObject("MyCompound", db_id2); - - verifyCompound(row1, baseline_struct1); - verifyCompound(row2, baseline_struct2); - } -} - -int main() -{ - rng.seed(time(0)); - - testCompoundDataWritesWithArgs(); - testCompoundDataWritesWithVals(); - testInvalidCompoundDataWritesWithVals(); - testCompoundDataWritesFromStruct(); - testCompoundMatrixDataWritesFromStruct(); - testDatabasePersistenceAcrossObjMgrs(); -} diff --git a/sparta/simdb/test/HDF5Database/test_dbs/placeholder.txt b/sparta/simdb/test/HDF5Database/test_dbs/placeholder.txt deleted file mode 100644 index 09e11edc38..0000000000 --- a/sparta/simdb/test/HDF5Database/test_dbs/placeholder.txt +++ /dev/null @@ -1,4 +0,0 @@ -This file serves as a placeholder to keep the 'test_dbs' -subdirectory in SimDB's test/Database folder in git. We -don't want to use things like boost::filesystem in SimDB -to create temp folders. diff --git a/sparta/simdb/test/SQLiteDatabase/CMakeLists.txt b/sparta/simdb/test/SQLiteDatabase/CMakeLists.txt deleted file mode 100644 index f0783e999d..0000000000 --- a/sparta/simdb/test/SQLiteDatabase/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -project(SIMDB_SQLiteDatabase_test) - -add_executable(SIMDB_SQLiteDatabase_test SQLiteDatabase_test.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_recursive_copy(SIMDB_SQLiteDatabase_test test_dbs) - -simdb_test(SIMDB_SQLiteDatabase_test SIMDB_SQLiteDatabase_test_RUN) diff --git a/sparta/simdb/test/SQLiteDatabase/SQLiteDatabase_test.cpp b/sparta/simdb/test/SQLiteDatabase/SQLiteDatabase_test.cpp deleted file mode 100644 index 31cde16f0d..0000000000 --- a/sparta/simdb/test/SQLiteDatabase/SQLiteDatabase_test.cpp +++ /dev/null @@ -1,1761 +0,0 @@ -/*! - * \file SQLiteDatabase_test.cpp - * - * \brief Tests functionality of SimDB's core functionality, - * including schema creation, INSERT/UPDATE/DELETE - */ - -#include "simdb/test/SimDBTester.hpp" - -//Core database headers -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/utils/BlobHelpers.hpp" - -//SQLite-specific headers -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include "simdb/impl/sqlite/Errors.hpp" -#include - -//Standard headers -#include - -#define DB_DIR "test_dbs" - -TEST_INIT; - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -#define CREATE_SQL_SCHEMA(obj_mgr, schema) \ - obj_mgr.disableWarningMessages(); \ - obj_mgr.createDatabaseFromSchema( \ - schema, std::unique_ptr(new simdb::SQLiteConnProxy)); - -void testBadSql() -{ - PRINT_ENTER_TEST - - simdb::ObjectManager obj_mgr(DB_DIR); - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("Dummy") - .addColumn("Dummy", dt::int32_t); - - CREATE_SQL_SCHEMA(obj_mgr, schema); - - auto db_proxy = dynamic_cast( - obj_mgr.getDbConn()); - EXPECT_TRUE(db_proxy != nullptr); - EXPECT_THROW(simdb::eval_sql(db_proxy, "THIS IS NOT VALID SQL")); -} - -void testBadFile() -{ - PRINT_ENTER_TEST - - const std::string fname = "test.db"; - { - std::ofstream invalid_file(fname); - invalid_file << "This is not a valid SQLite database!"; - } - - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_FALSE(obj_mgr.connectToExistingDatabase(fname)); -} - -void testInvalidSchema() -{ - PRINT_ENTER_TEST - - simdb::ObjectManager obj_mgr(DB_DIR); - using dt = simdb::ColumnDataType; - - simdb::Schema schema_with_nonscalar_cols; - schema_with_nonscalar_cols.addTable("Numbers") - .addColumn("MyScalar", dt::double_t) - .addColumn("MyNonScalar", dt::double_t) - ->setDimensions({4,7,2}); - - EXPECT_THROW(CREATE_SQL_SCHEMA(obj_mgr, schema_with_nonscalar_cols)); - - simdb::Schema schema_with_zero_dims; - schema_with_zero_dims.addTable("Numbers") - .addColumn("MyScalar", dt::double_t) - .addColumn("MyZeroDims", dt::double_t) - ->setDimensions({3,0,5}); - - EXPECT_THROW(CREATE_SQL_SCHEMA(obj_mgr, schema_with_zero_dims)); -} - -void testSqlSchema() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - - //Start by manually creating a Customers table with - //properties First, Last, Age, RewardsBal, and Password. - //These are chosen to include ints, strings, doubles, - //and blobs, which are the column data types currently - //supported. - simdb::Schema schema; - - //Set default values for all supported data types - const std::string default_first_name = "George"; - const std::string default_last_name = "Washington"; - const int32_t default_age = 67; - const double default_rewards_bal = 1000000.00; - - schema.addTable("Customers") - .addColumn("First", dt::string_t) - ->setDefaultValue( - default_first_name) - .addColumn("Last", dt::string_t) - ->setDefaultValue( - default_last_name) - .addColumn("Age", dt::int32_t) - ->setDefaultValue( - default_age) - .addColumn("RewardsBal", dt::double_t) - ->setDefaultValue( - default_rewards_bal) - .addColumn("Password", dt::blob_t); - - //Make sure we cannot try to set a default value for a blob - EXPECT_THROW( - schema.addTable("Blobs") - .addColumn("Foo", dt::blob_t) - ->setDefaultValue(0) - ); - - simdb::DatabaseID customer1_id = 0, customer2_id = 0, customer3_id = 0; - std::string db_file_path; - - //Make up some random-length (10 to 100 values) array of - //numbers that represent some kind of password - auto random_password = []() { - auto random_len = rand() % 91 + 10; - std::vector password(random_len); - for (auto & val : password) { - val = rand() * M_PI; - } - return password; - }; - - const std::vector customer1_password = random_password(); - const std::vector customer2_password = random_password(); - - std::tuple> customer1_info = - std::make_tuple("Alice", "Smith", 29, 74.28, customer1_password); - - std::tuple> customer2_info = - std::make_tuple("Bob", "Thompson", 41, 104.56, customer2_password); - - { - //Create the physical database from these schema objects - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - db_file_path = obj_mgr.getDatabaseFile(); - - std::unique_ptr customers_tbl = obj_mgr.getTable("Customers"); - - //Create two customer records - std::unique_ptr customer1 = customers_tbl->createObjectWithArgs( - "First", std::get<0>(customer1_info), - "Last", std::get<1>(customer1_info), - "Age", std::get<2>(customer1_info), - "RewardsBal", std::get<3>(customer1_info)); - - std::unique_ptr customer2 = customers_tbl->createObjectWithArgs( - "First", std::get<0>(customer2_info), - "Last", std::get<1>(customer2_info), - "Age", std::get<2>(customer2_info), - "RewardsBal", std::get<3>(customer2_info)); - - //Make sure the AUTOINCREMENT is working - database IDs - //should be unique across records in the same table - customer1_id = customer1->getId(); - customer2_id = customer2->getId(); - EXPECT_NOTEQUAL(customer1_id, customer2_id); - - //Helper to make a Blob descriptor out of an array of numbers - auto make_password_blob = [](const std::vector & password) { - simdb::Blob blob; - blob.data_ptr = password.data(); - blob.num_bytes = password.size() * sizeof(double); - return blob; - }; - - auto password1_blob = make_password_blob(std::get<4>(customer1_info)); - auto password2_blob = make_password_blob(std::get<4>(customer2_info)); - - customer1->setPropertyBlob("Password", password1_blob); - customer2->setPropertyBlob("Password", password2_blob); - - //Now create a third customer, but do not specify any of the - //column values. We will use this customer record to ensure - //the default column values we specified took hold. - std::unique_ptr customer3 = customers_tbl->createObject(); - customer3_id = customer3->getId(); - - //Also verify that we get a null TableRef if we ask for - //a table that does not exist - std::unique_ptr bad_table = obj_mgr.getTable("does-not-exist"); - EXPECT_EQUAL(bad_table.get(), nullptr); - - //Verify that we get a null ObjectRef if we ask for a record - //from a valid table, but a non-existant database ID. - std::unique_ptr bad_record = obj_mgr.findObject("Customers", 12345); - EXPECT_EQUAL(bad_record.get(), nullptr); - - //Go through the ObjectManager::findObjects() API for a few - //use cases, and verify the results. - std::vector> retrieved_customers; - - obj_mgr.findObjects("Customers", {customer1_id, customer2_id}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 2); - EXPECT_EQUAL(retrieved_customers[0]->getId(), customer1_id); - EXPECT_EQUAL(retrieved_customers[1]->getId(), customer2_id); - - obj_mgr.findObjects("Customers", {customer2_id, customer1_id}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 2); - EXPECT_EQUAL(retrieved_customers[0]->getId(), customer2_id); - EXPECT_EQUAL(retrieved_customers[1]->getId(), customer1_id); - - obj_mgr.findObjects("Customers", {customer1_id, 12345}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 2); - EXPECT_EQUAL(retrieved_customers[0]->getId(), customer1_id); - EXPECT_TRUE(retrieved_customers[1] == nullptr); - - obj_mgr.findObjects("Customers", {12345, customer1_id}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 2); - EXPECT_EQUAL(retrieved_customers[1]->getId(), customer1_id); - EXPECT_TRUE(retrieved_customers[0] == nullptr); - - obj_mgr.findObjects("Customers", {customer1_id}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 1); - EXPECT_EQUAL(retrieved_customers[0]->getId(), customer1_id); - - obj_mgr.findObjects("Customers", {12345}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 1); - EXPECT_TRUE(retrieved_customers[0] == nullptr); - - obj_mgr.findObjects("Customers", {}, retrieved_customers); - EXPECT_EQUAL(retrieved_customers.size(), 3); - EXPECT_EQUAL(retrieved_customers[0]->getId(), customer1_id); - EXPECT_EQUAL(retrieved_customers[1]->getId(), customer2_id); - EXPECT_EQUAL(retrieved_customers[2]->getId(), customer3_id); - } - - //The previous connection has gone out of scope and is closed. - //All we have is the full path to the database file, so let's - //try to connect to it again and inspect the record values. - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_TRUE(obj_mgr.connectToExistingDatabase(db_file_path)); - - //Get back customer1 info and verify the fields - std::unique_ptr retrieved_customer1 = - obj_mgr.findObject("Customers", customer1_id); - - EXPECT_EQUAL(retrieved_customer1->getId(), customer1_id); - - EXPECT_EQUAL(retrieved_customer1->getPropertyString("First"), - std::get<0>(customer1_info)); - - EXPECT_EQUAL(retrieved_customer1->getPropertyString("Last"), - std::get<1>(customer1_info)); - - EXPECT_EQUAL(retrieved_customer1->getPropertyInt32("Age"), - std::get<2>(customer1_info)); - - EXPECT_EQUAL(retrieved_customer1->getPropertyDouble("RewardsBal"), - std::get<3>(customer1_info)); - - std::vector customer1_retrieved_password; - retrieved_customer1->getPropertyBlob("Password", customer1_retrieved_password); - EXPECT_EQUAL(customer1_retrieved_password, customer1_password); - - //Get back customer2 info and verify the fields - std::unique_ptr retrieved_customer2 = - obj_mgr.findObject("Customers", customer2_id); - - EXPECT_EQUAL(retrieved_customer2->getId(), customer2_id); - - EXPECT_EQUAL(retrieved_customer2->getPropertyString("First"), - std::get<0>(customer2_info)); - - EXPECT_EQUAL(retrieved_customer2->getPropertyString("Last"), - std::get<1>(customer2_info)); - - EXPECT_EQUAL(retrieved_customer2->getPropertyInt32("Age"), - std::get<2>(customer2_info)); - - EXPECT_EQUAL(retrieved_customer2->getPropertyDouble("RewardsBal"), - std::get<3>(customer2_info)); - - std::vector customer2_retrieved_password; - retrieved_customer2->getPropertyBlob("Password", customer2_retrieved_password); - EXPECT_EQUAL(customer2_retrieved_password, customer2_password); - - //Get back customer3 info and verify the fields (DEFAULTS) - std::unique_ptr retrieved_customer3 = - obj_mgr.findObject("Customers", customer3_id); - - EXPECT_EQUAL(retrieved_customer3->getId(), customer3_id); - - EXPECT_EQUAL(retrieved_customer3->getPropertyString("First"), - default_first_name); - - EXPECT_EQUAL(retrieved_customer3->getPropertyString("Last"), - default_last_name); - - EXPECT_EQUAL(retrieved_customer3->getPropertyInt32("Age"), - default_age); - - EXPECT_EQUAL(retrieved_customer3->getPropertyDouble("RewardsBal"), - default_rewards_bal); -} - -void testSqlSchemaColumnModifiers() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - - { - simdb::Schema schema; - - EXPECT_NOTHROW( - schema.addTable("Customers") - .addColumn("LastName", dt::string_t) - ->indexAgainst( - "FirstName") - .addColumn("FirstName", dt::string_t) - ); - - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_NOTHROW(CREATE_SQL_SCHEMA(obj_mgr, schema)); - - auto tbl = obj_mgr.getTable("Customers"); - EXPECT_TRUE(tbl != nullptr); - - std::unique_ptr customer; - const std::string first_name = "George"; - const std::string last_name = "Washington"; - - EXPECT_NOTHROW( - customer = tbl->createObjectWithArgs( - "FirstName", first_name, - "LastName", last_name) - ); - - EXPECT_EQUAL(customer->getPropertyString("FirstName"), first_name); - EXPECT_EQUAL(customer->getPropertyString("LastName"), last_name); - } - - { - simdb::Schema schema; - - //Create a schema, but make a typo in one of the column names. - //It should not throw an exception until we try to give it to - //an ObjectManager for database instantiation. - - EXPECT_NOTHROW( - schema.addTable("Customers") - .addColumn("LastName", dt::string_t) - ->indexAgainst( - "FristName") - .addColumn("FirstName", dt::string_t) - ); - - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_THROW(CREATE_SQL_SCHEMA(obj_mgr, schema)); - } -} - -void testBasicDataTypes() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - schema.addTable("DTypes") - .addColumn("A", dt::int8_t) - .addColumn("B", dt::uint8_t) - .addColumn("C", dt::int16_t) - .addColumn("D", dt::uint16_t) - .addColumn("E", dt::int32_t) - .addColumn("F", dt::uint32_t) - .addColumn("G", dt::int64_t) - .addColumn("H", dt::uint64_t) - .addColumn("I", dt::string_t) - .addColumn("J", dt::char_t) - .addColumn("K", dt::float_t) - .addColumn("L", dt::double_t) - .addColumn("M", dt::blob_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_NOTHROW(CREATE_SQL_SCHEMA(obj_mgr, schema)); - - const int8_t A = -5; - const uint8_t B = 10; - const int16_t C = -20; - const uint16_t D = 40; - const int32_t E = -80; - const uint32_t F = 160; - const int64_t G = -320; - const uint64_t H = 640; - const std::string I = "minus seven twenty"; - const char J = '3'; - const float K = 0.14; - const double L = 0.00159265359; - const std::vector M = {0, 1, 2, 3, 4}; - - auto dtypes = obj_mgr.getTable("DTypes"); - auto row = dtypes->createObject(); - - row->setPropertyInt8 ("A", A); - row->setPropertyUInt8 ("B", B); - row->setPropertyInt16 ("C", C); - row->setPropertyUInt16 ("D", D); - row->setPropertyInt32 ("E", E); - row->setPropertyUInt32 ("F", F); - row->setPropertyInt64 ("G", G); - row->setPropertyUInt64 ("H", H); - row->setPropertyString ("I", I); - row->setPropertyChar ("J", J); - row->setPropertyFloat ("K", K); - row->setPropertyDouble ("L", L); - - simdb::Blob blob_m; - blob_m.data_ptr = M.data(); - blob_m.num_bytes = M.size() * sizeof(int32_t); - row->setPropertyBlob("M", blob_m); - - EXPECT_EQUAL(row->getPropertyInt8 ("A"), A); - EXPECT_EQUAL(row->getPropertyUInt8 ("B"), B); - EXPECT_EQUAL(row->getPropertyInt16 ("C"), C); - EXPECT_EQUAL(row->getPropertyUInt16 ("D"), D); - EXPECT_EQUAL(row->getPropertyInt32 ("E"), E); - EXPECT_EQUAL(row->getPropertyUInt32 ("F"), F); - EXPECT_EQUAL(row->getPropertyInt64 ("G"), G); - EXPECT_EQUAL(row->getPropertyUInt64 ("H"), H); - EXPECT_EQUAL(row->getPropertyString ("I"), I); - EXPECT_EQUAL(row->getPropertyChar ("J"), J); - EXPECT_EQUAL(row->getPropertyFloat ("K"), K); - EXPECT_EQUAL(row->getPropertyDouble ("L"), L); - - std::vector m; - row->getPropertyBlob("M", m); - EXPECT_EQUAL(m, M); - - row = dtypes->createObjectWithArgs( - "A", A, "B", B, "C", C, "D", D, - "E", E, "F", F, "G", G, "H", H, - "I", I, "J", J, "K", K, "L", L, - "M", M); - - EXPECT_EQUAL(row->getPropertyInt8 ("A"), A); - EXPECT_EQUAL(row->getPropertyUInt8 ("B"), B); - EXPECT_EQUAL(row->getPropertyInt16 ("C"), C); - EXPECT_EQUAL(row->getPropertyUInt16 ("D"), D); - EXPECT_EQUAL(row->getPropertyInt32 ("E"), E); - EXPECT_EQUAL(row->getPropertyUInt32 ("F"), F); - EXPECT_EQUAL(row->getPropertyInt64 ("G"), G); - EXPECT_EQUAL(row->getPropertyUInt64 ("H"), H); - EXPECT_EQUAL(row->getPropertyString ("I"), I); - EXPECT_EQUAL(row->getPropertyChar ("J"), J); - EXPECT_EQUAL(row->getPropertyFloat ("K"), K); - EXPECT_EQUAL(row->getPropertyDouble ("L"), L); - - int8_t A2; - uint8_t B2; - int16_t C2; - uint16_t D2; - int32_t E2; - uint32_t F2; - int64_t G2; - uint64_t H2; - std::string I2; - char J2; - float K2; - double L2; - std::vector M2; - - simdb::ObjectQuery query(obj_mgr, "DTypes"); - query.addConstraints("Id", simdb::constraints::equal, row->getId()); - - query.writeResultIterationsTo( - "A", &A2, "B", &B2, "C", &C2, "D", &D2, - "E", &E2, "F", &F2, "G", &G2, "H", &H2, - "I", &I2, "J", &J2, "K", &K2, "L", &L2, - "M", &M2); - - EXPECT_TRUE(query.executeQuery()->getNext()); - - EXPECT_EQUAL(A, A2); - EXPECT_EQUAL(B, B2); - EXPECT_EQUAL(C, C2); - EXPECT_EQUAL(D, D2); - EXPECT_EQUAL(E, E2); - EXPECT_EQUAL(F, F2); - EXPECT_EQUAL(G, G2); - EXPECT_EQUAL(H, H2); - EXPECT_EQUAL(I, I2); - EXPECT_EQUAL(J, J2); - EXPECT_EQUAL(K, K2); - EXPECT_EQUAL(L, L2); - EXPECT_EQUAL(M, M2); -} - -void test64BitInts() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("My64BitInts") - .addColumn("MySigned", dt::int64_t) - .addColumn("MyUnsigned", dt::uint64_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - constexpr int64_t min_signed = std::numeric_limits::min(); - constexpr int64_t max_signed = std::numeric_limits::max(); - - constexpr uint64_t min_unsigned = std::numeric_limits::min(); - constexpr uint64_t max_unsigned = std::numeric_limits::max(); - - auto tbl = obj_mgr.getTable("My64BitInts"); - - auto row = tbl->createObject(); - row->setPropertyInt64("MySigned", min_signed); - row->setPropertyUInt64("MyUnsigned", min_unsigned); - EXPECT_EQUAL(row->getPropertyInt64("MySigned"), min_signed); - EXPECT_EQUAL(row->getPropertyUInt64("MyUnsigned"), min_unsigned); - - row->setPropertyInt64("MySigned", max_signed); - row->setPropertyUInt64("MyUnsigned", max_unsigned); - EXPECT_EQUAL(row->getPropertyInt64("MySigned"), max_signed); - EXPECT_EQUAL(row->getPropertyUInt64("MyUnsigned"), max_unsigned); - - row = tbl->createObjectWithArgs( - "MySigned", min_signed, "MyUnsigned", min_unsigned); - EXPECT_EQUAL(row->getPropertyInt64("MySigned"), min_signed); - EXPECT_EQUAL(row->getPropertyUInt64("MyUnsigned"), min_unsigned); - - row = tbl->createObjectWithArgs( - "MySigned", max_signed, "MyUnsigned", max_unsigned); - EXPECT_EQUAL(row->getPropertyInt64("MySigned"), max_signed); - EXPECT_EQUAL(row->getPropertyUInt64("MyUnsigned"), max_unsigned); - - std::unique_ptr query( - new simdb::ObjectQuery(obj_mgr, "My64BitInts")); - - auto verify_int64 = [&](const int64_t expected) { - int64_t actual = 0; - query->writeResultIterationsTo("MySigned", &actual); - - auto result_iter = query->executeQuery(); - while (result_iter->getNext()) { - EXPECT_EQUAL(actual, expected); - } - }; - - query->addConstraints("MySigned", simdb::constraints::equal, min_signed); - verify_int64(min_signed); - - query->addConstraints("MySigned", simdb::constraints::equal, max_signed); - verify_int64(max_signed); - - auto verify_uint64 = [&](const uint64_t expected) { - uint64_t actual = 0; - query->writeResultIterationsTo("MyUnsigned", &actual); - - auto result_iter = query->executeQuery(); - while (result_iter->getNext()) { - EXPECT_EQUAL(actual, expected); - } - }; - - query->addConstraints("MyUnsigned", simdb::constraints::equal, min_unsigned); - verify_uint64(min_unsigned); - - query->addConstraints("MyUnsigned", simdb::constraints::equal, max_unsigned); - verify_uint64(max_unsigned); -} - -void testObjectQuery() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - - schema.addTable("ReportHeader") - .addColumn("ReportName", dt::string_t) - .addColumn("StartTime", dt::uint64_t) - .addColumn("EndTime", dt::uint64_t); - - schema.addTable("StatInstValues") - .addColumn("TimeseriesChunkID", dt::int32_t) - .addColumn("RawBytes", dt::blob_t) - .addColumn("NumPts", dt::int32_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - std::unique_ptr header_tbl = obj_mgr.getTable("ReportHeader"); - - struct RecordProps { - std::string report_name; - uint64_t start_time; - uint64_t end_time; - }; - - //------------------- Mini-test #1 ---------------------// - - RecordProps record1; - record1.report_name = "ObjectQueryTest1"; - record1.start_time = 5000; - record1.end_time = 100000; - - std::unique_ptr obj1 = header_tbl->createObjectWithArgs( - "ReportName", record1.report_name, - "StartTime", record1.start_time, - "EndTime", record1.end_time); - - RecordProps record2; - record2.report_name = "ObjectQueryTest2"; - record2.start_time = 6000; - record2.end_time = 97000; - - std::unique_ptr obj2 = header_tbl->createObjectWithArgs( - "ReportName", record2.report_name, - "StartTime", record2.start_time, - "EndTime", record2.end_time); - - RecordProps record3; - record3.report_name = record2.report_name; - record3.start_time = 5500; - record3.end_time = 114000; - - std::unique_ptr obj3 = header_tbl->createObjectWithArgs( - "ReportName", record3.report_name, - "StartTime", record3.start_time, - "EndTime", record3.end_time); - - //Now that we have a few records, let's ask for them back - simdb::ObjectQuery query(obj_mgr, "ReportHeader"); - RecordProps retrieved; - - //Look for records with StartTime>5200 AND EndTime<120000 - // (should be record2 and record3) - query.addConstraints( - "StartTime", simdb::constraints::greater, 5200, - "EndTime", simdb::constraints::less, 120000); - - query.writeResultIterationsTo( - "ReportName", &retrieved.report_name, - "StartTime", &retrieved.start_time, - "EndTime", &retrieved.end_time); - - std::unique_ptr result = query.executeQuery(); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record2.report_name); - EXPECT_EQUAL(retrieved.start_time, record2.start_time); - EXPECT_EQUAL(retrieved.end_time, record2.end_time); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record3.report_name); - EXPECT_EQUAL(retrieved.start_time, record3.start_time); - EXPECT_EQUAL(retrieved.end_time, record3.end_time); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //------------------- Mini-test #2 ---------------------// - - //Look for records with ReportName="ObjectQueryTest2" AND EndTime>=97000 - // (should also be record2 and record3) - query.addConstraints( - "ReportName", simdb::constraints::equal, record2.report_name, - "EndTime", simdb::constraints::greater_equal, record2.end_time); - - query.writeResultIterationsTo( - "StartTime", &retrieved.start_time, - "EndTime", &retrieved.end_time); - - result = query.executeQuery(); - - //Before validating the answers, clear out the old structs - retrieved.report_name.clear(); - retrieved.start_time = 0; - retrieved.end_time = 0; - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.start_time, record2.start_time); - EXPECT_EQUAL(retrieved.end_time, record2.end_time); - //Note that since we did *not* ask for ReportName or any other - //iteration values, this field should still be empty. - EXPECT_TRUE(retrieved.report_name.empty()); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.start_time, record3.start_time); - EXPECT_EQUAL(retrieved.end_time, record3.end_time); - //Note that since we did *not* ask for ReportName or any other - //iteration values, this field should still be empty. - EXPECT_TRUE(retrieved.report_name.empty()); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //Run another query looking for records with StartTime<50 - // (should be none) - query.addConstraints( - "StartTime", simdb::constraints::less, 50); - - query.writeResultIterationsTo( - "StartTime", &retrieved.start_time, - "EndTime", &retrieved.end_time); - - result = query.executeQuery(); - EXPECT_FALSE(result->getNext()); - - //Run another query without any constraints - // (should get all three records) - query.writeResultIterationsTo( - "ReportName", &retrieved.report_name, - "StartTime", &retrieved.start_time, - "EndTime", &retrieved.end_time); - - result = query.executeQuery(); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record1.report_name); - EXPECT_EQUAL(retrieved.start_time, record1.start_time); - EXPECT_EQUAL(retrieved.end_time, record1.end_time); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record2.report_name); - EXPECT_EQUAL(retrieved.start_time, record2.start_time); - EXPECT_EQUAL(retrieved.end_time, record2.end_time); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record3.report_name); - EXPECT_EQUAL(retrieved.start_time, record3.start_time); - EXPECT_EQUAL(retrieved.end_time, record3.end_time); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //Now let's make some table entries for StatInstValues. - //This has blob columns in it, which we will add to - //result set iterators to verify. - std::unique_ptr si_values_tbl = - obj_mgr.getTable("StatInstValues"); - - std::vector raw_si1 = {1, 3, 5, 7, 9}; - std::vector raw_si2 = {2, 4, 6, 8}; - const simdb::DatabaseID ts_chunk_id = 40; - - std::unique_ptr si_chunk1 = si_values_tbl->createObject(); - si_chunk1->setPropertyInt32("TimeseriesChunkID", ts_chunk_id); - - simdb::Blob blob_desc1; - blob_desc1.data_ptr = &raw_si1[0]; - blob_desc1.num_bytes = raw_si1.size() * sizeof(double); - si_chunk1->setPropertyBlob("RawBytes", blob_desc1); - si_chunk1->setPropertyInt32("NumPts", (int)raw_si1.size()); - - std::unique_ptr si_chunk2 = si_values_tbl->createObject(); - si_chunk2->setPropertyInt32("TimeseriesChunkID", ts_chunk_id); - - simdb::Blob blob_desc2; - blob_desc2.data_ptr = &raw_si2[0]; - blob_desc2.num_bytes = raw_si2.size() * sizeof(double); - si_chunk2->setPropertyBlob("RawBytes", blob_desc2); - si_chunk2->setPropertyInt32("NumPts", (int)raw_si2.size()); - - //Now run a query to get back both blobs one at a time. - //We should be able to incrementally pass over numerous - //chunks while only using one vector to do so. - simdb::ObjectQuery query2(obj_mgr, "StatInstValues"); - - int num_retrieved_si_values = 0; - std::vector retrieved_si_values; - - query2.addConstraints( - "TimeseriesChunkID", simdb::constraints::equal, ts_chunk_id); - - query2.writeResultIterationsTo( - "RawBytes", &retrieved_si_values, - "NumPts", &num_retrieved_si_values); - - result = query2.executeQuery(); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(num_retrieved_si_values, (int)raw_si1.size()); - EXPECT_EQUAL(retrieved_si_values, raw_si1); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(num_retrieved_si_values, (int)raw_si2.size()); - EXPECT_EQUAL(retrieved_si_values, raw_si2); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //------------------- Mini-test #3 ---------------------// - - header_tbl->deleteAllObjects(); - - //Verify the behavior of the "in_set" / "not_in_set" - //constraints. Run some queries that look like this: - // SELECT * FROM MyTable WHERE ThisInteger IN (4,56,99) - // SELECT * FROM MyTable WHERE ThisString NOT IN ('fiz','baz') - // etc. - RecordProps record4; - record4.report_name = "Foo"; - record4.end_time = 14000; - - RecordProps record5; - record5.report_name = "Bar"; - record5.end_time = 14000; - - RecordProps record6; - record6.report_name = "Biz"; - record6.end_time = 16000; - - RecordProps record7; - record7.report_name = "Baz"; - record7.end_time = 22000; - - std::unique_ptr objA = header_tbl->createObjectWithArgs( - "ReportName", record4.report_name, - "EndTime", record4.end_time); - - std::unique_ptr objB = header_tbl->createObjectWithArgs( - "ReportName", record5.report_name, - "EndTime", record5.end_time); - - std::unique_ptr objC = header_tbl->createObjectWithArgs( - "ReportName", record6.report_name, - "EndTime", record6.end_time); - - std::unique_ptr objD = header_tbl->createObjectWithArgs( - "ReportName", record7.report_name, - "EndTime", record7.end_time); - - //Run a query to get all records with report name that - //is either "Bar" or "Baz" (record5 and record7) - query.addConstraints( - "ReportName", simdb::constraints::in_set, {"Bar","Baz"}); - - query.writeResultIterationsTo( - "ReportName", &retrieved.report_name, - "EndTime", &retrieved.end_time); - - result = query.executeQuery(); - - //Before advancing the iterator, clear out the retrieved - //record data structure - retrieved.report_name.clear(); - retrieved.end_time = 0; - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record5.report_name); - EXPECT_EQUAL(retrieved.end_time, record5.end_time); - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record7.report_name); - EXPECT_EQUAL(retrieved.end_time, record7.end_time); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //Run a query to get all records with an end time that - //is NOT in (14000,22000) - expect only one returned - //result, record6. - query.addConstraints( - "EndTime", simdb::constraints::not_in_set, {14000,22000}); - - query.writeResultIterationsTo( - "ReportName", &retrieved.report_name, - "EndTime", &retrieved.end_time); - - result = query.executeQuery(); - - //Before advancing the iterator, clear out the retrieved - //record data structure - retrieved.report_name.clear(); - retrieved.end_time = 0; - - EXPECT_TRUE(result->getNext()); - EXPECT_EQUAL(retrieved.report_name, record6.report_name); - EXPECT_EQUAL(retrieved.end_time, record6.end_time); - - //There should not be any more in this result set - EXPECT_FALSE(result->getNext()); - - //------------------- Mini-test #4 ---------------------// - - si_values_tbl->deleteAllObjects(); - - //This mini-test is going to verify that we can recover raw - //blobs from the database as vectors of a specific data type. - //For example, store SI values as char vectors (which is what - //a blob is), but read them back as std::vector, which - //is the original data type (this simple example assumes no - //compression and just illustrates the point). - const std::vector mini_test4_raw_si1 = {4, 6, 7, 2, 4, 8}; - std::unique_ptr mini_test4_blob_ref = - si_values_tbl->createObject(); - - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &mini_test4_raw_si1[0]; - blob_descriptor.num_bytes = mini_test4_raw_si1.size()*sizeof(int16_t); - mini_test4_blob_ref->setPropertyBlob("RawBytes", blob_descriptor); - - //We wrote the int16_t vector into the database as a blob, which - //effectively stripped away the fact that it was *specifically* - //int16_t. Let's verify that even though the blobs are just raw - //char vectors, we can ask for them back in their original vector - //form (original data type, without the need to reinterpret_cast). - std::vector mini_test4_retrieved_si1; - query2.writeResultIterationsTo("RawBytes", &mini_test4_retrieved_si1); - - auto mini_test4_result_iter = query2.executeQuery(); - EXPECT_TRUE(mini_test4_result_iter->getNext()); - EXPECT_EQUAL(mini_test4_retrieved_si1, mini_test4_raw_si1); - EXPECT_FALSE(mini_test4_result_iter->getNext()); - - si_values_tbl->deleteAllObjects(); - - //Do this mini test again with a blob of floats - const std::vector mini_test4_raw_si2 = {-1, -9, 500, 334}; - mini_test4_blob_ref = si_values_tbl->createObject(); - - blob_descriptor.data_ptr = &mini_test4_raw_si2[0]; - blob_descriptor.num_bytes = mini_test4_raw_si2.size()*sizeof(float); - mini_test4_blob_ref->setPropertyBlob("RawBytes", blob_descriptor); - - std::vector mini_test4_retrieved_si2; - query2.writeResultIterationsTo("RawBytes", &mini_test4_retrieved_si2); - mini_test4_result_iter = query2.executeQuery(); - EXPECT_TRUE(mini_test4_result_iter->getNext()); - EXPECT_EQUAL(mini_test4_retrieved_si2, mini_test4_raw_si2); - EXPECT_FALSE(mini_test4_result_iter->getNext()); - - //------------------- Mini-test #5 ---------------------// - - //Let's make a small schema with some double columns, insert - //a few records, and run queries against it. The ObjectQuery - //code uses stringifiers to put together the SQL statements, - //and floating point stringifiers should be robust regardless - //of how many sigificant digits the column values have. - - simdb::Schema DoublesSchema; - DoublesSchema.addTable("Doubles") - .addColumn("Foo", dt::double_t); - - simdb::ObjectManager doubles_obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(doubles_obj_mgr, DoublesSchema); - - auto doubles_tbl = doubles_obj_mgr.getTable("Doubles"); - auto doubles1 = doubles_tbl->createObject(); - auto doubles2 = doubles_tbl->createObject(); - auto doubles3 = doubles_tbl->createObject(); - - const double foo1 = 3.0; - const double foo2 = 7.8899239572345; - const double foo3 = (0.1 + 0.1 + 0.1); - - doubles1->setPropertyDouble("Foo", foo1); - doubles2->setPropertyDouble("Foo", foo2); - doubles3->setPropertyDouble("Foo", foo3); - - simdb::ObjectQuery doubles_query(doubles_obj_mgr, "Doubles"); - - double stored_foo; - doubles_query.writeResultIterationsTo("Foo", &stored_foo); - doubles_query.addConstraints("Foo", simdb::constraints::equal, foo1); - - auto result_iter = doubles_query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(stored_foo, foo1); - EXPECT_FALSE(result_iter->getNext()); - - doubles_query.writeResultIterationsTo("Foo", &stored_foo); - doubles_query.addConstraints("Foo", simdb::constraints::equal, foo2); - - result_iter = doubles_query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(stored_foo, foo2); - EXPECT_FALSE(result_iter->getNext()); - - doubles_query.writeResultIterationsTo("Foo", &stored_foo); - doubles_query.addConstraints("Foo", simdb::constraints::equal, foo3); - - result_iter = doubles_query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(stored_foo, foo3); - EXPECT_FALSE(result_iter->getNext()); - - //------------------- Mini-test #3 ---------------------// - - //Verify the behavior of ObjectQuery::countMatches() - header_tbl->deleteAllObjects(); - - const std::string hello_world = "hello_world.csv"; - const std::string fizz_buzz = "fizz_buzz.json"; - - header_tbl->createObjectWithArgs("ReportName", hello_world, - "StartTime", 1000, - "EndTime", 5000000); - - header_tbl->createObjectWithArgs("ReportName", hello_world, - "StartTime", 2000, - "EndTime", 4500000); - - header_tbl->createObjectWithArgs("ReportName", fizz_buzz, - "StartTime", 1000, - "EndTime", 5000000); - - simdb::ObjectQuery count_query(obj_mgr, "ReportHeader"); - - //Zero-constraint queries always should find all records - //in this table - EXPECT_EQUAL(count_query.countMatches(), 3); - - //Add a constraint and ask again... - count_query.addConstraints("ReportName", - simdb::constraints::equal, - "hello_world.csv"); - EXPECT_EQUAL(count_query.countMatches(), 2); - - //Add a second constraint and ask again... - count_query.addConstraints("StartTime", - simdb::constraints::greater_equal, - 1800); - EXPECT_EQUAL(count_query.countMatches(), 1); - - //Add a third constraint which matches no records, and - //verify the query returns zero - count_query.addConstraints("EndTime", - simdb::constraints::less, - 3000000); - EXPECT_EQUAL(count_query.countMatches(), 0); -} - -void testObjectQueryOptions() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - schema.addTable("Metadata") - .addColumn("A", dt::int32_t) - .addColumn("B", dt::string_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - auto meta = obj_mgr.getTable("Metadata"); - - auto row1 = meta->createObject(); - row1->setPropertyInt32("A", 5); - row1->setPropertyString("B", "foo"); - - auto row2 = meta->createObject(); - row2->setPropertyInt32("A", 8); - row2->setPropertyString("B", "abc"); - - auto row3 = meta->createObject(); - row3->setPropertyInt32("A", 3); - row3->setPropertyString("B", "bar"); - - int a; - std::string b; - - simdb::ObjectQuery query(obj_mgr, "Metadata"); - query.writeResultIterationsTo("A", &a, "B", &b); - query.orderBy(simdb::OrderBy("A", simdb::ASC)); - - auto result_iter = query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 3); - EXPECT_EQUAL(b, "bar"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 5); - EXPECT_EQUAL(b, "foo"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 8); - EXPECT_EQUAL(b, "abc"); - EXPECT_FALSE(result_iter->getNext()); - - query.writeResultIterationsTo("A", &a, "B", &b); - query.orderBy(simdb::OrderBy("B", simdb::DESC)); - - result_iter = query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 5); - EXPECT_EQUAL(b, "foo"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 3); - EXPECT_EQUAL(b, "bar"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 8); - EXPECT_EQUAL(b, "abc"); - EXPECT_FALSE(result_iter->getNext()); - - query.writeResultIterationsTo("A", &a, "B", &b); - query.orderBy(simdb::OrderBy("A", simdb::DESC)); - query.setLimit(1); - - result_iter = query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 8); - EXPECT_EQUAL(b, "abc"); - EXPECT_FALSE(result_iter->getNext()); - - query.writeResultIterationsTo("A", &a, "B", &b); - EXPECT_NOTHROW(query.setLimit(0)); - - result_iter = query.executeQuery(); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 5); - EXPECT_EQUAL(b, "foo"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 8); - EXPECT_EQUAL(b, "abc"); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_EQUAL(a, 3); - EXPECT_EQUAL(b, "bar"); - EXPECT_FALSE(result_iter->getNext()); -} - -void testObjectCreationArgs() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - const int32_t default_a = 88; - const uint64_t default_b = 10000; - const double default_c = 100.55; - const std::string default_d = "someDefaultString"; - const char * default_e = "someDefaultLiteral"; - - schema.addTable("DTypes") - .addColumn("MyInt32", dt::int32_t) - ->setDefaultValue(default_a) - .addColumn("MyUInt64", dt::uint64_t) - ->setDefaultValue(default_b) - .addColumn("MyDouble", dt::double_t) - ->setDefaultValue(default_c) - .addColumn("MyString", dt::string_t) - ->setDefaultValue(default_d) - .addColumn("MyLiteral", dt::string_t) - ->setDefaultValue(default_e) - .addColumn("MyBlob", dt::blob_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - const int32_t a = 95; - const uint64_t b = 4000; - const double c = 5.678; - const std::string d = "foo"; - const char * e = "helloWorld"; - const std::vector f = {1.3, 1.4, 5.6, 8.8}; - - auto dtype_table = obj_mgr.getTable("DTypes"); - - auto record1 = dtype_table->createObjectWithArgs("MyInt32", a); - EXPECT_EQUAL(record1->getPropertyInt32("MyInt32"), a); - - auto record2 = dtype_table->createObjectWithArgs("MyUInt64", b); - EXPECT_EQUAL(record2->getPropertyUInt64("MyUInt64"), b); - - auto record3 = dtype_table->createObjectWithArgs("MyDouble", c); - EXPECT_EQUAL(record3->getPropertyDouble("MyDouble"), c); - - auto record4 = dtype_table->createObjectWithArgs("MyString", d); - EXPECT_EQUAL(record4->getPropertyString("MyString"), d); - - auto record5 = dtype_table->createObjectWithArgs("MyLiteral", e); - EXPECT_EQUAL(record5->getPropertyString("MyLiteral"), e); - - auto record6 = dtype_table->createObjectWithArgs("MyBlob", f); - std::vector my_blob; - record6->getPropertyBlob("MyBlob", my_blob); - EXPECT_EQUAL(my_blob, f); - - const int32_t a2 = 50; - const uint64_t b2 = 99999; - const double c2 = 5.848; - const std::string d2 = "mightyDucks"; - const char * e2 = "helloAgain"; - const std::vector f2 = {4.5, 5.6, 6.7, 7.8}; - - auto validate_multi_arg = [&](std::unique_ptr & record) { - EXPECT_EQUAL(record->getPropertyInt32("MyInt32"), a2); - EXPECT_EQUAL(record->getPropertyUInt64("MyUInt64"), b2); - EXPECT_EQUAL(record->getPropertyDouble("MyDouble"), c2); - EXPECT_EQUAL(record->getPropertyString("MyString"), d2); - EXPECT_EQUAL(record->getPropertyString("MyLiteral"), e2); - - std::vector my_blob2; - record->getPropertyBlob("MyBlob", my_blob2); - EXPECT_EQUAL(my_blob2, f2); - }; - - //Ensure the variadic function works correctly. Mix up the - //input arguments so those that have special enable_if handling - //appear at the beginning and the end of the parameter pack. - - //Blobs at the end - auto record7 = dtype_table->createObjectWithArgs( - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2, - "MyString", d2, - "MyLiteral", e2, - "MyBlob", f2); - - validate_multi_arg(record7); - - //String literals at the end - auto record8 = dtype_table->createObjectWithArgs( - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2, - "MyString", d2, - "MyBlob", f2, - "MyLiteral", e2); - - validate_multi_arg(record8); - - //Standard strings at the end - auto record9 = dtype_table->createObjectWithArgs( - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2, - "MyBlob", f2, - "MyLiteral", e2, - "MyString", d2); - - validate_multi_arg(record9); - - //Blobs at the beginning - auto record10 = dtype_table->createObjectWithArgs( - "MyBlob", f2, - "MyLiteral", e2, - "MyString", d2, - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2); - - validate_multi_arg(record10); - - //String literals at the beginning - auto record11 = dtype_table->createObjectWithArgs( - "MyLiteral", e2, - "MyBlob", f2, - "MyString", d2, - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2); - - validate_multi_arg(record11); - - //Standard strings at the beginning - auto record12 = dtype_table->createObjectWithArgs( - "MyString", d2, - "MyLiteral", e2, - "MyBlob", f2, - "MyInt32", a2, - "MyUInt64", b2, - "MyDouble", c2); - - validate_multi_arg(record12); - - //Create a record with an empty blob column value. This should - //result in a record with all default values filled in. - std::vector empty_blob; - auto record13 = dtype_table->createObjectWithArgs("MyBlob", empty_blob); - - EXPECT_EQUAL(record13->getPropertyInt32("MyInt32"), default_a); - EXPECT_EQUAL(record13->getPropertyUInt64("MyUInt64"), default_b); - EXPECT_EQUAL(record13->getPropertyDouble("MyDouble"), default_c); - EXPECT_EQUAL(record13->getPropertyString("MyString"), default_d); - EXPECT_EQUAL(record13->getPropertyString("MyLiteral"), default_e); - - my_blob.clear(); - record13->getPropertyBlob("MyBlob", my_blob); - EXPECT_TRUE(my_blob.empty()); -} - -void testObjectDeletionArgs() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - schema.addTable("DTypes") - .addColumn("MyInt32", dt::int32_t) - .addColumn("MyUInt64", dt::uint64_t) - .addColumn("MyDouble", dt::double_t) - .addColumn("MyString", dt::string_t) - .addColumn("MyLiteral", dt::string_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - //Fill up a table with a bunch of records. We will pick off - //several of these records to delete at a time and verify - //the deletion happened correctly. - auto dtype_table = obj_mgr.getTable("DTypes"); - - const std::vector foo_strings = { - "fooA", "fooB", "fooC", "fooD", "fooE", - "fooF", "fooG", "fooH", "fooI", "fooJ" - }; - - // - // MyInt32 MyUInt64 MyDouble MyString MyLiteral - // --------- ---------- ---------- ---------- ----------- - // 10 5000 3.5 fooA barA - // 12 5100 4.5 fooB barB - // 14 5200 5.5 fooC barC - // 16 5300 6.5 fooD barD - // 18 5400 7.5 fooE barE - // 20 5500 8.5 fooF barF - // 22 5600 9.5 fooG barG - // 24 5700 10.5 fooH barH - // 26 5800 11.5 fooI barI - // 28 5900 12.5 fooJ barJ - // - std::set remaining_record_ids; - std::unique_ptr record; - - record = dtype_table->createObjectWithArgs("MyInt32", 10, - "MyUInt64", 5000, - "MyDouble", 3.5, - "MyString", foo_strings[0], - "MyLiteral", "barA"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 12, - "MyUInt64", 5100, - "MyDouble", 4.5, - "MyString", foo_strings[1], - "MyLiteral", "barB"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 14, - "MyUInt64", 5200, - "MyDouble", 5.5, - "MyString", foo_strings[2], - "MyLiteral", "barC"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 16, - "MyUInt64", 5300, - "MyDouble", 6.5, - "MyString", foo_strings[3], - "MyLiteral", "barD"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 18, - "MyUInt64", 5400, - "MyDouble", 7.5, - "MyString", foo_strings[4], - "MyLiteral", "barE"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 20, - "MyUInt64", 5500, - "MyDouble", 8.5, - "MyString", foo_strings[5], - "MyLiteral", "barF"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 22, - "MyUInt64", 5600, - "MyDouble", 9.5, - "MyString", foo_strings[6], - "MyLiteral", "barG"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 24, - "MyUInt64", 5700, - "MyDouble", 10.5, - "MyString", foo_strings[7], - "MyLiteral", "barH"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 26, - "MyUInt64", 5800, - "MyDouble", 11.5, - "MyString", foo_strings[8], - "MyLiteral", "barI"); - remaining_record_ids.insert(record->getId()); - - record = dtype_table->createObjectWithArgs("MyInt32", 28, - "MyUInt64", 5900, - "MyDouble", 12.5, - "MyString", foo_strings[9], - "MyLiteral", "barJ"); - remaining_record_ids.insert(record->getId()); - - std::vector> remaining_records; - obj_mgr.findObjects("DTypes", {}, remaining_records); - EXPECT_EQUAL(remaining_records.size(), remaining_record_ids.size()); - - for (const auto & row : remaining_records) { - EXPECT_FALSE(remaining_record_ids.find(row->getId()) == - remaining_record_ids.end()); - } - - //Let's start by removing records one at a time with a - //single match constraint. - auto verify_deletion = [&](const std::set & remaining_ids) { - std::vector> remaining_objs; - obj_mgr.findObjects("DTypes", {}, remaining_objs); - EXPECT_EQUAL(remaining_objs.size(), remaining_ids.size()); - - for (const auto & obj : remaining_objs) { - EXPECT_FALSE(remaining_ids.find(obj->getId()) == remaining_ids.end()); - } - }; - - dtype_table->deleteObjectsWhere("MyInt32", simdb::constraints::equal, 10); - remaining_record_ids.erase(1); - verify_deletion(remaining_record_ids); - - dtype_table->deleteObjectsWhere("MyUInt64", simdb::constraints::equal, 5100); - remaining_record_ids.erase(2); - verify_deletion(remaining_record_ids); - - dtype_table->deleteObjectsWhere("MyDouble", simdb::constraints::equal, 5.5); - remaining_record_ids.erase(3); - verify_deletion(remaining_record_ids); - - dtype_table->deleteObjectsWhere("MyString", simdb::constraints::equal, foo_strings[3]); - remaining_record_ids.erase(4); - verify_deletion(remaining_record_ids); - - dtype_table->deleteObjectsWhere("MyLiteral", simdb::constraints::equal, "barE"); - remaining_record_ids.erase(5); - verify_deletion(remaining_record_ids); - - //Now let's remove a record with a multi-argument match constraint. - dtype_table->deleteObjectsWhere("MyInt32", simdb::constraints::equal, 20, - "MyString", simdb::constraints::equal, foo_strings[5], - "MyLiteral", simdb::constraints::equal, "barF"); - - remaining_record_ids.erase(6); - verify_deletion(remaining_record_ids); - - //Let's remove two more records using the "is in" constraint. - //We will do this for an integer column first. - dtype_table->deleteObjectsWhere("MyInt32", simdb::constraints::in_set, {22,24}); - - remaining_record_ids.erase(7); - remaining_record_ids.erase(8); - verify_deletion(remaining_record_ids); - - //Remove two more records using the "is in" constraint, this - //time against a string column. - dtype_table->deleteObjectsWhere("MyString", - simdb::constraints::in_set, - {"fooI","fooJ"}); - - remaining_record_ids.clear(); - verify_deletion(remaining_record_ids); -} - -void testObjectUpdateArgs() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - schema.addTable("DTypes") - .addColumn("MyInt32", dt::int32_t) - .addColumn("MyUInt64", dt::uint64_t) - .addColumn("MyDouble", dt::double_t) - .addColumn("MyString", dt::string_t) - .addColumn("MyBlob", dt::blob_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - auto dtype_table = obj_mgr.getTable("DTypes"); - - //Create objects with an initial set of column values. - //We will overwrite those values in batches shortly. - dtype_table->createObjectWithArgs("MyInt32", 10, - "MyUInt64", 3000, - "MyDouble", 345.75, - "MyString", "hello"); - - dtype_table->createObjectWithArgs("MyInt32", 12, - "MyUInt64", 3100, - "MyDouble", 545.50, - "MyString", "helloAgain"); - - dtype_table->createObjectWithArgs("MyInt32", 14, - "MyUInt64", 3200, - "MyDouble", 745.25, - "MyString", "goodbye"); - - //Overwrite the MyDouble columns to 123.45 and the MyString - //columns to "justOverwritten" for the records whose MyDouble - //value is less than 700 and whose MyUInt64 value is greater - //than or equal to 3050. - auto num_updated_rows = dtype_table-> - updateRowValues("MyDouble", 123.45, "MyString", "justOverwritten"). - forRecordsWhere("MyDouble", simdb::constraints::less, 700, - "MyUInt64", simdb::constraints::greater_equal, 3050); - - //Verify the update. - EXPECT_EQUAL(num_updated_rows, 1); - - std::vector> updated_records; - obj_mgr.findObjects("DTypes", {}, updated_records); - EXPECT_EQUAL(updated_records.size(), 3); - - simdb::ObjectRef * remaining_record1 = updated_records[0].get(); - EXPECT_EQUAL(remaining_record1->getPropertyInt32("MyInt32"), 10); - EXPECT_EQUAL(remaining_record1->getPropertyUInt64("MyUInt64"), 3000); - EXPECT_EQUAL(remaining_record1->getPropertyDouble("MyDouble"), 345.75); - EXPECT_EQUAL(remaining_record1->getPropertyString("MyString"), "hello"); - - simdb::ObjectRef * remaining_record2 = updated_records[1].get(); - EXPECT_EQUAL(remaining_record2->getPropertyInt32("MyInt32"), 12); - EXPECT_EQUAL(remaining_record2->getPropertyUInt64("MyUInt64"), 3100); - EXPECT_EQUAL(remaining_record2->getPropertyDouble("MyDouble"), 123.45); - EXPECT_EQUAL(remaining_record2->getPropertyString("MyString"), "justOverwritten"); - - simdb::ObjectRef * remaining_record3 = updated_records[2].get(); - EXPECT_EQUAL(remaining_record3->getPropertyInt32("MyInt32"), 14); - EXPECT_EQUAL(remaining_record3->getPropertyUInt64("MyUInt64"), 3200); - EXPECT_EQUAL(remaining_record3->getPropertyDouble("MyDouble"), 745.25); - EXPECT_EQUAL(remaining_record3->getPropertyString("MyString"), "goodbye"); - - //Overwrite the MyDouble columns to 777.777 for the records - //whose MyString value is 'hello' or 'goodbye'. - num_updated_rows = dtype_table-> - updateRowValues("MyDouble", 777.777). - forRecordsWhere("MyString", simdb::constraints::in_set, {"hello", "goodbye"}); - - //Verify the update. - EXPECT_EQUAL(num_updated_rows, 2); - - obj_mgr.findObjects("DTypes", {}, updated_records); - EXPECT_EQUAL(updated_records.size(), 3); - - remaining_record1 = updated_records[0].get(); - EXPECT_EQUAL(remaining_record1->getPropertyInt32("MyInt32"), 10); - EXPECT_EQUAL(remaining_record1->getPropertyUInt64("MyUInt64"), 3000); - EXPECT_EQUAL(remaining_record1->getPropertyDouble("MyDouble"), 777.777); - EXPECT_EQUAL(remaining_record1->getPropertyString("MyString"), "hello"); - - remaining_record2 = updated_records[1].get(); - EXPECT_EQUAL(remaining_record2->getPropertyInt32("MyInt32"), 12); - EXPECT_EQUAL(remaining_record2->getPropertyUInt64("MyUInt64"), 3100); - EXPECT_EQUAL(remaining_record2->getPropertyDouble("MyDouble"), 123.45); - EXPECT_EQUAL(remaining_record2->getPropertyString("MyString"), "justOverwritten"); - - remaining_record3 = updated_records[2].get(); - EXPECT_EQUAL(remaining_record3->getPropertyInt32("MyInt32"), 14); - EXPECT_EQUAL(remaining_record3->getPropertyUInt64("MyUInt64"), 3200); - EXPECT_EQUAL(remaining_record3->getPropertyDouble("MyDouble"), 777.777); - EXPECT_EQUAL(remaining_record3->getPropertyString("MyString"), "goodbye"); - - //Overwrite the MyString columns to "allThreeRecords" for the records - //whose MyDouble value is 123.45 or 777.777 - num_updated_rows = dtype_table-> - updateRowValues("MyString", "allThreeRecords"). - forRecordsWhere("MyDouble", simdb::constraints::in_set, {123.45, 777.777}); - - //Verify the update. - EXPECT_EQUAL(num_updated_rows, 3); - - obj_mgr.findObjects("DTypes", {}, updated_records); - EXPECT_EQUAL(updated_records.size(), 3); - - remaining_record1 = updated_records[0].get(); - EXPECT_EQUAL(remaining_record1->getPropertyInt32("MyInt32"), 10); - EXPECT_EQUAL(remaining_record1->getPropertyUInt64("MyUInt64"), 3000); - EXPECT_EQUAL(remaining_record1->getPropertyDouble("MyDouble"), 777.777); - EXPECT_EQUAL(remaining_record1->getPropertyString("MyString"), "allThreeRecords"); - - remaining_record2 = updated_records[1].get(); - EXPECT_EQUAL(remaining_record2->getPropertyInt32("MyInt32"), 12); - EXPECT_EQUAL(remaining_record2->getPropertyUInt64("MyUInt64"), 3100); - EXPECT_EQUAL(remaining_record2->getPropertyDouble("MyDouble"), 123.45); - EXPECT_EQUAL(remaining_record2->getPropertyString("MyString"), "allThreeRecords"); - - remaining_record3 = updated_records[2].get(); - EXPECT_EQUAL(remaining_record3->getPropertyInt32("MyInt32"), 14); - EXPECT_EQUAL(remaining_record3->getPropertyUInt64("MyUInt64"), 3200); - EXPECT_EQUAL(remaining_record3->getPropertyDouble("MyDouble"), 777.777); - EXPECT_EQUAL(remaining_record3->getPropertyString("MyString"), "allThreeRecords"); - - //Overwrite the MyInt32 columns to 10, the MyUInt64 columns to 1000, - //the MyDouble columns to 99.123, and the MyString columns to "totalReset" - //for every record in this table. - num_updated_rows = dtype_table-> - updateRowValues("MyInt32", 10, - "MyUInt64", 1000, - "MyDouble", 99.123, - "MyString", "totalReset"). - forAllRecords(); - - //Verify the update. - EXPECT_EQUAL(num_updated_rows, 3); - - obj_mgr.findObjects("DTypes", {}, updated_records); - EXPECT_EQUAL(updated_records.size(), 3); - - for (size_t idx = 0; idx < num_updated_rows; ++idx) { - auto & row = updated_records[idx]; - EXPECT_EQUAL(row->getPropertyInt32("MyInt32"), 10); - EXPECT_EQUAL(row->getPropertyUInt64("MyUInt64"), 1000); - EXPECT_EQUAL(row->getPropertyDouble("MyDouble"), 99.123); - EXPECT_EQUAL(row->getPropertyString("MyString"), "totalReset"); - } - - //Test updates of blob columns - const std::vector orig_blob = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &orig_blob[0]; - blob_descriptor.num_bytes = orig_blob.size() * sizeof(double); - for (size_t idx = 0; idx < num_updated_rows; ++idx) { - auto & row = updated_records[idx]; - row->setPropertyBlob("MyBlob", blob_descriptor); - } - - updated_records[0]->setPropertyString("MyString", "hello"); - updated_records[1]->setPropertyString("MyString", "hello"); - updated_records[2]->setPropertyString("MyString", "world"); - - const std::vector new_blob = {500, 600, 700, 800}; - dtype_table->updateRowValues("MyBlob", new_blob). - forRecordsWhere("MyString", simdb::constraints::equal, "hello"); - - //First record's blob should have been overwritten... - std::vector test_blob; - updated_records[0]->getPropertyBlob("MyBlob", test_blob); - EXPECT_EQUAL(test_blob, new_blob); - - //Second record's blob should also have been overwritten... - test_blob.clear(); - updated_records[1]->getPropertyBlob("MyBlob", test_blob); - EXPECT_EQUAL(test_blob, new_blob); - - //But the third record, whose MyString did *not* equal "hello", - //should still have the same blob values as before. - test_blob.clear(); - updated_records[2]->getPropertyBlob("MyBlob", test_blob); - EXPECT_EQUAL(test_blob, orig_blob); -} - -void testTableRefErrors() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - simdb::Schema schema; - - schema.addTable("DTypes") - .addColumn("MyInt32", dt::int32_t) - .addColumn("MyBlob", dt::blob_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - auto dtype_table = obj_mgr.getTable("DTypes"); - dtype_table->createObjectWithArgs("MyInt32", 100); - - const std::vector my_blob = {4, 5, 6, 7}; - const auto & updater = dtype_table->updateRowValues("MyInt32", 200, "MyBlob", my_blob); - EXPECT_THROW(dtype_table->createObjectWithArgs("MyInt32", 500)); - (void) updater; -} - -void testTableRefObjectReturn() -{ - PRINT_ENTER_TEST - - using dt = simdb::ColumnDataType; - - simdb::Schema schema; - schema.addTable("Dummy") - .addColumn("x", dt::double_t) - .addColumn("y", dt::string_t); - - simdb::ObjectManager obj_mgr(DB_DIR); - CREATE_SQL_SCHEMA(obj_mgr, schema); - - auto table = obj_mgr.getTable("Dummy"); - auto record1 = table->createObject(); - EXPECT_NOTEQUAL(record1.get(), nullptr); - - table->neverReturnObjectRefsOnCreate(); - auto record2 = table->createObject(); - EXPECT_EQUAL(record2.get(), nullptr); - - table->alwaysReturnObjectRefsOnCreate(); - auto record3 = table->createObject(); - EXPECT_NOTEQUAL(record3.get(), nullptr); - - simdb::ObjectQuery query(obj_mgr, "Dummy"); - EXPECT_EQUAL(query.countMatches(), 3); -} - -int main() -{ - srand(time(0)); - - testBadSql(); - testBadFile(); - testInvalidSchema(); - testSqlSchema(); - testSqlSchemaColumnModifiers(); - testBasicDataTypes(); - test64BitInts(); - testObjectQuery(); - testObjectQueryOptions(); - testObjectCreationArgs(); - testObjectDeletionArgs(); - testObjectUpdateArgs(); - testTableRefErrors(); - testTableRefObjectReturn(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/simdb/test/SQLiteDatabase/sample.db b/sparta/simdb/test/SQLiteDatabase/sample.db deleted file mode 100644 index 442eae8a973447517eec78c502c1d082085d7007..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 278528 zcmeEv349dg`Tsk6&tAisfLzNFLI{xT>4#adh2YPD7^|7V_g_n6(yX7&jE{ryZn@P2n@-sgSZ z_xrr_&U?)>O{-UQ1%0l~{=Ob>&{aZ45RM~NE*Bxh4*!?H|Ipuf_+bbg0CG{k2m5I! z^G1Aalp+gXOK~O$UpvOwN870NV{1R(V0p_@W4_sRrb#e%7+y1s(4VdQQn#0Tft$`v z7%a)dcHWe0LR~a3$DCVR$kC3j-ge)P=B^%Jz}MI13)FP>_iky89M><2{Kjj!HEU|C zn`>S5E0@$Bs8>@O&eLLj2xpryo8rP~d^$pc)*1MM1u6I?p zG&k0-1j&ZlmCbXar0SPQ@#nZ$j^zW)cGP*ya`;mCsGx6*Id|$*juypaJCftEgJf1^ zKG*oR`ul>*eBO3npPbVII%2eJ(?w|0ruvoscHgotU!S+Ht#fDVDsP{!Hz=K}UlMa| zjKZcd7Zrk%O=GURWzE{G$}5QRPv~}79)s6Dk8XM zg>*>yo1@fIK=TB1Zb1PzV3cW9h$?e2LP@G3rb_ZNUa_ueZb37pU)DB4yhf@C|m}uE^b?%3KVH!=w?{>+$vm zyV@j^h7*Q1cx_jpYg1QuS8%7|io2No&2iOqdjnYA!(wG17fp;pa<01|V1WuVoui|p zTA8Nc&TgNAYB>cw&nrw5bwMEzX<|?)#yE70*_=BawEE@S2s;HBF)BTa!aPwIRN4pA zQ?wvZs!019PfGz6Oy=C>Zs`6*ks(#hP#segQO;5HWpe^*qOU8&BMe1Hi6!b3qdC{@ z=Jw`C5fvRjJ`<({v_(s(WUsnE*y-;}SV!x7H~W)RVb)T&ZBbM;sD{{5$DEd0>PQ*n z9$~odLiMDh4d&biXmCqnOp>^R7)dI8-8hI*!V%&+iU|^N&5G{-6wpRJJ_ZJAW6Tmx zG(j4MK@1a?QE8dzoJbVUaYa?~`oIdGcXLy{f_sVN6~4{E9)BPh)jf;md6mv2jaSxV zc>>#@jq$YxeB1hcy=}gjL%pILF&Ag`N3nZ-YV9Y*$!Yf$U6JDw9 z_HGUM+7n)jelp4#g~9E3y8}W_c$~mL?N4JsV?bj-V?bj-V?bj-V?bj-V?bj-V?bj- zW8nWD12vqL~|gA-|G^KU7~wIS^0u8H~eLiA&TVzhj1J* z3pGUeKzKuVM7T~kSEvz!|Mv>3dV?bj-V?bj-V?bj-V?bj-V?bj-V?blzf6PFx zNngT6_Q{17+5I@XJn39?@n-jnqbsV)kU_1aj%Ut=u32w1j3s(xX&l+ z_0?PxY;5iJwQPlrt-jDELfmT<&{=i*2|U~4FoDxA;s@JTC+*EQ3Li)m9wmG&{7d*$ z_%qA}Gz*o&RDpAT?tIVrqVs;|14ziPkUzTLjWKFRj6 z?OEG}wk}(_O|ZUYeb9QUb)$8j)x!Uge}o_4H}Z4%QI zYjuU(=iJY^i@6T2m@|@>$#YW1Eyqb4^*Qd)p|TAo!gcW6>h`X{R&TJavlVue^!r-- zgRq0`B;0t}+UD=?4VJXR?ut!bd1Kp}f@49fQH|K@vh^x8h|wBW6&#~dgV)&%z7+~3_Dt$A5NBZz_ZhZb8pe+`pIPwc33H-IQml3Ytz6hvxgtpJgRp+tJTJC@8|4l!2` z6*CAsAG>Qxj)u5xL&a@WSZ8s5JqRryTBy2W8AR+EDk9#4ZbkK~nzIxnTU1I`dg>td z7FDt5M_c=-!deh)QYW~`vqZI)ifAoI7S^cNQnAppShW^Uw3Y>h)vC34=6kACYblS` zQc+l`T1&adbChZ=Wzky73m2)@QdZ_UQni-SXf36M3sq|=E%7W+twr%X5ew(5)>7j3 zRH)XXc%~E=dQ@u>i#+A3wJ4q|^9svUYjMx@l&aRExcTQ4mZ;WJTv#GP4JicAk-TYz5V?Br$nXT2F=IBwsU18HK0Bt*Er3|-29KqD zYD}9t3!+yhi#}D%hbZ2`qa=7zxMt1-nff7RrnvJU9z1dfBMH-1JRK(I%>WVTg(Mf5 zXurK8>BRivzF)ko!xQ;~AXNH@GTJY=m?wB#f5a#W-& z3huEgQdYFy&b%=yQg%DsqgAA=XtC{iqg13^Y@0h0q9tiDl&!RN>IjIQq{V2dL``X}z8Gf`Au1x1S2j1AezShltzh$HYPEN6K_V9qgVza}k(H2ezr!13ga4?_e z^nsm$ps%O34@QUZ#viU+JtekNiZV}X-- zaI&YXtuJ)eFvc7{+uhy8;_3wxoSh)4m-XpJ88Jm(zqP9^Bx^Uoxm@0&hxfC#1V-rL zXo|(63$%6m+WWhGeO8@`@MA3&>HX}voQYUASS*3Ao~`ggRUqhvv2pp{3JBc;8zG!M z;Dj)bN@r$lkR~ZO;d2PdBH?r4AHw6jPxy=Qq42KoJKdMQ9Z2VIRRF!6UeZLLpC> zB8(SC3l6~ya|&NO|Kh^sea=?rdS{cf!MW5~?OfA@0jYC;27%=9K1tk|Jwe!{jc^v+26JQ7G^qrVSm>Cg#97=J@)x{3g$)LxBbfYg6#?0gSNYD zx7e<=U1~emcAD*EThP{RYquR|YqqVhEwLSGE49tB&9F_hjj`En2J1K0&#j+W{|Iv_ zuUlWVK4*Ob=2Y&o-eSGhda3nX>uJ`LtwC$IwcUE0wb{DDy2N^S2AI>u_Z z8u)Mc&-qVa_T?@9b^b;EIsOU$LH;iO7XDiPQvO{2H2!2h$allMObfr1ue5y0d-x(g zpP$0#@&a$R9I||BIcRy;{D$cjgWs^uP-)0F2>LHAZ&+TjJa2i*@`&YtWxwSH%N3Rj zU`A(;WxK_1>9Dj~)>#@YOD&ZakEO_xZ<%7rwFnlo`H=Zb^QY#6=6B7nn4dR4Wq!nb zz`WmlgZT>c1?GL`J?8CZzq!NQYF=kZ(?PnjMu9Wd=T-C(-Hbb)D~X^&~U$#3c~wVKwM8cj=0l_rm=$dqrI0$_ZYVu{l*SsE6hAK8kZU?jUHo> zG2b}Fm}?Y_X2T)FmxfOb2MzBU-Y~pkc;4`o;Ss|D!+ygJhARvg81@1KKEv}Ep3CqYhG#Q8i{Y6J_c1(!;pq%dW4M>$07J^~REB#Pp2Bc9 z!;=~AVt5k6oeXy{+|KYshW!kK3Ghp#&8ouFT>**wle%4!;K7&W4M9gu?*KUJci*qhHDwNFl=Vn#BdG6)eKiLY-G5S zVFSYzh-Pj%;%x3{hV=}WF9j$tjsB@AmAE@oKGu!>9^{DR@X7=F(1Glu_U_z#AEXZR_@zcKtP!%rCgh2h5x|IF|sh6fq` ziQ$I~|H$wIhVL_ckKrE}zRU0(hHo=`i{YCLf6wrD41de;4TisA_-lr*GyE09*BHLa z@Rtl_`3~!Q8--w6TksBCZFQ2|ng4N_&8LpAxY8kGQ z;Yt~87`6GVi_)y;X)ZMkl}n8&XeI>8P1X6Y#Gjy;Y=C!$#8}Yr^|4f z40~l5kb%l@s!`8ztM_1d3WmEeJQ>4X7@maTP7HToxE;e2G3>`Mh+zQ3J`A^ExD`V` zhP@c}VAze}77S0ounWUZ3_CE~jG+(1b`0Av+=QVQ!{ae*#qfI=Zp8373^!nSEQaeb zJO;yc7_P;z1;b_xn=o7>g^txySiB0uMhsVC*nr^*43}egG=}vUF2isshIJU$Vz>mu z8Vna>SdC#7hLso|h2bI$kHl~xh6^y9k6{Id9t_JdEW@xA!x9Wd4Bb-bD3(H@2*Y_8 z&c$#JhJ_f;#;^dxSs3PHI1|G>3};|C9m8oDPQ}oL;S>xfV>k)Ji5O16@CXdYV>k}O zTnuwC9E;%?3`b))3d4~Yj*vnJ>=1y^DTQ{26zc3!sJCHg#gNC)f}t5h6NW|%4G_Zk z13N0@F7lD|{{J-l?}+d*jP{=pZWYcKwhPC@=-mw?b`D1Aufo{;CK!)zgE4qDjJrp| zX!|`FVc!R%Y6>Ii)i8RV>fr7FfJC%EjRB1TjRB1TjRB1TjRB1TjRB1Tje-9K13A;o z@OHFCuZz4DO|I_bKFgV!@cch>TnW$LmNO;c`CD=(Cp>?1&ZLCrZ_JsP@ceZ-6B3@k zI_HRl=dZ{apYZ(hoN)=y&(6tBc%IM6NqF9zGdAIQQ_h$~=gGG@qZ6M0CTCQ_^IzwT zOnCmQoDm7n|2s!Wc>b;&XTtM$xq&2schbxNaiC)e{h|I09k$c!b&G5P0wv0BM?Y5KLnwAm5Hb_+~W1 z*P{@=8j0}l5eRn)2zNLUZg(KuW=FW$hH#@5;W}LYUyaNE#kl-0;PSr%m;de3@;|57 zOQBAu*Ymuz{tqAh)BZFDGzK&VGzK&VGzK&VGzK&VGzK&VGzK&VGzPxI3`pz$+W7xF zY+$Wu8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w8Uw(9pps zIy!PK29686X`Cl}b({wt41T=ymzq+WNP8d)s!pI=ceFK7U7_w+Da9zaS6k6jGc_eYrM?C+g+W%cQ~w zYb3+*pGPQ*C2Ce0IW8+oO$c2K7wE-F)RmKoDas-x8y-`Xi4&=BY-XlVIIWFjINlVc z*)TdQU1KsaMQNmD!()n4@d)Z0otY^VUVbAPjyFX~HcU~HrYXi#*QiWPQ4%TH z@R*`R97la4Gc$$4EpjBo@um>7VG1!#Q{+Ju_Eg~E??B*XEh zaA(65?les?mb#pon8F`rb-gaN#=5GJ?6gx^Sb!oAENF2G- z%9Eb_9_R?B*kx|13ujiUE~)j=TIvpoBbs@h)D+ezd+TyHJ<9WX?GCTxf!tLieQ(Rx z7ts?J6FcBe>atkLdOCK!ly2CyIqUu>cFqnTIP34muX?%U2rlNm-38rz=1|?`4Y+!I z-avn!uifSEbp<&j+UOBxD`}x4TM}h4k$?E83`&=?kiA4& zFmu*c6Ii8WHJ^aJDChJmBYx^#zHaZ+< z5^{x+f=w_w4>`Ybe&+nE^B{ah@J;8hov%25?tIqyr1KHyea<_bw>ocdUgf;Rd7g8h zlR9@f`<;Ggm$S{e(YemK+PU0W>#T%t4wg9QI`f@VofDj6or05h>K)%YzI6P<@fXL3 z@HN8UIez7M$??478OP&}ha3kSw>xfjT<5sLagpO3_)g&-$4*DU(c|cFcpV!YEsjP< zy`#pl$l-AmI|>}rz}x>Ahr?mAAF_XG|I~gEeEq*+f5ra1{VDq+_5|-7ect+%^%3g<>wfDE)+?+RSoc}?ShrjK)(&f{b)B`*y3|@}^;nCn`PM1cT&rL; z^N08^`A_+S{JZ=c{44zP{8RAV#RL3){s#UE{sMj`}q#Om0!me<*|7W{oB_2P*U4i&8^XLB%cNrc($X$x~0Cx%E{oKWf_i+~?-pgHx zc!0YA@gDAc#Jjok5bxs7MZA+c2k{Q>Y{c8Svk-s4or$=g+lP27cLw73xziDE;!Z=n zf!m9CJvV@OEk_Zr;Z8-oira&DC3gzq<=k$>%ea#fFXeV2Uc#M(crmvV@gi;q;)UFH z#0$6+5zpuP5zpg-i05(v#B;bl#Iw0=h-YzI5zplOi2Jx+#51@a#M8NM#M8Jfhz zgebK?<1b7RSW@pz2v&z~DjzQN;BvPXYzr0mh38$rIp^MdTspL5DC{W%Bu0?*sY zzYuNYb3`lo4ADaViD)GMKs1oQBkIYgh&u8&M2`Fwk&sU$B!|df5dTX)M*I)?Gvb%z zBg8MrLBxNNKOufjK1BSC{1Ne=7nlZ{%IXzmj(lKOt`;{)N1S z_%V4C@z3PfcOl#AMt5&AL3KwUc?`f1Bg$Odk~)>cOyQ|eER>8`SgE` z+=1sGCAT9!!kqd)O!njPL*zEZ2bov@2gogWd_TDv@jm9(|6Xzv9v>h#BHqLN`rl2i z$KyN6b%;M8*COsG*C5_Xu136tT!r|3awXzTUi--X`)PwJCivfhc_|97DG|Ly4g|3viu-;dt^gXsM~fZqT6(EI2~)djI#K_y2bE{@;e) z|2Luce=mCfKOVjRx1#s|@1ghqjp+UVIQ0I%0lohpyGGKr9=-n`gWmtwq4)o_=>5M1 zz5h3(_x~pJ{=Wvj|F5c-q#Bn=;mV~_*ia{hD{7^1`4TBS8omG5qxb)1=>2~wdjGFO z@Bg*v{eKC1|F1#s|BKQ4e>HmluR`ztmFWHdDD?in2)+LwiQfMgqWAv==>31bDCw#| z@Bbe3{$GyX|I5((e<^zZFG27BB6|OKqxb)!0!eD#EGe9uFNJevN?~E16waO@g$3yS ze-?WG&qwe7Gtv8h9(w2~xdjEH!_x~yA{eLog|DS~3|0kmN{|V^* z{|NN{KOVjRj~gZF%0=)0Iq3a=EPDSRgWmr~qxb(&=>2~rdjB7R-v6EW`o9fd|F`1n z|2)3_Z^769&G`Di3C1B+cmu$LLVEw76TT+!Py5pt&=}Ad&=}Ad&=}Ad&=}Ad&=}Ad z&=}Ad&=~mcF`(!41pWW%gnts@KM-ht8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w z-w_5(Is<|C6uvfa2BP`@|Be_{D~85^#(>6v#(>6v#(>6v#(>6v#(>6v#(>5^QU)~t z|4Hf5E@}*D3}_5!3}_5!3}_5!3}_5!3}_5!3}_5|Cm4|4|JT0%_nk1MRtSv&jRB1T zjRB1TjRB1TjRB1TjRB1TjRB2;K^f52{|BW;yPz?kF`zM^F`zM^F`zM^F`zM^F`zM^ zF`zN;HpNqg~J# z&=}Ad&=}Ad&=}Ad&=}Ad&=}Ad&=}Ad_|7n(jsL$hrql|eF`zM^F`zM^F`zM^F`zM^ zF`zM^F`zM^F)%0t+W!ARsnITI3}_5!3}_5!3}_5!3}_5!3}_5!3}_5!418x8(8mAY z8B=Nn(HPJe&=}Ad&=}Ad&=}Ad&=}Ad&=}Ad&=?q$0d4>Ppwws=GzK&VGzK&VGzK&V zGzK&VGzK&VGzK&VGzPvi3~1~B-x*VC1<@GL7|s`xh*So4)nj7m^f?z}K%H}z)=B^%Jz}MI13)FP>_im|Q zA_>iLt?_POyff$vxE8NyTs+6MvcG3lFd$!A=MB{Od$#uZ0s$YYnB!{jp5X6m>}!W~ zdOM_8t|hf~)h#QUUBv}7FUOo)TgcIluHJUvju?Hdk>mO$k>8k?TvodJl}l=miOr}& z(ABsy_PT2p(9suM)ikHJw;e(}s@d7r?UPQ(KO*|-mlV*lvF6;uLT+Fx(jK{nd5Jm9 zQ4k+`MIa(Bx?MSA^4w6}w`CQ}M>hB9K z^LfF%a!w2Ah|#i57okm?>R0;PeapIhecryd&Yi8RynVjjpmeT&NzAn|3Y*4UR0v8o zjk)fc6)9Bxk~z_83aEaRoTcdrvlJ~Dlj`WCS&9}MGD`)ta->{|%H(@jNlc95Q3|`n zTvTZsMfMao-W*pnKXS#9OtRA3<8w9F9wXi4zHV=@>qK8ueZ6#RVs)aUMwoM#7I3s$ z<}oPU(ha_#x7{1`wkqpFlj7n?C69`0F@i{>B7$pHNQacaIZ8dyh6r+#Xk<-dbf%QK z7@;Ipp+qI8O64+h(Ya&ng-&6^jOPlrJI%S3c^u6}?xNFCG*{6V1O;o+R}>-=W6>9t zw?e_dIEOhmFOSE9|n?AW7bi7ybW>*@~G*QWXv{x+zw{@wty0X&yL3qBv2NK^e9zn`@|a)q8` zH|Lg=a1`nsESOnIfhUz=JmqX2a z=(J5yk(=s6=M~ZA7BqCpHH+05d{Pt)}CLCcD2P;^|f7ru1#ItUBR7-EAC?UH^)`e z?G0ev4ddi@Fk)1C7KM4DE~vB*rl)8@pj46eHJ+9NDwxc<%iYlZi6TRy zEmX%8MU-<4|e+d z64ufB-p&3bRhYHZZCezT6sjS%)G?=}mO4@fxkngXJ?UtJIky2C+>#iRB<>(ak_ulp z4q}vWgt(4kf<#=iqWeDuv{8?bfq~i>vxE~(kcMFp!-Qp2S|&Ot62&l}iuA(j11o&q z%}ww8Uq>w8Uq>w8Uq>w8Uq>w8Uq>w8Ux=22Bh_Wr)hvVKXi6Go^ur0du#`7&DK9y zXYr?6?z6beRd7-J(-_bg&=}Ad&=|-%19agL=G?Ywj<(9nw6I&F+t;!cc5B4!9BYkT zi*6b;>LkVb%i!_ilEmfRL6e1rvhYH>w7ey4y4pD2oLgPZu~nJDRSkY+V$8b4VC7f2 zR1urSxK)`{cE%M@=QvmkgWWVGvUL=D`Jy&>u)TMoy$teRzax}p7(y1gSHy|lBf=CE zXqyY!Mn`RPVUxTPiR)qE?S#1G8$QxD7qVtb+gzw<2VrMGf+cO>D!kCEZ7v*SzqP!} zAbL0L99L+sN)&scQ<2Rt+U7#o4;9%p!M3dYk2e=Ogb#_1TS&NtmW#69wfk--czzRS zCc5h;>MZ1Isidxk52`}Av}~R2R~5da%(F+2Kjkq)rR|ZEXXGuqrSh3y+*JGUvo#n` zimU4H7k5`3IOWi!AKdq7CC0B>TXoEg&z;Y1+aw=9y|!xi(_aMMc{6m}QdN59KW^V| z9_0Acma1tBwtjilYhGENG*><7>-^c$OOKb2Prj_GDsS1u`d4m}j}4bs<^SbskE!bW z^6@*jRPFub%H@qK?!)6M+uK$DJ>uU-pE>#?`Fc#DLc9<%qw1{>My%@Tnp*Yh?Yp@Z zwbQGls6)w<|Lm_iBmbHG1%G%zPXETvD%06d&-+PNkR5xg9He6PITbzf@u|mEO?~dt z-Q#cUV8`pL>L#2%rGL6lK7RU`D#6WJR-D$(j$`iH*^x%{mBr@URYH1O!bxB7-Ie|5DXxA(rORcoJZthjF;EB_Gkr1YYy z3$J`OXZPvX%K82KiB;Ce-+1f?`n`#di%+b2{g+MO-_t%IU%$7%>Okr0MeYBg@^Mw5 z>gz9l=3Vsh9{HGbSACOz@woq%ZI_Qfx~gjWf-Agt?z~k#{^atig4+(fvyR*>AOH5& zs$ItSUp#lh{doNJYj0O=nA7p?M~@C&{{MXQ`(qa_oL057dg>oOdwLq$KTckEFP?|0 zk?`*iE8eS0c;Tlv$>+Z~Amh@fmGSPqNO}Cp%kpuma;e5YTzW$Jip7xrKi#M-|C)Q0 z$9o@A9+Q~ATQ01+^Rrw3@lo+F<$7@DvvT}DoS0bd@0YWB%IZx_a)ivP`fc6rzfb$H zL{2ZZ-p0j`IWbiHQ&+HdV4a-))?HOi6`$7K@S`*2ZF1@8{djGaRZ+hxp`S`B8s$ROi;iHS6dqh6|=s?x) z&Utv!=Y2nvk6*vL>b?6vsQ%|;tlj?N&Z?h(cK7EG%w_Taa7Wc0%U1MPEqz#y|KHoH zCL8|vXpO+yxsz_G+V}oFf9(2->1)2Gs_?eoEbH2Tn;d`6QVO z^b77k_4oig-c+^g^~zJPG$hoXsS$!hylBtV8CBXdRePoqwa@&} zvoY29A-NE0&s6ytpgmLnKX|6Xb7O)|dU#0V)RDaOix8fJ;E+}Ln&^bT6X6R8v_Fjj zjRB1TjRB1TjRB1TjRB1TjRB1TjRB1Tje+k314g}$Ctn-%JSR;E7!OJN|HpAW5$+cn zoS!)_a27g#;h1Yb+g@sW%=$O$a=y*-wE4H@GE;-`4#STPQ}iC)rQBU0@Ll_(u5mo+ z@94K!I4M_qFy1!6nW~=*g+I&EMY} zERS4t`PM6iN1wL`f6Bi$b@isBK%7i{xi&%_9n{OEm&vkL@F}s$ zE4%ZFWH`PwWiH61D=PA=G;&;4l$uZw9P@M+=*3CYm6M4n$|5Bj9#fQw6RB@(W~L}r zYl_lrn4&aIQ%s<)F`1a6bhu4XDjq?7qcbx_iCR;XWWy9CX_{g@b&blz6eYuLiV|@g z^^MHT6rx&Fh}kfOn5HRmscS?grVxkQ6rz|zeL`lYaH}eyG&vU7E^IGbn&9Exc zk#hMryZXWXY~SW?|MnC+N-cG1v3*D!xzoy%p8Ou@2&ULwZK(@qR;n(k_0d}D4v8b0 zd7ji1)>sVDs^u;+_Jw_Yt(*P++8y4Wt>F2zM(Q5>`XYMbVjBPT9ub|?WwDa=bnJR5 z-9THXuf4z9*T=fvft`V%ucx&Sk^*IrX@Z6?O6K2syk1$(F3mw^#D2x5zggi+GrOR2!ULq}+ zIcuv4tWvU?PrzQUx7W|;SQ+{00_Vw+ySQ1g6m0Ek3q>!I{CpOm0)CT`C$(iwi4^p7 zwe^KW8lrxyQ{gKtE|+y!HW`xW={gRyl@J>D2eI9(N{SH3r-_9F-*MR26*Z`+bY5Y1p*0fTQ(i6>DN1oy350yX`8 zlJjzxbl~dk@7V-S24n6fjEAWi@cn{!Rnx!5_E z_eAjL-{C_~omkuF?z7B<=RY_1+zzP~i#(E2QW>p^shFksE1sQN$}M#Tg|ku>!Y6HU z0Z6)2RELsoNm5KlQj{c1(vd8YBumqgER`h7(vd8aB+Jv0ESDrb=}3Ac$%=F&DeA3U*9`F-u+=W0uCcP+n6XvNXbwWftw63LFYuv@Ol4bGPb2xH~JO zaCf@H@O(PW#gpwF+tXaga1Z-9++8frq`oO&j|`SF=0^|4Bez}rQpWsj%!$lTtD)12 zdDJyIlR1(3k&+G1QpS972K7zK%oG)BO;M2zQ&gmBis{rfF%wf%47VvN#A(zwAv05W z)SAMR4O4j1G{scvIwBKOc!t{)9??a8<1;fwxmr_{XTucbX_{h+H2xn!DvA9@+o!PJ zznMSF@|yWI^90ibW4obUf4A;#?hEn-M2P>R73DTsS`IXE0m>;KH!uSMo(|aLoqW*R z@88-AA9qN!dR`VAC8gFbRb}ELy1NXrJw8BpD!T(IrleTJjV(g_qbS)@ZmOE2?5$oj zw8DNQEiKK4*-B%}Je+1L6&KRoCD|~WV!=1I2=Qi1w(P5(dmME~FaD|*4Vi5LEfuq2 zwvyN~52x8m#QAi$I~!(GEDpyOA>M4smWb7Jk7kQrC{`~TGFt^LEzXA7#Mm+qr`bf& zLw6Tt!)%I$k$8(V~U zvn3n!rf9b4k*|8u(A`!_OXp<6Y{ju<9!|3rizW2fLdf>WMRc2TX<9u4DKAi`l5_^Q z(jif##@RgCnZHvxt(J=ZLDsBON#c;`rl%AD$$Z&By|Uk@X8wP3e|NW{PY=hid>l}0 zlT2~>Uv8#~3sajIgqhbUbEwIC=;kP*^JZDewn=m&pYje;sr+@U%!jL(;=D69=HqQ~ z&bACzY*Z`~MK9o_xuBx0rzGDYt*xgd-A0bg^ZXF1EhOzsb+A%rI{;bsFC@8V#-bC-rONhyU6cjRB2;#0>cAU}J-0Zp_98 z^_`XQ_J#y|4I>0CZ>K!Tu#^5NS%y@>G!=;U1wFFN5yKZdNH}jcSt;%8Wuy>J1_07_h3t} z>a!v(z}63tYM^*RKi#G5d8|?EoUcOZ?8ZulREj?UKUm_8Zz5rTDl+`X7j|Bc{t7H5v%F$ z`Pnd=V%a#h2=Qi1ws@?bdo){lMo+gYT3V3}vz5n|c{t5hE>_aro@|&+@n&^w5#r63 z?EUH#%@+NZwR+Le-FB2T{9VxM5?FyCT2$D}vTG;GrcbU)&rhSMh}9^Yi=EVZwO zVbXYi%rL3FE3nlIZ`!ln1EIsg_6{hP1&53ryV)W#np#iARZwoqQMIbXW9Y6%ss8$7cQT}yl46lABs&=6$(HCsvMNiUc<)oS-9xo^9Tit*!({GI zu7}@bZgDN$)sPL7DHiCI1&D8Ok}c1LWL26hdU;O0XvkzOR9uk_lNE#W2;#?>bLsOUX;#?|8M?JaKi^&!_BsNjw(Fqpk)F(9f3#F=WXz8`mH3K>IK=RnK zfu3}y2vS|5Q~$ig9+EqHp-#Q5a=A{uT!=vSmt6pwpRDnxQdvQ{|XsD>l*C$hyk9&zSMR~1-fEG6T70VR) z(^`nAo}YT||vpD8czKy85TOaDdcG1$~fdIWfcIhirhM~t#jDV29@Q?an;j5GG zYK3fPJh`G+_)?Z2ex#mk*()TglCJ1wFZH4!qjga6dr{1pW-ek#^xZ?banc)Dv{HcX~i=u#FS-ek#^xk9okO%}b(rCu~-vQ{dt z&4$U!LaK+~WM$&_=&qJ*m`w4Oscvq+)Y6Oja7o_3)dl zR6LIEYRZPm6mO_13lMLzWbdbjWL26h`u$Y(qM^HN0~Oa~!(=5P)x&SH67g8NYjrkE zmf13N^uB-9w+fDlqTfvo@iw%zLnd1=k;2RYI{a zT^Ql~+WDUINAP8UPG_0Z>3Gxepo2P&b>usU{U!Tt_JDnbeZ1|Twx?`Y+I+U7Y&Po$ z)&tgatsAW#>uCN@{C)hn{06?7A7}Z%@~~yMWtAn@{IU5y^J(Vg=Eu^`r{Mr?K2dX*X!)n`?~F)B5Pjb-apYEUfkuPj)rQiHpp ztVN{;#ftQbf@YN(ikG{aAZlxJLX``sN9U~pk)=b4)E}`LqPZ=cBeFd_Wb68}Rd5Zg zAGe=YVgJ}Q#bpJJAO_YST5Re3l@QfGRMbQ(TXpUR5Cuw-YYB_O1bgkQ6(G_ulnCtk zUov|+#9Td8Ol%`+N{)uOZ9~OXPRuRNuLq&!Lkm?`EQ5$WLq$w5`&Bh(DM+@cl&tjB zLF_H6Vk>5Ajw-AL!6tQri#$tIYf(%v99dYST1&-3&tlbD6l3BAh1IIHc;to#oQPg{rlbmUtGZ)}rX&iG}l3YbkMi zDpYGxJX4AbJ*u^cMV@liS`^Qfd4*-FwYcYcN>yu7JX_`zmZ;WJTv#GPXr$wel)r>jU= z@kE%IH%&##6%*W3RivzV3LKH=QjxM}ynBj@loc)ixV*_KQZCPRPg0SxqP5S-o2Vk? zva#+7DpFRo@MH3hP?2)!X!m#(DJxp_QF-H3q+Bx6ovR{cMN2*+FGoenqTn8@B4tJE z?aUjaB4xM3Jz7P|iWb|RH%djy#kRR4AzG3aL)l7Or;dQ=Nm`7SI)0o0QIzeo>@kBM z?Sw1SdCq}1w*1iw%SZ>DoMPeZ;Q@oiW`|RwEu0QcSti@yU_Q_36MvDyJkbi*%iwyV z4>g!Z^KcbZ>Z88MU>swClX-A5`YQ^CG3M~u$hQvkf(gz}kkrecDA0{EVv4*z>Z1*I z1Dwm{Eqe6xKSB>jQ!Ex;_$vTbor&;cEtcw*=0<7!zmdF6gkKBS3jyIsVWji-&I8UJ z&IV_W<1aAQKhv?wQDFbV{-XUB80#;vkF>pId&YLN?QGjN+j?8IZMJQM^$Y9U)}L7K zvR-2Cw^myn{O|Z%`EC3np11tka+9Uo;<4z=zccSQ_nT|XBTes`?l$c*)thpSAHtZQ z8aEoJ7~X?1{|$yy4XuV+!%Txw|B3!3{ayM?_5J#0{X+dDy+!va%mDmIceQSt?ns@$ z{f4`p+sf5&R>=Bm%2@ylh3*s~9Czr@YzmzvT?4NpcdhaH``{BTi~Bck2200%zORJ* zpqxSnN#|1$S#dX#{j(`_mQ-mG$!+BnI!LOtDDF&h>ud_0B~@D7kT*EV*eWg$|Rhb|@io<5UX$C*49RBK9U)H_n$f6^+hns_W-GTmj5QSWuL#6JGGA+1BuANMw zBZIlRP0o2&Zy*?G_3iMr^}{y{$TbrwbYpaGP^O5`6$tdh<*Ub1smGG1$mFIza`j{i zJrqp$=7i~9HIYJp1k+VSVCi0&L#2L*GQ*p<_LD0oQ|N7A!Z#*N_=<@X`Wcw8A_7bJ za`+$`BwV1#FnDEmlglSl=tf}5*CkB(vWXNr4Vbbb0!#VQTq<=I;!-Xqm*!EaPXP63 zH8$U^(aDFVc!11H7E-BGunptEcZhZb7G^RO&V?N}`)wSVX0M!g`1k?zzhak}J(tskQ3u3eIcx1%h3@9cAQ#A}VzmR7;eS z^NXm|V^A$oLe49qQs+UnghMwS=3TQ$(e1gldUma&{4wdJ#iPw1&AltB6V+ z3e^&&cgcX zQ*x4Y?5-ac^;h(ISfp-E57Jq4~7hOffoS+0})xw{n? z;8`nQ2oBHJQf!tjR0eIHQ7y8DX1Q+Ip1*B({d}Z6Xov~NgQ)27b9dIQd zWMEwhJOP4|P5ptL{+#V_TGB0_mNF%s@69<;abD6bo$tx*hx1bD~!h$d%<(t_90x*G-OaLaO+Glk`JXiop z77x1#ZSE})Q@WqR%}2ara^>XIro0nCWQYZ)#yW3#7ewnyo-#h)l^+y2es(9w!IDg} z9EIe{Iv_4ATn!mlF&jYHYEVpOtu6STN)2T#WgAs$P)uqy7aXTjLur#V{vSa) zi7>-?s^cC0D^sm;z|f_CP2a5hSa+&!Ja<30lDr8KmMKTsu}6~wJju<)js3X)J`&)2 z2D`TP@uYQbtI|_h=~S1UqqJyDD`7%+h|p1qPM%%rzG(b z;v)ThWa$@Q+`7-*0k(~QyIXx3!u`Fdj^esOzULYJ=BUC|bBgwkdAnPEX7M={_wHtj z@^No>rzjuyc6W;Mac_60C?EHBcZ%|H?{20jANO{5it_Pqcc+*@g15WXrx5#gw|Yrz zH=^F|RxgRwHuQG4f8VCfaL>lS-K{>o@I9N-+uiEZk9)gYy=bVY%D1~yl#hG6J4Jb| zg@}2(J4ODq79xY!l(5s)OQ-k}IEYqnhZG$|r<5?}u!l}<2M5tjF%F{DCxp#mD0-6j z1qSV|DrFTb!VD>h22ebon>CLZ7xxtM8_8 z53N2!a_{qyxJ4TOk0#rQaG@~W`GB*=zQT64^=DQSzr-?Ne%kb3(~-uVhKKc^>5DlJ zL~D#4_s1Mhp{CG0fG#ox5P2h?oKF{1Bd}@PMr|-ck@i%~YxDQ{io>gLjLq=VMV=Li znf7NZJjK0){^t~_EUqE>|A%o6!49bW+B4;TWNhy8d(MV zx;DYi1ZkY*?da(9b$Ek5?CfTk=jhMNDEr7snF=jAx(hN)K+H6Ol1vjQ%`|~B`EwcM zTv|!>Qe)6cM<%Vl32SZW{=~Hgww9K#1}Bmpv=StW9KM^PRmpYt94LY=n?)!+d_^dg zi*Pm+0i2U&&k7}nuLvb_5zc}lkZcw~JbXnEVp z@V7++v~p)QOK|wxB1$WFWU~Z^zb!g-Z*aSnbkv?vqx2UT>h-XWeWK>DCG8$?qSO*~ zexhBss>o2HB$D-m)n+4ES6FQt$$G!H ztZ%7WWetnr6XTTLezp3-JA`C?(mJS`h2*dp>f5~~2rHol$roJZ@`W}Y$?9NjSB zOfMNfH%`zW(x0w3>dw*GxJ$UP5VtpRTq2^-O(k#X*bPDT=L>9lss7)a>;MlPRAxLc_P_VNnlgC{Y*n|x} z-xQ9?_G>1Y*O8-!o4X-nhFg6$)pIRvp`)QkHob#RS60cm+^70p!61XI#K|OKEwMnP z?-FK|z0V*U{Te9PsPAIIuBLP8NGtI-(CI_e=ETI|cAu!~D^Dm#=3=vMiK}2Td<0k` zJCosw$L>j|cn2*}QmN3yFdM;N38fISR*J;q_oVsBtQ2fc@d_w~Gi#+tJcb{}Qm|>o z%b^sGtd%11IDQyQ!DbdOgHqVDR*J-9`C%*tn_avVN@2@dDH4z8hp`lFhVc?8g*9uX z_+PbA7egudtd-(_)ka+erLbhJ6#uI>>Ov@mIcuf(U$s#eKq*XFE5-k+jXGZ*|F0s# zX~Goe#g4BWt@d~Aqir456Zw}d?^))XSD0=wK4KIM)AT3l&f`8L|0b(qM*rbMI*YpL z4C>}de@{Pr<2Y&*2U9q#uRp|#_#Vaels=xPUWZ7Yrd_lctmo*VUijKnIyqImHloP2 zViO4tLz4L}R-NtgsO>;%gmAb9JB-OwV3eXvj8d*B-tZWuTqHjNqs+_9D2g|26uFK! zN;2=nYWPLVM*AgJBNR5ulVFs&nHZ%^QM}^!c2@(swm#@7^PGsKLn%9&dewZXGMx!#~US?hha7RqEVuK3#$*s#nQ)mpv_{^H8z>aihb~2wCQU)s{R3(Izm~j0gy)5;1+P#F-u_>9UIxDYi<~;g z&mGq~+8kx}uffy*C3c^^(DtqES=(i{4Ys+~&%oo~IpF1gI{z>JasE8MiO;kA#qy+O zucgH@!TeYA1Lo7g!~Z1H2d0O>+h2oeGTN`6}?UOrS72aRq*t8o9=wwcHKH%nQo?TB=;Zg9quviOz`?Yo8!V04)H&V zD7bt{=^Fqu2f!DT4jg-M{2+dM$Cyzz@)28D!`8)0FO&L@P5geBh`YrzZ^3Fj+J!ajFK~B37G@TQ3K#R zNe3P0A7y3?;FHO=m>Mn0C+T# zl#_3xU15$L0H;Md@P$dfRXD>OI{=P}Lg%C0VU8F8=R-R1hY1;B4lzela4rNX^W@to zmzWL;PK9*f6O(+4PBBMP@FfJLw2*JYZZYi?+z9EwFDCg~@{Boxf|DTKcyQA96*ejX z^3hld-hp(}!5`qiqZ$tKkw7KiK;tbsoqU>nR7$}ukj^Xl9aKEI6`hmV7uw0eSrohm z>FS3TBY&!(;7CZ4b74_-DaM7rh4V`5uO-WMr&C>qLvq|y%$d2c2K4@E;ckZi0YfACQ7P^9Zfo)fJ4C5oGt zyjw`YMUk#il_Yt`L%~0hDm|jYArE=Gkb=h|ReHqvcuSM$gDBSXpHw!5^FH)zc zg8be?!H1DLJqqVMycns|;~~HGQ1D}}##9tz%!)ag-7 zACg}eQgCRbPEQ$m-9y2rkvct!88Py!LJDq;)afZDuX!kVHd3cYG08(-Eu`SwNS&S% z@=Ff||3>QcDCRB5D}@wX9I4YIl9xRcyd0_1qZo>lmkKF3I#Q>{O9K0{6q8?g=z5iU+{*Ug=Y{kbm3rLs$l{83i4AAZBeU7*&h6) zkT$E;;~~#^Xp>q!%J$&dLb^t+o^tYI4_&QRkFq^@rjV{utEY@S?V*io^(fnerwZvx zwR%d)k36(NtsZ52@MIxfp;k``dBQ`NtJR}y4<0Y1N2}EXbK4$TuU3z;J$S5;E>o+= zO&;~orE2vk+k;06X`Na<#pK~~S_`q&I2Knn2M^7rOF&SK`||nZ!E#!oQj4-NcwjbN ztWrw_xxbuNtJI=w3htXtt5j<7kbBE%rAjTzhTy<#dX!2n<>a1nx=5uKWixR1YH_h zwLE08nJ1XG8L8n7{XzXAT?2PBd4QD5L++DkEl;-h#BBJF%+L%z=8*@}DX;QJy%X>{ z?WRj$ep2w$wltUS-HK{6RA#fXc<(MxYYG!q?|(&S(i(~F?P+eQa37B4dpE4#zXq0B zoS|hDx7tva$;t}8JMAU>SHUvX8Cm9Vtl_(15&xH9nW_veqiCLnvP@Q%@!e^!|x8=DT4z|3$FOkr`S> z(GMQVGFe&Bcc;Cg{{mQMVMdlY983CcSkwOnSY|GsW@2g`UevP`j}gu}D2?=BX}PhlTSIanrx z4X=vPsUq8oRSoRn&(Fr@r1|NZ677s+#Al+w(Mq8Q98avg7! zifkCAB3+|A3q~o)#3&Vt;th{cDn#;QFp8L&Q516@id@GV#gh%Ac+xe>GxGTV6C$j2 zZgxBYU-_G66Rl_RS6cpN{;#>&)NcHt;THy%zF2n#cRBfl93Q{rp8O28&||F9U_ITr z0x^}bx)?WBkD53*2#xi+Od4zPa15kH7)bvK9;0hBdyI;y45Y=Z45Y=h2htycWm+<_ z%;6YFi!hM>BUq+6L(8NxkQTEtkQUP(NPhs9Y0Ahlhhrcu!a(|cu*{kaEtASXTFlBo zT1~?}B9-GqTL#7)XmSkbVa& zvob@=q%x2evoeqt(;i5_4VG!h$TEjxAT7c``Yo``iVQ82%0ODo%0OC7dm#NLSY~-f zmN^^)X%Pm}zX!`4otb5n+dhUEAGotJly;{*mi`@BralwPxDUr*+6|-W--2b9WoVgH zhSTn>jHlgc52)V&%Ph^vGKXVC?S>)sZ@@Bj8CoWlF||7@gKBr$qv~ISWpIH!?Iqik z7s$hVwdbX<3NMn^rSZSRG?@q^VElig{YBebF#cc6UuwD0GS$4s^ss5N@$uv%eC?{n zfX0BvfX2XKX5e`0gPB!sZW=SI(Jyzb3w>tKd!nz;+u@6vG5ws5q40Sg|7yST{Wmp_ zm=r%9q%ukQt+mhSINHXO!0JHi`NCIE6n*Z5D+#GRPcczBZ~vsr=_csw7#G6pvnjvq zt9;od=JnYG9|}@Ko8l8eY6P$}>%s==gYF48_XdCJlMBgaeegT8NrU3q zsSV-dKHsip>H)`t@Qr%qhuc+4#(lC~wPf7K+Eq)INVzRdksK^%sU%sJj%1l6S)Pt$ zxg_aHM>6hn?5Y{6kaW&ZN9TM=(w%C}Lvr16Epn%$){XVZEv6-lHOXC)mMqpKcUqOn zjkU?0R&{b?eR8K&q1;%b+-X%QH`Xb4MOs{8t#YSTt=w3z#I)*_i1kWLt6qt6`;=C_ z60u&1Y1Jzc>y?;Ry%MoriD}g<5$lzhR=pCjUWsYdD-r9Jm{z?Kv0jO3)hiL}RY_X) zss!s*%Ce1h1qEqs{+DNOKbOY;0=bfye{Y^{nrM95xYfuT?lvsdzo$P%_pWXM_de%? z%S(sopsl3vQO5qpO`1wqc`>Uf ziJXwCOEijSH3IT7${~@nokZcQj*=JBG)fZAu5zAUO$J)(K)JPr_VJ{qutvF576^KS zfqD3(lpX)o?(p_(?e@89{QdBCtiDKJFD@q59L_?g(rs35FG3)s}v z)8Fk4cKLfh-D9 zBxlk{C(x}{(n3eJNX3s8P2`#|DvQ$PEMzc|7CC!{%g1Ie$}i$dZ?8YZ zj<4IhHQ;NX6&pRu*1*%r_JzKD8f6r@y*<=GPvD8C#FMBxqG!sQ{=Pn6Z_p(j zFz3Iq*}-^ndjDNdyLhswa*>o?lut^9Gus-;rEQ|@4kjR{{ht9n?SwX%FB_=mQeKIS zb#7VL-`x#2Z@{$`#tq(0UEN*5ozb6!R6Uv~iT;$c8Yy^0Mz!2(B;=M`FVpZPEukG& zvTYJ9`=!M~sO0I)?59&$Ion@cIE`)w>y2D={m#_$9iA#sF+Y)X)&MT1)m4U%ZO=-t zpDYa-)9NK_1NnJFtTSl?`H;`=A&7Eo19>&=tUOVp4dhjGr48iOZ*TwqaUh>!7XkYz zb0b4F)%6JG8suKWzoha17;*v;HaNFAUbMe&Uu0{t-orn~PqCDlFErg|;*7b5KK+@x z54f+mW{9{f@l3#!0r0qi(GmN_JvBpFY$+5DNRmw_B-DmG>Y65I(5JdvmqTZ-#}E8h+NM9*WQ=FM^U8hS7l~0)7{hEpdcVuC~{)%#S{_!d*_9)AWJZ3lFi+gBz)hb=a& zrpS>Yd(q;;xXT`kqj@CejBM~|ZYJ6)LItny;CeV?R7#+4Ku$m%A(h;#_NytNhr)H6k{CICx zt$+@i2p#z~zBLRT`5J8rbdbd8uq}sDQ{+fGqAi8OO-JZTC_F~NtD^;ss;)|ej=UP* z8itNMjh^A3?Y}HBI&5p8)D$_Aj%e$j@YF0`9ihvg@E8T@XbyC&OoWcy8s8d*j$Dm4 z13FeDMu%3UN=vbT>9kz8%YKk05M>H)LPt777p;9g$ zqaYnJ&|yxB$K4IguJNs5=*ZTn*X;ink=Ka&hWZC}o_emTDqks&D{GZ;N{0Ni{HDA? zo-g;3n|MF=KJH!ay};WU_5eH}ZIu>E=SeNZU&a06ZQ>9e_z0y0wK6kKx38-P%Ej<6zr+ z?FAXSwSf?i;e>qM+CPZnVB723_zd0JK8VL~T)uAY9>j65?JZ<%hHh;h#A7%nU$^!S z;yBp$a2=hYTU!V57#8L0*3Lm32iu+*=V$2F#z8!Wqw;lY-yn{IZMVXa8M?J?5Rc&q zO*eN9GPkpgcD>jmY0m4Sn;Qn1n`K5f8s1tr_X_fbZ9K`Y%i;ODxgSs|oF)V2x~I>B zYqVlPEmv#!9P+RX-P{_eh|6&3*}AzKP({55*EZKVMAOa9fz0ZOUc{g=Y?C;+n{Mt4 zG^$=B(8iyeshhh3jj8uoe4Y%-)y=JdX4ZSOT|YiBT{kxc8o^;UpqFlL12nJRL#=ld zTf^($LpOH?n!sz@uTVEP1DeZsylvF?obI~0HPARt)4qkexgii|PPPHfKHYV5dmzr7 z3VRpo=B7ZLIoX~-XLr}lErK|6D(F?Hn;Qdh=45*c6?WInZGt#+$}cF?3;A=hJ&E$W z>jnHd<>eLX`TRNAo<_Od^*sKZa&rpxT>hMFPb96op2MG0j;~PH_;a#7m9o3*KK`7v ztU^7TKPTIhsb_aRi$5n{k3zjCe@?cWMECA`5B{98GjsLsaCrT9Jo^(WBVEq~vk~>) zA3&YbO3#2p!!}2@o=4rX^=|On6gW*?yF!OQo60mOb9+R%QdRI8i^sBBj2OlXR zu{T}1>s`RiymIR`gC|kvj(TS}d_a`L;ThDav)&1g8XV;)eEM|ksdt3qqHRuXf9`bX zpmzX+Xq!_P4LK`EKMM|;9wql$o;7J*^)#>=UcXg)O>YlJnvd|hstnI>d%m>mqPGL1 zdbb1b)mGEn@|@lFOli|aZ^Lu;!q%GJn&<4c=g65|^fP(RUeHR@Tk)LT_UvfcMQ_P- z_WTx_-h$`sw&%qeUGy_}&Yss?)0^|0-S&)V)m)xpujrQ=8_BI)MO(wnk?>9}ej;B`C`;l9x5aGqj= zqZ&=W@?yOSULMyhv~g)_Hhgu@$dixt z=wX3|wCfr{SIsA9k84X4xWb@s6TGt6@(7_wGpvOK`gG>S1I7Oj98_B+p_T* zHdTu?^J2Bk?U?e9P~k6k-Tv2I!?8Wkt;aJ){n9S}X1Y?#x~@ER72~?DoTs$P)!BcV zF2lO6BP#8$aXcqM|7u!Bt?$!u)EKy>RQo68URNom*?06ZsQuGY9$Yh$i!1=kpI&}f?3|38h)BkE9P zuzaoeYUv$upLmAQ%rnJ3Id!w^Zs(Vd&m9>l>9hh48i6ALbtUcfZqX$G17Q3h9U4Hw z;|I1OJFA0M#|$D?Wi&YefA`KhRRaA~Xchfo{GhA(7L@4tL9cMHV`?I?I<3)uKwXz) zP-hzt4R<2)GL9x4h|!Bs0Rg{h@Pa-EsOy{z>I%cXZb<4tTF@7$>y!-YY=h0=PDD}{ zO=iGf|9I7fiVgTpgVgl_>N+Nax`J@88P?sO>bwg4Il7(JCU0O1zvrUtPI}u4;H2Fe|)P;%|_)UXXT_I4{J{i>I zg?rtQ)PcOA0H|x14C-uiP~lEQQWs6?z+eA()rATk_)UY<xYrFy9Y`Va zfVwuxpw2eI7w$wPb$R=`tx-*kOoo)6s z+=)o)qDd(D>mR8L6;kk<2C374x>m`cP7C+CA*ln2g%7A}nGEV|Q`6y2L{b+`Zoyyw zNL{Gtg5NYqT{ckHA{o^A!o6-t>OhW>W%mEQWD8LTEB)mi-VeP#={olVj^mEZlqU2_ z_@RTJ>UriqQx)Nh`>|+VHi2GW(0#W301m%G77sWdrmu6Mej}%^181$j6fsX<$8zeB zw_JWRtNFml*eIC3&Z%Ykn*UT2hSV{A9m^@8BqQ|}x&{^9JWrn&Z-_^%li2*%n{}y% z&hYZv+9$O6?Sj|%dHw;qPb4ebf~vR_za`!~LZ;36EwR-HR_UR`*38ti;Hs5-T(2g4)y6c1&R^n) ziFAI`fV1}L^cd|UCe>rKZ@fpqJ=V6`Hg{kk^4EBe2zdwOZ+q73g*E*>)SssJ6v*;1 z%k675^Pero$@!R0Y?X0{zmwiWAgd;>vQN?Tp9(#;4zHQCtF=2&GhH9!+2KWYu>V56 zyV+yt9e==UtW)>=rL;}p^P5?X1-v`g@6t1&!_YbIc-J~NXXU=?aZZ1bPW23^=GN8z z-$2yi%5eEc?~T&O;>Thqp_6BxdtT}@u4kMl9VZ>bQijnR$PJ`d$khJ? zeW*Dq5V7QcY@-6bYSs$0?l05)EA$~ym#r$oH3s}WStQjq(HyX?MPqCo97SOeu&sZD z!8%n)TjDphu54&a#r_<9F!)a^!W9PmC$q{CINtI(riK(S<#k!37XxkQCWAKHBy*q~ zbh-tHo0tfA(FOeQ_BHcPTE4JmibMCSKDNu ztzR-|%L`=Q@U+3i@+6?`oMg~un^?BH5J_9KY2|<|CvBnA%KWB5+9m>ReUm|3ZXol9 zrwt~RF96#5B!f2Fq_W+GNZO)JDFFoWXv<3mZL!aj*9?=z>as@9H~arn$U&mMtKJUl|K+Nw%A3l)%6w&@(o+6X zen?&=50f)x$$Q-UzV|urJ>Iq6a_?wwp0~A^NMA~?O6#R5QWx<@@p*BLI96;c92K4r zRtQ6cHlD9N&v*>aXjljE8}|-a2XKtLUFx@~J5z5;9iQ6V^_}Y-*AuQAT#H>3UHPsy z&i^<+alY)l&uKWzoFkn1&Nfc3<6Flb$K8%HM<0hH<%5*FQWm5Xq=@t#dLNxj&!H|T z_FEw%5`=gRSl-B54lu(omo6G5q|gWI*{da=JeF^mixrh{9Bg}GCXZ$q=CVaRx#W>Y z@(pvLBA#4q?|0gski|uuUJeXma%N6nDl0&xU8|DHmU!5!rl}VXjxy+dL3&NyYQ4=gokP9V=!R&n~a7 zoKsRsHnlU%1&Z<_j#*SvRR()4lDlQYT$<=~^MJAiGs#^o40Fw)uAUU?uDQp+9wX$g z(+qQYp|)`3eO2W%iszJ7Rgybf8|G?4S#bEhHHVj1R`-NM$ek?=a}A-elfkd#&US{m zSddw+%~sjV=2uo#7MCn7nOR*0<+`JdVJ;Np3qPQ&va%X}zWoftToI_Vy_ho=k=xrD z<^n+GiSMfQ#J9CE%*B4p6Wb18C*Ih?FqilVEAs3G)nsEk!(7M5JoD|fo_RwX!(6|| zJhSZpcII1K8saUux3n?Lb$HBE+YVr-zPY7guD%m_ z>TGgz7sFg*$81d_aEP!tG=spi18Vf9K8Cr*&O+-z*bQ`PRk39NMa(WndU^(qFAjcZ z9g|IN>|>a#?L;*%uG{>E4u-h~Pv2;{ll9q#x#rGf^Ppfnid9Cd^(=tKUp>F9s^|EU z%Br&YrFmpswqdTm!*7yHuFp2iwRiYUa>#YrhPet4zllb!%{G*nOnl^;Y(tL8B%55F zZFsp%ii0(FRkk6;WRgqPW*cHmCOKqHwjso1qLC}J4NpuaK5|92;f~29n_S-2NClIj z^|mo$eIAgWSCH@@45vx@WS*;lj;#2&upontoZVM!Cy1ePyghIhShsidc=b zWtzS`RwLh1O}{i&qwFPZ^<`j`2MasZO77y8`cn8kv*verh+9pxx~0Cv{u4Zst)Hq| z=!@Z}ZZ%dj7FoYkw$!WbM(|Lyep=K@uY#X?)EJphHS4#9t@O%>-`L}1!5R7@_^qpH z2OCb8mls23GpZ|>RkY9-!Vg#q>j$%d<}dSG=nL$>n58g(nb%UUfM3vG%LWT!{#4#l zpC9p4Hd@VXtIvaM04{{(QDq`DvN@46rm8V~*xSz%pN-4o92ISJ!*AO{soyhCU4p&E>1>H=LTU7xU&~ zo9n(P!(0@fFPFk8R{Vc4!TJC7-s`+h=*_PdK5_k=(vMzBib-Ev58M8sy*@6w;=dfi z$;ZN|NMty9;CWO3=uG%E69}NLb4@hv!5hYB=*no^w)SvxesAzP=mr!0b0M63Ol{%h z{O2-jCQuupD~NDAkzfUA^b!as9}WCQhLiJq5xnh$+_5Icx_VlSE{BDa^A{FtEFx4i zM*E0x@)+$S!pUQ_j|eA^(LN%aJVyJ7P|+CeBf`mJw2urYkMRVxgp>21Av~O%-xBW~ zA>riwme_2A;pAlyPF@68ZDcq({~3c`OjtV)Nrf=$NfrU3nu3;ko6`C-jzTw`G{JA z$@x!JS1@@jXMlptg0+*+gKBQQFg%!?|3vJSHguV|fU(s$A}5ifFBQj41Isyq%NBYC zGeJl`9IC)k5gtk&@S^@!%L)=Oh;mKT6-XX1<<3i^GhqII7!*9=xNTdHIZ%+gsEf9; zTfmmniC&2jHy!5xhl2kJ$8Gltl)K@1>!s0YX8*q>g!ZeY$`NI`qRJ1*eZ2>~Go|CI6t-5=SCkfxZK}QzXa;BX|cbl zT?G}W7se;Ha%*5Q>NtDOfV7zvIER&1F&q91qw^(;EDW=*^9jM7mGw+Bz=g78x}cpp z7c|QS9bA}`Oc#8o&IO<4f*)KcO{NRkr_P0J%Z1h8!t7+akay}_$g^Bn1um2%(}mnq z=R&UK!e!vXtYo@y>hDM^!G)R0bm7$Bkyd~UGm`1TslOvF2N$L%(}h!iN4nIg)~AW2 zbnumf?CTJ4Je(&$G)5X6SGA~YMs-zL#r(>&>7}KMN=m0!mEaow2|s^B7lz>|P)!I+ z&7`ugTivq(ysrUT63&s6gmdI3;T(D0tefDn3cXk$Q!}S>ypshdm&aGYOm5oPqJe1` zlT#1BK{G4Ir4{AQT0sgIU0D_mCUNQ zY{64o1uHlE|EG};h}u`V)a#d?6Mq)7gfh>)?oZt3(!8j> zZyvQgiFv_#=r@2DOOobA?cMVR^diUNVm)}VIB8zg-a~IdFWAW8I`E=8XyZ;`h2oT?JlLB+ZN8>mIchyqKRfFMh9k)EXF3oF|fH6R&I^^~EpheP&`H zT+~#2#W%(n_56XirYHl9T=vmM7r9OlZD^6}1kuJ7xlRymP?75dff2B$xURTt&`9Hp}o;OdIZ4J z{!T_SfmF|{4qHpHVg{S~uSd|IYx_N>h3{YtnllfLKIJbnP6zhodH(5POJDMy#iqcw zl{zv?DcX8K+{F%E35drC2V_jO!jnMCX^D`MV{@-zNP$Ivo&ZvsCPs>F!2(;UBT0$2 z+7D0t;-wwB&JT}Kkd()PlqQLgqS@SQ7*b$KpY1@3x%YPhSBhn z4}XRsF?u{kj1+5ph|~MHPGb!V^@o(1R^Aj8uV) z@{O|72Q#GR!ZOSvunJ8LQ{zVtgz)C7${NpWx3E0Ef0^Ne8P-8zQ@Jx11Khf2dKkcs^!+8wOb(pu%EmYl60>dDtS$jbGe<%R>=-k`9y z2lK&+x*Yu!R_>dsjTD%m?-w>fU%M%{)6vYWeD7XqP>4nt6cz$uPE9B3c0K2?^1pM5 zL7<)W3u|X}*m64_)!xeaj+OeV+C9%sg6GlBVda0@68*B;J+I9+7SHRpw{qU7x71gP zFa@p8wBOAFQ_ReNtDkM%qRgwhGPvqU$t>%NtGT<_uKK`vN@hVETg@dH_}X@QV2VYm z)E8)Ga%#j1Z&Hn%CuFRD$^L`+GEe#OI3y*kqd0MzL zQdhgyJHK-L(=jsTBKiazHn`S~{-s7bO!3bPo8qr|SlC{iBHs;tG56R*=KA;g%Z;w^ zs-#xef+Mt8#?mbsmM0)tpK zX``*!k_DYjjJY^9DQCE6xH z+@yqV|HESxyi%S6QqD|-lmeT34MPg-^z%m`#oSyz0a9#x{MbqzNlLUWet7B^uax|F zuaswjl$ME*l5cabVMu{}e0BmU=C<((kYZbq+*ayHQlf3(!%a%));&B%!7F74ka9*M zq~zJ$YZy{sm!4;Ul;(+%Vq3r6R_aJnqHWK^Q@?nngl^2kV-zIiX;GL!z7~pz`kPRs zo=|^OpA$;dZ`Ci=L+XdHC%{|k9`!}_X?2_Wpz^uW)?46B^9s`6rH`do)O*#t)D7wl z>NV;W>MC`aS_%6Hl&aI!De45ZNFA;YQv0d}s!z>SJFD&0R_f`hth!aI{Hpw{98JtaLN-6!2C-7H-Tu@x(%DydwWC0!(qmqtqGN`0hU zsk_ukY9lq1ypmJ=P5epxPW(drM0`(tQ+x&XR(MK$M7&SDQ@mNcR=iwXAy$dy;woL!x&Xy_^sT)$)rLIX`mAWLgBDFNNICWxb zQR>jteyIhiS*cx9+o!fjRa4!rldco4W3D5vL#_j^eXc#OU9KIjZLTe@O|A{Db*?q8 zRjwtj3RkJC81|qjat(F$a}~Ilj|k@EV3!GrWr7T83*FUdiwZhLa2ma~WR3u#DjxhNTQ=Gb~{^i{VU$GZ;>1IE`U3 z!;2YCWq1+8DGV=UIGN!jh7%cHz;FV?@eIc?9LsPF!_f?j7@p5?6vL6igp}3LzDD5Z z40;~J;S7f{9LjJA!@&&CWjKi8K!yVt_Gj3S;W-TZGVH^!H^Z|T_F`Dbuz+Dc!#sw$ z409N241El<8D=r;$*>2*?hG@933MscLk4~pXg7xG47)Py!mu;LP7FIT?7;9WhG`7j zGi=AOEyFepTQfY9VJn6$8Ma_}2E*nIn=w3{;b{z;GHk+7WvDQe8G0E?49&Nj1yBzH zes)t2Vk&jR_mjwPG}Zh}{zF|1oeUieQy5Z)#6ohC;s3ySnvmbffAI6Kw+z2w_%*|$439AUis6?GzhL+|!@n{7jNxI1hZz2q;in8gVfZn_j~IT)@F2qj z3_oD_KEwAIzRU104Buh6pW!}+Z!>(0;hPNKVE8)2*BI_)xQF4N8NSN!6^1V}e2L+U z40ki!#qduIUtst=!{-?Ok>RrpcQV|;@EL|rGkl8SlMJ6=_&CGu47V|SjNzkZ^ODIU z`0*j~Fye#cA?xc0@#6zzE5j|;*AL*wd&&I_@3X$X7e8($e_(iz_4Q`_co*5k@NVnt zyG$5E?zG?z3vRdIHVZadu)%^`Ex5&kn=QD>f*UQk!GiS`th3;H3$C-^S_`hR;A#u5 zvS6(RYb?0Zf-5Yz+ycV_-2%S_t6dIC$E?EiGE7%ux&qVXm|lwMGEA3Zx&+h3m{wz2 zg=rkkEFrAO-JWR_mor~!un3iEW2h&naXJcA|=`2iVVmbrU>6lK#v>4Ni zF`bI(MVL;(^g>K0V>$`biI`r1=>$y2V>%Agv1TfdG1KhPm=hePMCJYv;(GRVVZ_%draG5 z+7{C`n6}3BOiWv0+7i=H9JNQb()9)c$Irnysd*X=+QgiRw{_@{4j@IjS614l4VVy~=K7 zr?Oqys%%y^D(jWC%4%h)vOt-mOj9N)qm^Mwf2B~#R??L;rKQqD@hC+8MLsScl@H4Y z<^A$rdAGb%-Y##IH_IF4_3~PIwY*ebAkUGf$&=*KuqR=ExlqoQ)8#a|rQAgJ$i({# zXcvxp4?}-_zjv>9w|A#^yLT(});D_B!>)y^y-U3dpszm7I|=qL9OmusE%au4(_uWr z?5%rXH^X0~F9J^MX-J-a?;CdUl9=dih$Tx1jN1~Aodjjv9Ab-eMLa*D*|F)5fJ-|fY?_A#J(aR_7wrKuLy{J zML_H;0%Bhg5c`UN*jEI^z9Jy@6#=oY2#9?}K?;CdUl9=dih$Tx1jN1~Aodjjv9Ab-eMLa*D*|F)5fJ-|fY?_A#J(aR_7wrK zuLy{JML_H;0%Bhg5c`UN*jEI^z9Jy@6#=oY2#9?}Kp)K7>1)67BM`Z;V6b9 z5&u6P@J0XsKOX2s{{s_RfeEa@gjHaIDlj1xn1BjQ6a)s!YZ)ju&zr~qG1>y5k?8^Jzv7y@Jg5kZVvOkkHXyUU|2)& zQ<$->hPheU^##nm8m`H%F3xYAyPa1#7dtP8j2r(NYoM_P{x8%(6T{s4w-s!H{SQmP zNscx*%pHF_!anzZ4^erNqfHERx8GJUSN5CrJvo|dnEU)r2kQswj+hGH3zQs5H_T0d zhu3dLzA7}#U4hG@9~xZ4CS*5C@@2MR?iy@v>K>(VK~vZgk9^V9Fn1F+HxG|u0vl|R z&$}7szQQA-ABhoLtZ$9?>!LCeW7;iqg zPccJOqDaSj-c;xQyoHohY>IKSd`FZb~-VTx`t`#$N9BoFB4M zz{j99Jv$Mtsdg%q^BO4UKLQnSFOZ-lpaRxxHLd={B<-7*U z`GderL4wTKZo&02lN9B=7PoT#05FrE5HqJjIj@0o{sUkpFF|H(53Tx`Ny>_LTHMO{ z_ko$*gqS%M%6Scx^X~yOIf*f2f9 znX$ba)yGUy1oXbR74+`_Gua6-b1EeCKG4wb2WGMoWXASBSsyb=QPKP2meKD6W_l*X z%&E}P`#?zlHZap8L1t|4;Po++6e+zgZY}*=z)bgqm^l?@53oHM$q#|H&7^0iyn@{;VETzg9n2KUEK?@2GF6ud2J$XVoXwN7XIrJ?b6m zE$TY;D%DU|s!P;`>Rh!%EmkM1dkPO!-)OU)iU;ro61YpzKf{R~}aGS2iiPDK{zCDQgtJvRtWFDwHy1rZQETsEkoY zD1()LN-rfx>7jI0&Qe+{%@tJ<6o>pD`GowV{H^?@d`SLKeph}=-Xp&#KPNveZ<8OC z@0IU@sD&HkYve2BRq`^qQl2N5%G2d3@&vg^9xe}(`^p8N9Ox!@0JVQJS(a11C%q@U z$Gk^C>wmzz4`Lm5d3SiXfzp4IcY}AGca3+IcM0hHOCbVcqPNI9)Y}i%K`?dx?Y%9$ zs@E-@gt&-f(h=#9bU@lC?U8m#JEU#W7HN~T0iq?=NUNkJQiW716-yJPB5A19Pb!eI zq^?qXsfDCUZtghUBpiT$dxTxW4q=45Y|C##wuZnP$85G z#ll3P$khBtDE?1an*W3~{s$6P`S1H*s`IZvrT;|K`u9V%e^=D|t4>q#f1*CkzoP<2 z&de|$`VI@z+|R-^_pva|w^^9xTP#fTO%|s41`E@CorP(>My)W-y(~;~4-3=$Gt=|G z%JlrN(EBjImzkdbC8poXH4)HO1J>sMEI>bll zwTKVXYY-ozS0g@1uR`2P*CKAAYY^|JS0es_UV(TIy&Q29H4yKnI^vzwk9Y@Njd&Yf zg}9MkhIlJoiFgZLfp{}rj(8Kj6!Av74DklK6mdOWg1C-aVV&1gE3ET6YK3)PORccZ zYp500c{R1dI|l>R#@ki)C%jof?8pnms2aO)1X#Zr%tV~PCvE6I#*LG ztaBB$!a6UbR#@jsYK3*KpjKGta%zQjUP`U7&SlgJ>s(5$u+Ama3hP`MXWxsY06oeQWH)>%QVu+I6^3hSIlt+38=YK3*qrB+zyCDaP*ETdLf z=NxKG@|eJ^u_k9P4{J9fmlK4n-`cLl7^fgAu3Fa}h71 zgAk|CfruB<0f>`nf5b_&AL2xM4&nv0FX9B+2XQ>@jW~{;jX0L}LL5U25l7Pk#3GuH zcs|WT97S^xN75X`^QeY6oca)l(QL$_Gz)PE?TI*;_CP$Bc1IjUGZ67rv?XE|ZGqU6o`KkdHb?AEn;~Y>(-FJT(-6~XQ^c;c31SzjB6g+>{N@7r2>e%i*;^V8PUiQk_|9f+-H3Svu25nE7# zc!s6tr_IU#;OAzRo}Zphe#6hFkzWy;nu>nNAGH05D)|Lbu_XOewj}-3OMb@hCGr!Z zNdAH7A%917kslGA(979YYKOj=_Jt86BnMh8OZxR28e1rIJ@-^ZKauo5O?-7x?|($>)ealD{DyC!Zl6BZm=xAcqjYCx1o!j(m#vE%^lT8}c#Y z*W@F_qvS)xBjh0BSL6WVm*fM)FUb3dpOg0x|3=&oWC!A_XewsXh_!Lw1KS}Px&rgth5g%u={_W%*{Jf29MtqFv`X42CMWF6wI zxXVt5N;G3f2FYq56L%s{dD@`hPj9 z|1Vu+Dy^5H`hO{^|CgZpe=(~6t5N-5h3fxGRR1qR_5VUt|1UuGe+8=l=cD?69;*M# zQT;y`)&G~E`o9d-|8r3NUyADg*{J?6LG}MERR7OJ_5Tc1|4&Et|1?zp7o+i-MJn&qF2>i3wRR6a;%`ATlRR5oW>i_1b{%?lr|I<8Zckb=}%18|0C1&A7{G$V@%io1Jm_?&vgCYFp#GB{U0!0|NBhW{~puzzsq#}e_^`*#sOpOFw({W z;{|wKG!7USVLdbs827|_XdEz>u^t)+jAI@!#`u3any5`6{(rIeP3aSe{~rkP|63vc zUxE1lD(9aae{qaUxttz_!@mFh-UyPA^EBgp&?3+tacB`j#k~_t%1Y-{!QQ{q7ndxW zURn~OG5EpX%ort*iZK=T&0KkcGC&M$?(GXuHm`bK+G5xMvLvl+K5j2Orwlf-EnPHy zUTkq5a4u^P%h9|Yk_MLx(F#@84T)knl>zKFZ zwu_y^HoL9>f@YUjED70_lZT0-`1ZQC-1&BeaUSe<-8qi^u8W2Q3LN`(*F{6D?7#6Z zFop+WtzxdtTEyKp<|Fqn$w^8ad`_Hk@RrYC`!kGTR=icb9^1UEwSB{G((D?Hog7xJ zR%wqyG~CbtLmrwE+l6E4oNT%`TiGKS(vlJnuf-h?e*~i8h9nXVr=5y;cn#v=zXE0k zC&)}J@$gzw;^DQp7NiYry?F+gLwGEz)ZgcnTaJHUQ0?m zycTyn{2^fGoP?M;74h&I#KZp;nCY7!GqJ?OYe|WR*W!+c{}h<%lMpkfA|76Yc=%6% zncfL9lbB#V#;%I>IvDqE@*kW1f9l>wU{|(2N$6oWo}#e^8f&1j1{!Oiu?GIn*T4d! z2!@lJ_Dy6sIb@m1;IK0Lu9o5DW9vs5=fmp>?XjsM_My^fW7z5S{$*#xHc&NTNVm-N zSdH*?Cc7J0`eMuuEq$h?7LTQ7TIR%KnPXb!#$%alTIR)LnP*z&$77jqS{B4(SzuZg z#$#D%TK0;^vX^P;i>nd&tX33XGs1S{i>o2|uqFB8YDzwAOTM@olMh>yFRtd~!}jEh zt3mm&Mfu`tQa)@`zPK8d4_lQlu4d)KcBRGDt~6{{T3qc)v+hrEwJQzVl@?dK(y(1= zakVQA+m#kqyV9^-X>qkH4cnC#SG&@%U1@Q(D-GL~7FWB{uwCWE)vj`|UBzr$G^AU% ztk@pRIQ81inv43u?EgDmyNUZFcgNH=@Sn!N#u{j>fyNqWtbzZE8klU16-eKHea*0* zkS!K^R#r`~s_bd)!!T&s%<_`7K^4{WtI{fFLp)JgW!j+fikWlkGO(TvKj{u*40JYy zJbgHgIj$mntA-)f<>m8BDl5|#Km^Y88D-^VRm;-j8SKu<&W!DO2am|Oo{$c!8h-TW z7^6k9u&sY;>|JiN76P81Drc=?A79_WC<6AH_Pu*q?Bxy)6>yAC)HiYv*Q>G*I)3rJ zC!1ZyxZaW*JMum2i6m<5$j@Lm)_N+cu_ND;C(hCuvuo_gbJta4N1nf3{Wo^xV|+!x So_vqk-i87r?ABYtvHu5#v$in+ diff --git a/sparta/simdb/test/SQLiteDatabase/test_dbs/placeholder.txt b/sparta/simdb/test/SQLiteDatabase/test_dbs/placeholder.txt deleted file mode 100644 index 09e11edc38..0000000000 --- a/sparta/simdb/test/SQLiteDatabase/test_dbs/placeholder.txt +++ /dev/null @@ -1,4 +0,0 @@ -This file serves as a placeholder to keep the 'test_dbs' -subdirectory in SimDB's test/Database folder in git. We -don't want to use things like boost::filesystem in SimDB -to create temp folders. diff --git a/sparta/simdb/test/SharedDB/CMakeLists.txt b/sparta/simdb/test/SharedDB/CMakeLists.txt deleted file mode 100644 index 78b615ae5e..0000000000 --- a/sparta/simdb/test/SharedDB/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -project(SIMDB_SharedDB_test) - -add_executable(SIMDB_SharedDB_test SharedDB_test.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_recursive_copy(SIMDB_SharedDB_test test_dbs) - -simdb_test(SIMDB_SharedDB_test SIMDB_SharedDB_test_RUN) diff --git a/sparta/simdb/test/SharedDB/SharedDB_test.cpp b/sparta/simdb/test/SharedDB/SharedDB_test.cpp deleted file mode 100644 index e86b6cc3c0..0000000000 --- a/sparta/simdb/test/SharedDB/SharedDB_test.cpp +++ /dev/null @@ -1,359 +0,0 @@ -/*! - * \file SharedDB_test.cpp - * \brief This file contains tests which verify database contents - * when more than one database connection / async task queue was - * used to write the data to disk asynchronously. - */ - -#include "simdb/test/SimDBTester.hpp" -#include "simdb/schema/DatabaseRoot.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/MathUtils.hpp" - -#include -#include - -#define DB_DIR "test_dbs" - -using ObjectDatabase = simdb::ObjectManager::ObjectDatabase; - -TEST_INIT; - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -//! Registered schema builder for the Random namespace. -void buildRandNumbersSchema(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addColumn("RandInt", dt::int32_t) - .addColumn("RandFloat", dt::float_t) - .addColumn("RandDouble", dt::double_t); -} - -//! Registered schema builder for the Incrementing namespace. -void buildIncNumbersSchema(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - schema.addTable("Numbers") - .addColumn("IncrementingInt", dt::int32_t) - .addColumn("IncrementingFloat", dt::float_t) - .addColumn("IncrementingDouble", dt::double_t); -} - -//! Factory method DatabaseRoot will invoke when it needs -//! to create ObjectManager's bound to SQLite database files. -simdb::DbConnProxy * createSQLiteProxy() -{ - return new simdb::SQLiteConnProxy; -} - -//! Test data structure used for writing, reading, and -//! verifying record values in a database. -struct TestData { - int32_t ival = 0; - float fval = 0; - double dval = 0; -}; - -struct RandomDataFactory { - TestData makeRandom() { - TestData d; - d.ival = simdb::utils::chooseRand(); - d.fval = simdb::utils::chooseRand(); - d.dval = simdb::utils::chooseRand(); - return d; - } -}; - -//! Test data structure used for writing, reading, and -//! verifying record values in a database. -struct IncrementingDataFactory { - TestData makeRandom() { - TestData d; - - d.ival = curr_ival + (rand() % 100) + 1; - d.ival = curr_fval + ((rand() % 100) * 3.14) + 1; - d.dval = curr_dval + ((rand() % 100) * 75.123) + 1; - - curr_ival = d.ival; - curr_fval = d.fval; - curr_dval = d.dval; - - if (curr_ival < 0 || curr_fval < 0 || curr_dval < 0) { - throw simdb::DBException("Overflow detected"); - } - return d; - } - -private: - int32_t curr_ival = 0; - float curr_fval = 0; - double curr_dval = 0; -}; - -//Helper class which holds onto randomly generated -//data structures for database writes, keeping those -//structures in memory so we can verify the database -//independently. -template -class Answers -{ -public: - explicit Answers(const size_t max_num_structs) : - max_num_structs_(max_num_structs) - { - assert(max_num_structs_ > 0); - data_.reserve(max_num_structs_); - } - - const DataT & createRandomData() { - if (data_.size() < data_.capacity()) { - data_inds_.emplace_back(data_.size()); - data_.emplace_back(data_factory_.makeRandom()); - return data_.back(); - } - data_inds_.emplace_back(rand() % data_.capacity()); - return data_[data_inds_.back()]; - } - - size_t getNumDataStructs() const { - return data_inds_.size(); - } - - const DataT & getDataAtIndex(const size_t idx) const { - return data_.at(data_inds_.at(idx)); - } - -private: - const size_t max_num_structs_; - std::vector data_; - FactoryT data_factory_; - std::vector data_inds_; -}; - -using RandAnswers = Answers; -using IncAnswers = Answers; - -void testObjMgrsSharingSameFile(const size_t num_tasks = 100) -{ - PRINT_ENTER_TEST - - simdb::DatabaseRoot db_root(DB_DIR); - - simdb::DatabaseNamespace * rand_namespace = - db_root.getNamespace("Random"); - EXPECT_TRUE(rand_namespace->hasSchema()); - - simdb::DatabaseNamespace * inc_namespace = - db_root.getNamespace("Incrementing"); - EXPECT_TRUE(inc_namespace->hasSchema()); - - ObjectDatabase * rand_db = - rand_namespace->getDatabase(); - - ObjectDatabase * inc_db = - inc_namespace->getDatabase(); - - //Helper class which takes ObjectDatabase ownership, - //and queues database write requests onto separate - //ObjectManager task queues. These requests should - //end up in the same database file despite using - //multiple ObjectManager's and multiple task queues. - class TaskScheduler { - public: - TaskScheduler(ObjectDatabase * rand_db, - ObjectDatabase * inc_db, - RandAnswers & rand_answers, - IncAnswers & inc_answers) : - rand_db_(rand_db), - inc_db_(inc_db), - rand_answers_(rand_answers), - inc_answers_(inc_answers) - {} - - void scheduleOne() { - const TestData & rand_db_data = rand_answers_.createRandomData(); - const TestData & inc_db_data = inc_answers_.createRandomData(); - - std::unique_ptr rand_db_task( - new RandDbTaskWriter(rand_db_, rand_db_data)); - - std::unique_ptr inc_db_task( - new IncDbTaskWriter(inc_db_, inc_db_data)); - - rand_db_->getObjectManager()->getTaskQueue()-> - addWorkerTask(std::move(rand_db_task)); - - inc_db_->getObjectManager()->getTaskQueue()-> - addWorkerTask(std::move(inc_db_task)); - } - - private: - //Helper class which adds records to the Random namespace - //on the worker thread. - class RandDbTaskWriter : public simdb::WorkerTask { - public: - RandDbTaskWriter(ObjectDatabase * rand_db, - const TestData & rand_db_data) : - rand_db_(rand_db), - rand_db_data_(rand_db_data) - {} - - private: - void completeTask() override { - rand_db_->getTable("Numbers")->createObjectWithArgs( - "RandInt", rand_db_data_.ival, - "RandFloat", rand_db_data_.fval, - "RandDouble", rand_db_data_.dval); - } - - ObjectDatabase * rand_db_ = nullptr; - TestData rand_db_data_; - }; - - //Helper class which adds records to the Incrementing - //namespace on the worker thread. - class IncDbTaskWriter : public simdb::WorkerTask { - public: - IncDbTaskWriter(ObjectDatabase * inc_db, - const TestData & inc_db_data) : - inc_db_(inc_db), - inc_db_data_(inc_db_data) - {} - - private: - void completeTask() override { - inc_db_->getTable("Numbers")->createObjectWithArgs( - "IncrementingInt", inc_db_data_.ival, - "IncrementingFloat", inc_db_data_.fval, - "IncrementingDouble", inc_db_data_.dval); - } - - ObjectDatabase * inc_db_ = nullptr; - TestData inc_db_data_; - }; - - ObjectDatabase * rand_db_ = nullptr; - ObjectDatabase * inc_db_ = nullptr; - RandAnswers & rand_answers_; - IncAnswers & inc_answers_; - }; - - RandAnswers rand_answers(100); - IncAnswers inc_answers(100); - - //Before handing the ObjectDatabase's over to the - //TaskScheduler, ask for their database file names, - //and ask them to create ObjectQuery's we can use - //later to verify the database records' values. - const std::string & rand_db_fname = rand_db->getDatabaseFile(); - - std::unique_ptr rand_db_query = - rand_db->createObjectQueryForTable("Numbers"); - - const std::string & inc_db_fname = inc_db->getDatabaseFile(); - - std::unique_ptr inc_db_query = - inc_db->createObjectQueryForTable("Numbers"); - - auto task_controller = db_root.getTaskController(); - rand_db->getTaskQueue()->addToTaskController(task_controller); - inc_db->getTaskQueue()->addToTaskController(task_controller); - - TaskScheduler scheduler( - std::move(rand_db), std::move(inc_db), - rand_answers, inc_answers); - - for (size_t idx = 0; idx < num_tasks; ++idx) { - scheduler.scheduleOne(); - } - - task_controller->flushQueue(); - task_controller->stopThread(); - - //Start from scratch with new database connections to - //these database files. - simdb::ObjectManager rand_obj_mgr(DB_DIR); - simdb::ObjectManager inc_obj_mgr(DB_DIR); - - EXPECT_TRUE(rand_obj_mgr.connectToExistingDatabase(rand_db_fname)); - EXPECT_TRUE(inc_obj_mgr.connectToExistingDatabase(inc_db_fname)); - - rand_obj_mgr.safeTransaction([&]() { - int32_t rand_ival; - float rand_fval; - double rand_dval; - - rand_db_query->writeResultIterationsTo( - "RandInt", &rand_ival, - "RandFloat", &rand_fval, - "RandDouble", &rand_dval); - - EXPECT_EQUAL(rand_db_query->countMatches(), - rand_answers.getNumDataStructs()); - - auto rand_query_iter = rand_db_query->executeQuery(); - for (size_t idx = 0; idx < rand_answers.getNumDataStructs(); ++idx) { - EXPECT_TRUE(rand_query_iter->getNext()); - const TestData & actual = rand_answers.getDataAtIndex(idx); - EXPECT_EQUAL(actual.ival, rand_ival); - EXPECT_WITHIN_EPSILON(actual.fval, rand_fval); - EXPECT_WITHIN_EPSILON(actual.dval, rand_dval); - } - }); - - inc_obj_mgr.safeTransaction([&]() { - int32_t rand_ival; - float rand_fval; - double rand_dval; - - inc_db_query->writeResultIterationsTo( - "IncrementingInt", &rand_ival, - "IncrementingFloat", &rand_fval, - "IncrementingDouble", &rand_dval); - - EXPECT_EQUAL(inc_db_query->countMatches(), - inc_answers.getNumDataStructs()); - - auto inc_query_iter = inc_db_query->executeQuery(); - for (size_t idx = 0; idx < inc_answers.getNumDataStructs(); ++idx) { - EXPECT_TRUE(inc_query_iter->getNext()); - const TestData & actual = inc_answers.getDataAtIndex(idx); - EXPECT_EQUAL(actual.ival, rand_ival); - EXPECT_WITHIN_EPSILON(actual.fval, rand_fval); - EXPECT_WITHIN_EPSILON(actual.dval, rand_dval); - } - }); -} - -int main(int argc, char ** argv) -{ - REGISTER_SIMDB_NAMESPACE(Random, SQLite); - REGISTER_SIMDB_NAMESPACE(Incrementing, SQLite); - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(SQLite, createSQLiteProxy); - REGISTER_SIMDB_SCHEMA_BUILDER(Random, buildRandNumbersSchema); - REGISTER_SIMDB_SCHEMA_BUILDER(Incrementing, buildIncNumbersSchema); - - if (argc > 1) { - size_t perf_num_tasks = atoi(argv[1]); - testObjMgrsSharingSameFile(perf_num_tasks); - } else { - testObjMgrsSharingSameFile(); - } - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/simdb/test/SharedDB/test_dbs/placeholder.txt b/sparta/simdb/test/SharedDB/test_dbs/placeholder.txt deleted file mode 100644 index 09e11edc38..0000000000 --- a/sparta/simdb/test/SharedDB/test_dbs/placeholder.txt +++ /dev/null @@ -1,4 +0,0 @@ -This file serves as a placeholder to keep the 'test_dbs' -subdirectory in SimDB's test/Database folder in git. We -don't want to use things like boost::filesystem in SimDB -to create temp folders. diff --git a/sparta/simdb/test/SimDBTester.hpp b/sparta/simdb/test/SimDBTester.hpp deleted file mode 100644 index b00187c36e..0000000000 --- a/sparta/simdb/test/SimDBTester.hpp +++ /dev/null @@ -1,862 +0,0 @@ -// -*- C++ -*- - -#pragma once - -/** - * \brief SimDBTester implementation. Utility class and - * associated macros for SimDB unit tests. - */ - -#include "simdb/utils/MathUtils.hpp" -#include "simdb/test/Colors.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace simdb -{ - //! ostream operator so we can do EXPECT_EQUAL(vec1, vec2) - template - inline std::ostream & operator<<(std::ostream & os, - const std::vector & data) - { - if (data.empty()) { - return os; - } - - if (data.size() == 1) { - os << "[" << data[0] << "]"; - return os; - } - - //These vectors can be too long to really print to - //stdout in a useful way... let's just truncate to - //something like "[6.5, 3.4, 5.6, 7.8, 1.2, ...]" - // - //If the vector is less than or equal to 5 elements, - //they will all get printed out. - const size_t num_data_to_print = std::min(5ul, data.size()); - std::ostringstream oss; - oss << "["; - - for (size_t idx = 0; idx < num_data_to_print-1; ++idx) { - oss << data[idx] << ","; - } - oss << data[num_data_to_print-1]; - - if (data.size() > num_data_to_print) { - oss << ",..."; - } - oss << "]"; - os << oss.str(); - - return os; - } - - /** - * \class SimDBTester - * \brief A simple testing class. Used for checking SimDB API - * correctness and reporting error messages as appropriate. - * - * The user of this framework should not have to instantiate this - * class directly. You can if you want to, but you should use the - * TEST_INIT macro. - * - * Example usage: - * - * \code - * TEST_INIT - * - * int main() { - * EXPECT_TRUE(true); - * EXPECT_FALSE(false); - * EXPECT_NOTHROW(int a = 3); - * EXPECT_THROW(throw a); - * EXPECT_EQUAL(2+2, 4); - * EXPECT_NOTEQUAL(2+2, 5) - * EXPECT_NOTEQUAL(p, nullptr) // Given: Thingy* p; - * - * REPORT_ERROR; - * return ERROR_CODE; - * } - * \endcode - */ - class SimDBTester - { - public: - SimDBTester() : SimDBTester(0, std::cerr) - {} - - bool expectAllReached(const uint32_t expected_reached, - const uint32_t line, - const char * file) - { - bool ret = true; - if (methods_reached_.size() != expected_reached) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test failed to execute the " - << expected_reached << " expected methods at least once." << "\n" - "Instead, " << methods_reached_.size() << " were reached." - << std::endl; - - //List the methods that were in fact reached. - cerr_ << "The test only reached the following: " << std::endl; - cerr_ << SIMDB_CURRENT_COLOR_GREEN; - for (const auto & s : methods_reached_) { - cerr_ << "-> " << s << "\n"; - } - - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "' FAILED on line " - << line << " in file " << file << SIMDB_CURRENT_COLOR_NORMAL - << std::endl; - - ++num_errors_; - ret = false; - cerr_ << std::endl; - } - return ret; - } - - bool expect(const bool val, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if (!val) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } - return ret; - } - - // Try and compare bytes and display values on failure instead of characters - bool expectEqual(const uint8_t v1, - const uint8_t v2, - const bool expected, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if ((v1 == v2) != expected) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << ". Value: '" << static_cast(v1); - - if (expected) { - cerr_ << "' should equal '"; - } else { - cerr_ << "' should NOT equal '"; - } - - cerr_ << static_cast(v2) << "'" - << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - - ++num_errors_; - ret = false; - } - return ret; - } - - // Try and compare different types - template - bool expectEqual(const T & v1, - const T & v2, - const bool expected, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if ((v1 == v2) != expected) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << ". Value: '" << v1; - - if (expected) { - cerr_ << "' should equal '"; - } else { - cerr_ << "' should NOT equal '"; - } - - cerr_ << v2 << "'" << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } - return ret; - } - - // Try and compare different types - template - bool expectEqual(const T & v1, - const U & v2, - const bool expected, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if (compare(v1,v2) != expected) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << ". Value: '" << v1; - - if (expected) { - cerr_ << "' should equal '"; - } else { - cerr_ << "' should NOT equal '"; - } - - cerr_ << v2 << "'" << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } - return ret; - } - - // Comparison operation for two integers having different signed-ness - template - typename std::enable_if::value - && std::is_integral::value - && (std::is_signed::value != std::is_signed::value), bool>::type - compare(const T & t, const U & u) { - return (t == static_cast(u)); - } - - // Comparison operation for everything else - template - typename std::enable_if::value - && std::is_integral::value - && (std::is_signed::value != std::is_signed::value)), bool>::type - compare(const T & t, const U & u) { - return (t == u); - } - - // Overload for comparison with nullptr - template - bool expectEqual(const T & v1, - const std::nullptr_t, - const bool expected, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if ((v1 == nullptr) != expected) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << ". Value: '" << v1; - - if (expected) { - cerr_ << "' should equal '"; - } else { - cerr_ << "' should NOT equal '"; - } - - cerr_ << "null" << "'" << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } - return ret; - } - - // Overload for comparison of nullptr with a var - template - bool expectEqual(const std::nullptr_t, - const T & v1, - const bool expected, - const char * test_type, - const uint32_t line, - const char * file) - { - bool ret = true; - if ((nullptr == v1) != expected) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type - << "' FAILED on line " << line << " in file " << file - << ". Value: '" << v1; - - if (expected) { - cerr_ << "' should equal '"; - } else { - cerr_ << "' should NOT equal '"; - } - - cerr_ << "null" << "'" << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } - return ret; - } - - template - typename std::enable_if< - std::is_floating_point::value, - bool>::type - expectEqualWithinTolerance(const T & v1, const T & v2, const T & tol, - const char * test_type, const uint32_t line, - const char * file) { - bool ret = true; - if (tol < 0) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" - << test_type << "' FAILED on line " - << line << " in file " << file - << ". Negative tolerance supplied." - << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - ret = false; - } else { - ret = simdb::utils::approximatelyEqual(v1, v2, tol); - if (!ret) { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Test '" - << test_type << "' FAILED on line " - << line << " in file " << file - << ". Value: '" << v1 << "' should be equal to '" - << v2 << "' within tolerance '" << tol << "'"; - ++num_errors_; - } - } - return ret; - } - - void throwTestFailed(const char * test_type, - const uint32_t line, - const char * file, - const char * exception_what = "") { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "Throw Test Fail:'" << test_type - << "' FAILED on line " << line << " in file " << file << std::endl; - - if (exception_what != 0 && strlen(exception_what) != 0) { - cerr_ << " Exception: " << exception_what << std::endl; - } - cerr_ << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - } - - /*! - * \brief Compares two files - * \param a filename 1 - * \param b filename 2 - * \param expected Expected file equality. TRUE expects files to match. - * \param ignore_commented_lines Do not compare text for lines that begin with '#' - * Pass in FALSE if ou expect there to be differences. - * \post Generates a test error if any of the following conditions are true: - * \li Either file cannot be opened - * \li Files differ in length and exptected=TRUE with the exception of - * lines starting with '#' - * \li Files differ at any position and expected=TRUE with the exception - * of lines starting with '#' - * \li Files are same length and each char is identical and expected=FALSE - * \throw Does not throw - * - * Tracks line and col positions for error printouts. Newlines mode is - * always '\n'. - */ - void expectFilesEqual(const std::string & a, - const std::string & b, - const bool expected, - const uint32_t line, - const char * file, - const bool ignore_commented_lines = true) - { - std::ifstream fa, fb; - std::stringstream err; - try { - fa.open(a, std::ios_base::in); - } catch (std::exception &) { - } - - if (fa.fail()) { - err.str(""); - err << "Could not open file \"" << a << "\""; - fileComparisonFailed(a, b, line, file, err.str()); - } - - try { - fb.open(b, std::ios_base::in); - } catch (std::exception &) { - } - - if (fb.fail()) { - err.str(""); - err << "Could not open file \"" << b << "\""; - fileComparisonFailed(a, b, line, file, err.str()); - } - - if (!fa.fail() && !fb.fail()) { - uint32_t line_num = 0; - uint32_t last_line_pos = 0; - uint64_t pos = 0; - bool was_newline = true; - while (true) { - char cho, chn; - cho = fa.get(); - chn = fb.get(); - - // Ignore lines starting with '#' - if (was_newline && ignore_commented_lines) { - was_newline = false; - if (cho == '#') { - while (true) { - // Read until after newline - cho = fa.get(); - if (cho == '\n') { - cho = fa.get(); - if (cho != '#') { - break; - } - } - ++pos; // Increment pos on this file, but not the other - } - } - if (chn == '#') { - // Read until after newline - while (true) { - chn = fb.get(); - if (chn == '\n') { - chn = fb.get(); - if (chn != '#') { - break; - } - } - } - } - } - - if (!fa.good() || !fb.good()) { - if ((fa.good() != fb.good()) && expected == true) { - std::stringstream msg; - msg << "Files were different lengths: "; - if (!fa.good()) { - msg << a << " was shorted than " << b << " at char '" << chn << "' #" << pos; - } else { - msg << b << " was shorted than " << a << " at char '" << cho << "' #" << pos; - } - fileComparisonFailed(a, b, line, file, msg.str()); - } - break; - } - - if (cho != chn) { - err.str(""); - err << "Files differed at pos " << pos << " (line " - << line_num << ", col " << pos - last_line_pos - << ") with chars: '" << cho << "' != '" << chn << "'"; - if (expected == true) { - fileComparisonFailed(a, b, line, file, err.str().c_str()); - } - return; - } - - ++pos; - if (cho == '\n') { // prev char (pos-1) - ++line_num; - last_line_pos = pos; // Line starts here - was_newline = true; - } - } - - if (expected == false) { - fileComparisonFailed(a, b, line, file, "Files were the same"); - } - } - } - - void fileComparisonFailed(const std::string & a, - const std::string & b, - const uint32_t line, - const char * file, - const std::string & error) - { - cerr_ << SIMDB_CURRENT_COLOR_BRIGHT_RED << "File comparison test between \"" - << a << "\" and \"" << b << "\" FAILED on line " - << line << " in file " << file << std::endl; - cerr_ << " Exception: " << error << std::endl; - cerr_ << SIMDB_CURRENT_COLOR_NORMAL << std::endl; - ++num_errors_; - } - - void reachedMethod(const std::string & method_title) - { - methods_reached_.insert(method_title); - } - - static SimDBTester * getInstance() { - static SimDBTester inst; - return & inst; - } - - static std::unique_ptr makeTesterWithUserCError(std::ostream & cerr) { - return std::unique_ptr(new SimDBTester(0, cerr)); - } - - static uint32_t getErrorCode(const SimDBTester * tester = getInstance()) { - return tester->num_errors_; - } - - static SimDBTester * inst; - - private: - SimDBTester(const uint32_t num_errors, - std::ostream & cerr) : - num_errors_(num_errors), - methods_reached_(), - cerr_(cerr) - {} - - uint32_t num_errors_; - std::set methods_reached_; - std::ostream & cerr_; - }; - -/** - * \def TEST_INIT - * \brief Initializes the test. Should be placed OUTSIDE of a - * code block SOMEWHERE in the test source - */ -#define TEST_INIT simdb::SimDBTester * simdb::SimDBTester::inst = 0 - -/** - * \def EXPECT_REACHED() - * \brief Add this method to be checked against whether or not it was called - * at least once. This macro can be placed anywhere inside the function expected - * to be called. - */ -#define EXPECT_REACHED() simdb::SimDBTester::getInstance()->reachedMethod(__FUNCTION__) - -/** - * \def ENSURE_ALL_REACHED(x) - * \brief make sure that the same number of methods were reached as were expected by this test. - * \param x the number of unique methods that you expect to be reached at least once. - * x should equal the number of times EXPECT_REACHED() is placed throughout the test. - */ -#define ENSURE_ALL_REACHED(x) simdb::SimDBTester::getInstance()->expectAllReached(x, __LINE__, __FUNCTION__) - -/** - * \def EXPECT_TRUE(x) - * \brief Determine if the block \a x evaluates to true - * - * Example usage: - * \code - * if (EXPECT_TRUE(true) ) { - * std::cout << "Wadda know, it was true!" << std::endl; - * } else { - * std::cerr << "True isn't true anymore?!? What's this world coming to?!" << std::endl; - * } - * \endcode - */ -#define EXPECT_TRUE(x) simdb::SimDBTester::getInstance()->expect((x), #x, __LINE__, __FILE__) - -/** - * \def EXPECT_EQUAL(x,y) - * \brief Determine if the block \a x is equal (using operator= on x) to y - * \note Types must match. Comparison is made at run-time. - * \pre x must be printable to cout/cerr using the insertion operator - * - * Value of x and y, if not equal, is printed in error message. - * - * Example usage: - * \code - * if(EXPECT_EQUAL((const char*)x, "12345")) { - * std::cout << "Wadda know, it they were the same!" << std::endl; - * } else { - * std::cerr << "Values differed" << std::endl; - * } - * \endcode - */ -#define EXPECT_EQUAL(x, y) simdb::SimDBTester::getInstance()->expectEqual((x), (y), true, #x, __LINE__, __FILE__) - -/** - * \def EXPECT_NOTEQUAL(x,y) - * \brief Determine if the block \a x is not equal (using operator= on x) to y - * \note Types must match exactly. Comparison is made at run-time. - * \pre x must be printable to cout/cerr using the insertion operator - * - * Value of x and y, if equal, is printed in error message. - * - * Example usage: - * \code - * if (EXPECT_NOTEQUAL((const char*)x, "12345")) { - * std::cout << "Wadda know, they were different" << std::endl; - * } else { - * std::cerr << "Values were same" << std::endl; - * } - * \endcode - */ -#define EXPECT_NOTEQUAL(x, y) simdb::SimDBTester::getInstance()->expectEqual((x), (y), false, #x, __LINE__, __FILE__) - -/** - * \def EXPECT_WITHIN_TOLERANCE(x,y,tol) - * \brief Determine if the block \a x is equal (using operator= on x) to y - * within a specified tolerance - * \note Types must match exactly. Comparison is made at run-time. - * \pre x must be printable to cout/cerr using the insertion operator - * - * If x and y differ by more than (>) the specified tolerance, the x and y - * values and the tolerance value will be printed in an error message. - * - * Example usage: - * \code - * if (EXPECT_WITHIN_TOLERANCE(x, y, 0.02)) { - * std::cout << "The x and y values were within 0.02 of each other" << std::endl; - * } else { - * std::cerr << "The x and y values differed by more than 0.02" << std::endl; - * } - * \endcode - */ -#define EXPECT_WITHIN_TOLERANCE(x, y, tol) simdb::SimDBTester::getInstance()-> \ - expectEqualWithinTolerance((x), (y), (tol), #x, __LINE__, __FILE__) - -/** - * \def EXPECT_WITHIN_EPSILON(x,y) - * \brief Determine if the block \a x is equal (using operator= on x) to y - * within 2x the machine epsilon, as determined by std::numeric_limits::epsilon() - * \note Types must match exactly. Comparison is made at run-time. - * \note Types compared cannot be integral. - * \pre x must be printable to cout/cerr using the insertion operator - * - * If x and y differ by strictly more than (>) the specified tolerance, the x and y - * values and the tolerance value will be printed in an error message. - * - * Example usage: - * \code - * if (EXPECT_WITHIN_EPSILON(x, y)) { - * std::cout << "The x and y values were within machine epsilon of each other" << std::endl; - * } else { - * std::cerr << "The x and y values differed by more than machine epsilon" << std::endl; - * } - * \endcode - */ -#define EXPECT_WITHIN_EPSILON(x, y) simdb::SimDBTester::getInstance()-> \ - expectEqualWithinTolerance((x), (y), \ - (std::numeric_limits::epsilon()), #x, __LINE__, __FILE__) - -/** - * \def EXPECT_FALSE(x) - * \brief Determine if the block \a x evaluates to false - * - * Example usage: - * \code - * if (EXPECT_FALSE(false) == false) { - * std::cout << "Wadda know, false IS false!" << std::endl; - * } else { - * std::cerr << "False isn't false anymore?!? What's this world coming to?!" << std::endl; - * } - * \endcode - */ -#define EXPECT_FALSE(x) simdb::SimDBTester::getInstance()->expect(!(x), #x, __LINE__, __FILE__) - -/** - * \def EXPECT_THROW(x) - * \brief Determine if the block \a x correctly throws an exception - * - * Example usage: - * \code - * EXPECT_THROW(throw 10); - * \endcode - */ -#define EXPECT_THROW(x) { \ - bool did_it_throw = false; \ - try {x;} \ - catch (...) \ - { did_it_throw = true; } \ - if (did_it_throw == false) { \ - simdb::SimDBTester::getInstance()->throwTestFailed(#x, __LINE__, __FILE__); \ - } \ - } - -/** - * \def EXPECT_THROW_MSG_SHORT(x, expected_msg) - * \brief Determine if the block \a x correctly throws an - * exception with the given message (no line number/file) - * - * Example usage: - * \code - * // The thrower: - * void foo() { - * throw simdb::SimDBException("Hello"); - * } - * - * // The message: - * // ex.what() -> "Hello: in file: 'blah.cpp', on line: 123" - * - * EXPECT_THROW_MSG_SHORT(ex.what(), "Hello"); - * \endcode - */ - #define EXPECT_THROW_MSG_SHORT(x, expected_msg) { \ - bool did_it_throw = false; \ - try { x; } \ - catch(simdb::SimDBException & ex) \ - { did_it_throw = true; \ - if (strcmp(expected_msg, ex.rawReason().c_str()) != 0) { \ - std::cerr << "Expected msg: " << expected_msg << std::endl; \ - std::cerr << "Actual msg: " << ex.what() << std::endl; \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, ex.what()); \ - } \ - } \ - if (did_it_throw == false) { \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, "did not throw"); \ - } \ - } - -/** - * \def EXPECT_THROW_MSG_LONG(x, expected_msg) - * \brief Determine if the block \a x correctly throws an - * exception with the given message on the file and line number - * - * Example usage: - * \code - * // The thrower: - * void foo() { - * throw simdb::SimDBException("Hello"); - * } - * - * // The message: - * // ex.what() -> "Hello: in file: 'blah.cpp', on line: 123" - * - * EXPECT_THROW_MSG_LONG(ex.what(), "Hello: in file: 'blah.cpp', on line: 123"); - * \endcode - */ -#define EXPECT_THROW_MSG_LONG(x, expected_msg) { \ - bool did_it_throw = false; \ - try { x; } \ - catch (simdb::SimDBException & ex) \ - { did_it_throw = true; \ - if (strcmp(expected_msg, ex.what()) != 0) { \ - std::cerr << "Expected msg: " << expected_msg << std::endl; \ - std::cerr << "Actual msg: " << ex.what() << std::endl; \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, ex.what()); \ - } \ - } \ - if (did_it_throw == false) { \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, "did not throw"); \ - } \ - } - -#define EXPECT_THROW_MSG_CONTAINS(x, expected_msg) { \ - bool did_it_throw = false; \ - try { x; } \ - catch (simdb::SimDBException & ex) \ - { did_it_throw = true; \ - if (std::string(ex.what()).find(expected_msg) != std::string::npos) { \ - std::cerr << "Expected msg: " << expected_msg << std::endl; \ - std::cerr << "Actual msg: " << ex.what() << std::endl; \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, ex.what()); \ - } \ - } \ - if(did_it_throw == false) { \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, "did not throw"); \ - } \ - } - -/** - * \def EXPECT_NOTHROW(x) - * \brief Determine if the block \a x throws an exception incorrectly - * - * Example usage: - * \code - * EXPECT_NOTHROW(throw 10); - * \endcode - */ -#define EXPECT_NOTHROW(x) { \ - bool did_it_throw = false; \ - std::string exception_what; \ - try { x; } \ - catch (std::exception & e) \ - { did_it_throw = true; \ - exception_what = e.what(); \ - } \ - catch (...) \ - { did_it_throw = true; } \ - if (did_it_throw == true) { \ - simdb::SimDBTester::getInstance()-> \ - throwTestFailed(#x, __LINE__, __FILE__, exception_what.c_str()); \ - } \ - } - -/** - * \def EXPECT_FILES_EQUAL(a, b) - * \brief Determine if the block \a a and \a b contain the same exact data - * with the exception of lines beginning with '#' - * - * Example usage: - * \code - * EXPECT_FILES_EQUAL("a.out.golden", "b.out"); - * \endcode - */ -#define EXPECT_FILES_EQUAL(a, b) { \ - simdb::SimDBTester::getInstance()-> \ - expectFilesEqual(a, b, true, __LINE__, __FILE__); \ - } - -/** - * \def EXPECT_FILES_NOTEQUAL(a, b) - * \brief Determine if the block \a a and \a b contain different data - * with the exception of lines beginning with '#' - * - * Example usage: - * \code - * EXPECT_FILES_NOTEQUAL("a.out.golden", "b.out"); - * \endcode - */ -#define EXPECT_FILES_NOTEQUAL(a, b) { \ - simdb::SimDBTester::getInstance()-> \ - expectFilesEqual(a, b, false, __LINE__, __FILE__); \ - } - -/** - * \def ERROR_CODE - * \brief The number of errors found in the testing - */ -#define ERROR_CODE simdb::SimDBTester::getErrorCode() - - /** - * \def REPORT_ERROR - * \brief Prints the error code with a nice pretty message - * \note This is separate from returning ERROR_CODE, which must be done - * after this macro. See the example for simdb::SimDBTester . The reason for - * this separation is so that errors can be reported before teardown, which - * could fail uncatchably (i.e. segfault) when there are caught errors - * earlier in the test (e.g. dangling pointers). - * - * Example: - * \code - * int main() } - * // ... - * REPORT_ERROR; - * - * performDangerousCleanup(); - * - * return ERROR_CODE; - * } - * \endcode - */ -#define REPORT_ERROR \ - if (ERROR_CODE != 0) { \ - std::cout << std::dec << "\n" << SIMDB_UNMANAGED_COLOR_BRIGHT_RED \ - << ERROR_CODE << "ERROR(S) found during test.\n" \ - << SIMDB_UNMANAGED_COLOR_NORMAL << std::endl; \ - } else { \ - std::cout << std::dec << "\n" \ - << "TESTS PASSED -- No errors found during test.\n" \ - << std::endl; \ - } -} - diff --git a/sparta/simdb/test/Thread/CMakeLists.txt b/sparta/simdb/test/Thread/CMakeLists.txt deleted file mode 100644 index 36c5353821..0000000000 --- a/sparta/simdb/test/Thread/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -project(SIMDB_Thread_test) - -add_executable(SIMDB_Thread_test Thread_test.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_test(SIMDB_Thread_test SIMDB_Thread_test_RUN) - -add_subdirectory(StandaloneCpp1) diff --git a/sparta/simdb/test/Thread/StandaloneCpp1/CMakeLists.txt b/sparta/simdb/test/Thread/StandaloneCpp1/CMakeLists.txt deleted file mode 100644 index 1bf00eb3cc..0000000000 --- a/sparta/simdb/test/Thread/StandaloneCpp1/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -project(SIMDB_StandaloneThread_test) - -add_executable(SIMDB_StandaloneThread_test1 StandaloneThread_test1.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_test(SIMDB_StandaloneThread_test1 SIMDB_StandaloneThread_test1_RUN) diff --git a/sparta/simdb/test/Thread/StandaloneCpp1/StandaloneThread_test1.cpp b/sparta/simdb/test/Thread/StandaloneCpp1/StandaloneThread_test1.cpp deleted file mode 100644 index 6d07c8138b..0000000000 --- a/sparta/simdb/test/Thread/StandaloneCpp1/StandaloneThread_test1.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/*! - * \file StandaloneThread_test1.cpp - * \brief Negative test for SimDB's AsyncTaskEval class. - * - * This test runs in its own dedicated CPP main() since it verifies - * exceptions based on static counter values. Running this negative - * test together in the same unit test program as other tests would - * likely cause unexpected behavior. - */ - -#include "simdb/test/SimDBTester.hpp" - -#include "simdb/async/AsyncTaskEval.hpp" - -TEST_INIT; - -using simdb::AsyncTaskEval; - -//! Helper task which does nothing when called up on -//! the worker thread. Used for exception testing. -class NoOpTask : public simdb::WorkerTask -{ -public: - ~NoOpTask() = default; -private: - void completeTask() override final { - } -}; - -void testWorkerThreadUsageExceptions() -{ - simdb::TimerThread::enableStressTesting(); - std::vector> task_threads; - EXPECT_EQUAL(AsyncTaskEval::getCurrentNumTaskThreadsCreated(), 0); - - //Start by maxing out all of the task threads that we *are* allowed to create. - size_t task_thread_count = 0; - while (task_thread_count < AsyncTaskEval::getMaxTaskThreadsAllowed()) { - EXPECT_NOTHROW(task_threads.emplace_back(new AsyncTaskEval)); - EXPECT_NOTHROW( - task_threads.back()->addWorkerTask( - std::unique_ptr(new NoOpTask)) - ); - ++task_thread_count; - } - - //Now try to make just one more... this should throw. - // (not yet, but soon) - EXPECT_NOTHROW(task_threads.emplace_back(new AsyncTaskEval)); - - //If we attempt to put any bit of work on this last - //worker thread, it should throw. The reason why it - //throws in the call to addWorkerTask() and not from - //its constructor is that the actual worker thread - //is not instantiated until the first task is placed - //in the work queue via addWorkerTask() - EXPECT_THROW( - task_threads.back()->addWorkerTask( - std::unique_ptr(new NoOpTask)) - ); - - task_threads.clear(); - simdb::TimerThread::disableStressTesting(); -} - -int main() -{ - testWorkerThreadUsageExceptions(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/simdb/test/Thread/Thread_test.cpp b/sparta/simdb/test/Thread/Thread_test.cpp deleted file mode 100644 index ff3187f344..0000000000 --- a/sparta/simdb/test/Thread/Thread_test.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/*! - * \file Thread_test.cpp - * \brief Test for SimDB threading utilities - */ - -#include "simdb/test/SimDBTester.hpp" - -#include "simdb/async/TimerThread.hpp" -#include "simdb/async/ConcurrentQueue.hpp" -#include "simdb/async/AsyncTaskEval.hpp" - -#include -#include -#include -#include - -TEST_INIT; - -using simdb::TimerThread; -using simdb::AsyncTaskEval; -using simdb::ConcurrentQueue; - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -/*! - * \brief Simple counter that asynchronously increments - * an integer at fixed intervals. Note that these methods - * do not use a mutex since the data accessed on the main - * thread and the timer thread is just a size_t and thus - * is inherently thread safe. - */ -class TimedCounter -{ -public: - explicit TimedCounter(const double interval_seconds) : - timed_eval_(interval_seconds, this) - {} - - size_t getCount() const { - return count_; - } - - void start() { - timed_eval_.start(); - } - - void stop() { - timed_eval_.stop(); - } - -private: - void execute_() { - ++count_; - } - - //! TimerThread implementation. Calls back into the - //! TimedCounter::execute_() method at regular intervals - //! on a background thread. - class TimedEval : public TimerThread - { - public: - TimedEval(const double interval_seconds, - TimedCounter * timed_counter) : - TimerThread(TimerThread::Interval::FIXED_RATE, - interval_seconds), - timed_counter_(timed_counter) - {} - - private: - void execute_() override { - timed_counter_->execute_(); - } - - TimedCounter *const timed_counter_; - }; - - size_t count_ = 0; - TimedEval timed_eval_; - friend class TimedEval; -}; - -//! Test basic functionality of the TimerThread class -void testTimerThreadBasic() -{ - PRINT_ENTER_TEST - - //Set up a simple counter that increments every 250ms - TimedCounter counter(0.250); - const size_t expected_count = 10; - size_t current_count = 0; - size_t last_printed_current_count = 0; - - //Flag to help protect this test from running forever - //in the event of a bug in the TimerThread code - bool forced_exit = false; - - //Start the timer and wait until it reaches the expected count - current_count = counter.getCount(); - EXPECT_EQUAL(current_count, 0); - counter.start(); - while (current_count < expected_count) { - if (forced_exit) { - break; - } - if (current_count != last_printed_current_count) { - std::cout << "Current count is " << current_count << "\n"; - last_printed_current_count = current_count; - } - current_count = counter.getCount(); - } - - //Cap the loop above to a few hundred seconds. In case it - //goes haywire, at least the unit test will be killed in - //a reasonable amount of time. - std::thread forced_exit_thread([&]() { - size_t sleep_count = 0; - while (current_count < expected_count) { - std::this_thread::sleep_for(std::chrono::seconds(2)); - ++sleep_count; - if (sleep_count > 100) { - forced_exit = true; - break; - } - } - }); - - counter.stop(); - forced_exit_thread.join(); - EXPECT_FALSE(forced_exit); -} - -//! Single-producer, single-consumer ConcurrentQueue test -void testConcurrentQueue() -{ - PRINT_ENTER_TEST - - ConcurrentQueue queue; - const size_t data_num_elements = 1000000; - - std::vector recovered_data; - bool keep_consuming = true; - - EXPECT_EQUAL(queue.size(), 0); - - //Let's start the consumer thread first to give the 'sleep_for' - //call a better chance of getting hit - std::thread consumer([&queue, &recovered_data, &keep_consuming]() { - size_t item = 0; - //Enter an infinite loop. We only break out of this when - //we retrieve all the elements from the ConcurrentQueue, - //or we are forced to stop (because we're actually stuck - //in an infinite loop - the test will still fail, but we'll - //break out of this while loop) - while (keep_consuming) { - if (queue.try_pop(item)) { - recovered_data.emplace_back(item); - } else { - //Back off a little bit to give the producer a - //chance to write some more data into the queue. - //This reduces contention and mimics what we would - //want to do in production code, as opposed to - //spinning over the try_pop() method. - std::this_thread::sleep_for(std::chrono::milliseconds(25)); - } - } - - //The producer is done writing data into the queue, - //but that doesn't mean the queue is empty at this - //time. We could have been asleep in the while loop - //above when a few more items went into the queue, - //for instance. Flush the queue if needed to get all - //the data. - while (queue.try_pop(item)) { - recovered_data.emplace_back(item); - } - }); - - //Randomly create some test data - std::vector test_data; - test_data.reserve(data_num_elements); - for (size_t idx = 0; idx < data_num_elements; ++idx) { - test_data.emplace_back(rand()); - } - - //Start putting those random data values into the queue - std::thread producer([&queue, &test_data](){ - for (size_t idx = 0; idx < data_num_elements; ++idx) { - queue.push(test_data[idx]); - } - }); - - //Go until the source values are all spent, and sent - //into the queue - producer.join(); - - //Flip the switch that tells the consumer it can break - //out of its infinite loop. It will greedily get any - //leftover data out of the queue if there is any. - keep_consuming = false; - - //Wait until the consumer thread is done, and then - //check all the recovered data values against the - //source values that were originally sent into the - //queue. - consumer.join(); - - EXPECT_EQUAL(test_data, recovered_data); - EXPECT_EQUAL(queue.size(), 0); - - //Make sure the emplace() method is doing the right thing - typedef std::tuple CustomerInfo; - ConcurrentQueue customers; - customers.emplace("Bob", "Thompson", 41); - customers.emplace("Alice", "Smith", 29); - - CustomerInfo customer1, customer2; - EXPECT_TRUE(customers.try_pop(customer1)); - EXPECT_TRUE(customers.try_pop(customer2)); - - EXPECT_EQUAL(std::get<0>(customer1), std::string("Bob")); - EXPECT_EQUAL(std::get<1>(customer1), std::string("Thompson")); - EXPECT_EQUAL(std::get<2>(customer1), 41); - - EXPECT_EQUAL(std::get<0>(customer2), std::string("Alice")); - EXPECT_EQUAL(std::get<1>(customer2), std::string("Smith")); - EXPECT_EQUAL(std::get<2>(customer2), 29); -} - -int main() -{ - srand(time(0)); - - testTimerThreadBasic(); - testConcurrentQueue(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/simdb/test/Utils/CMakeLists.txt b/sparta/simdb/test/Utils/CMakeLists.txt deleted file mode 100644 index ceccdeabba..0000000000 --- a/sparta/simdb/test/Utils/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -project(SIMDB_Utils_test) - -add_executable(SIMDB_Utils_test Utils_test.cpp) - -include(${SPARTA_CMAKE_MACRO_PATH}/SimdbTestingMacros.cmake) - -simdb_test(SIMDB_Utils_test SIMDB_Utils_test_RUN) diff --git a/sparta/simdb/test/Utils/Utils_test.cpp b/sparta/simdb/test/Utils/Utils_test.cpp deleted file mode 100644 index 11a7c5fc78..0000000000 --- a/sparta/simdb/test/Utils/Utils_test.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/*! - * \file Utils_test.cpp - * - * \brief Tests for various utility classes in SimDB. - */ - -#include "simdb/test/SimDBTester.hpp" - -//Core database headers -#include "simdb/utils/StringUtils.hpp" - -//Standard headers -#include - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -void testTransformedString() -{ - PRINT_ENTER_TEST - - simdb::utils::lowercase_string lower("HeLlOWoRlD"); - - //Quick check for the size() and empty() methods - const std::string lower_cppstring(lower.getString()); - EXPECT_EQUAL(lower.size(), lower_cppstring.size()); - EXPECT_FALSE(lower.empty()); - - //Test equality with const char* - const char * lower_expected1_cstring = "helloworld"; - EXPECT_EQUAL(lower, lower_expected1_cstring); - - //Append const char* - lower += "_HELLOAGAIN"; - - //Test equality with std::string - const std::string lower_expected2_cppstring = "helloworld_helloagain"; - EXPECT_EQUAL(lower, lower_expected2_cppstring); - - //Append std::string - lower += std::string("_GoodBye"); - - //Test equality with std::string - EXPECT_EQUAL(lower, "helloworld_helloagain_goodbye"); - - //Append char - lower += '!'; - - //Test equality with std::string - EXPECT_EQUAL(lower, "helloworld_helloagain_goodbye!"); - - //Invoke copy constructor from lowercase to uppercase - simdb::utils::UPPERCASE_STRING upper(lower); - EXPECT_EQUAL(upper, "HELLOWORLD_HELLOAGAIN_GOODBYE!"); - - //Final check for clear() and empty() methods - upper.clear(); - EXPECT_TRUE(upper.empty()); -} - -int main() -{ - testTransformedString(); -} diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index 9a274f19bf..414f51d38f 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -16,130 +16,44 @@ #include "sparta/trigger/Triggerable.hpp" #include "sparta/pevents/PeventTrigger.hpp" #include "sparta/pevents/PeventController.hpp" -#include "sparta/collection/PipelineCollector.hpp" #include "sparta/log/Tap.hpp" +#include "sparta/collection/PipelineCollector.hpp" namespace sparta { namespace app { /*! * \class PipelineTrigger - * \brief Trigger used to enable/disable Pipeline colletion + * \brief Trigger used to enable/disable Pipeline collection */ class PipelineTrigger : public trigger::Triggerable { public: - PipelineTrigger(const std::string& pipeline_collection_path, - const std::set& pipeline_enabled_node_names, - uint64_t pipeline_heartbeat, - bool multiple_triggers, - sparta::Clock * clk, sparta::RootTreeNode * rtn) : - pipeline_collection_path_(pipeline_collection_path), - pipeline_enabled_node_names_(pipeline_enabled_node_names), - pipeline_heartbeat_(pipeline_heartbeat), - multiple_triggers_(multiple_triggers), - clk_(clk), - root_(rtn) + PipelineTrigger(const std::string& simdb_filename, + const std::set& enabled_nodes, + const size_t heartbeat, + sparta::RootTreeNode * rtn, + const sparta::CounterBase * insts_retired_counter) { - pipeline_collector_. - reset(new sparta::collection::PipelineCollector(multiple_triggers_ ? getCollectionPath_() : pipeline_collection_path_, - pipeline_heartbeat_, - clk_, - root_)); - + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, enabled_nodes, heartbeat, rtn, insts_retired_counter)); } void go() override { sparta_assert(!triggered_, "Why has pipeline trigger been triggered?"); triggered_ = true; - std::cout << "Pipeline collection started, output to files with prefix '" - << pipeline_collector_->getFilePath() << "'" << std::endl; - startCollection_(); - - if(multiple_triggers_) { - std::cout << "#" << num_collections_ << " pipeline collection started" << std::endl; - } + pipeline_collector_->startCollecting(); } void stop() override { sparta_assert(triggered_, "Why stop an inactivated trigger?"); triggered_ = false; - stopCollection_(); - - if(multiple_triggers_) { - std::cout << "#" << num_collections_ << " pipeline collection ended" << std::endl; - ++num_collections_; - pipeline_collector_->reactivate(getCollectionPath_()); - } + pipeline_collector_->stopCollecting(); } private: - void startCollection_() - { - if(pipeline_enabled_node_names_.empty()) { - // Start collection at the root node - pipeline_collector_->startCollection(root_); - } - else { - // Find the nodes in the root and enable them - for(const auto & node_name : pipeline_enabled_node_names_) { - std::vector results; - root_->getSearchScope()->findChildren(node_name, results); - if(results.size() == 0) { - std::cerr << SPARTA_CURRENT_COLOR_RED - << "WARNING (Pipeline collection): Could not find node named: '" - << node_name - <<"' Collection will not occur on that node!" - << SPARTA_CURRENT_COLOR_NORMAL - << std::endl; - } - for(auto & tn : results) { - std::cout << "Collection enabled on node: '" << tn->getLocation() << "'" << std::endl; - pipeline_collector_->startCollection(tn); - } - } - } - } - - void stopCollection_() - { - if(pipeline_enabled_node_names_.empty()) { - // Start collection at the root node - pipeline_collector_->stopCollection(root_); - } - else { - // Find the nodes in the root and enable them - for(const auto & node_name : pipeline_enabled_node_names_) { - std::vector results; - root_->getSearchScope()->findChildren(node_name, results); - for(auto & tn : results) { - pipeline_collector_->stopCollection(tn); - } - } - } - pipeline_collector_->destroy(); - } - - std::string getCollectionPath_() const - { - if(pipeline_collection_path_.back() == '/') { - return pipeline_collection_path_ + std::to_string(num_collections_) + '_'; - } - else { - return pipeline_collection_path_ + '_' + std::to_string(num_collections_) + '_'; - } - } - std::unique_ptr pipeline_collector_; - const std::string pipeline_collection_path_; - const std::set pipeline_enabled_node_names_; - const uint64_t pipeline_heartbeat_; - const bool multiple_triggers_; - sparta::Clock * clk_ = nullptr; - sparta::RootTreeNode * root_ = nullptr; - uint32_t num_collections_ = 0; }; /*! diff --git a/sparta/sparta/app/CommandLineSimulator.hpp b/sparta/sparta/app/CommandLineSimulator.hpp index 6a78f8eabd..c38057aaa5 100644 --- a/sparta/sparta/app/CommandLineSimulator.hpp +++ b/sparta/sparta/app/CommandLineSimulator.hpp @@ -570,18 +570,6 @@ class CommandLineSimulator */ MultiDetailOptions report_opts_; - /*! - * \brief Builtin sparta command line options for SimDB - */ - MultiDetailOptions simdb_opts_; - - /*! - * \brief Builtin sparta command line options for SimDB - * (internal / developer use only - not visible to - * command line help printout) - */ - MultiDetailOptions simdb_internal_opts_; - /*! * \brief Application-specific options * \see getApplicationOptions diff --git a/sparta/sparta/app/ReportDescriptor.hpp b/sparta/sparta/app/ReportDescriptor.hpp index 7010252c70..77f107526b 100644 --- a/sparta/sparta/app/ReportDescriptor.hpp +++ b/sparta/sparta/app/ReportDescriptor.hpp @@ -29,16 +29,10 @@ #include "sparta/utils/Utils.hpp" #include "sparta/report/format/BaseFormatter.hpp" #include "sparta/app/FeatureConfiguration.hpp" -#include "simdb_fwd.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/utils/SpartaException.hpp" #include "sparta/utils/ValidValue.hpp" -namespace simdb { - class AsyncTaskEval; - class ObjectManager; -} // namespace simdb - namespace sparta::app { class SimulationConfiguration; } // namespace sparta::app @@ -66,14 +60,6 @@ namespace sparta { class StreamNode; } - namespace async { - class AsyncTimeseriesReport; - class AsyncNonTimeseriesReport; - } - namespace db { - class ReportHeader; - } - namespace app { class Simulation; @@ -202,37 +188,6 @@ namespace sparta { */ std::string orig_dest_file_; - /*! - * \brief Timeseries object which sends this report's metadata - * and SI data values to a database. - * - * Use of this database is featured off by default for now. - */ - std::shared_ptr db_timeseries_; - - /*! - * \brief Wrapper which writes non-timeseries SI data values - * into a database. - * - * Use of this database is featured off by default for now. - */ - std::shared_ptr db_non_timeseries_; - - /*! - * \brief The "simdb" feature has a number of modes it can run: - * - * 1. SI compression disabled, row-major SI ordering - * 2. SI compression disabled, column-major SI ordering - * 3. SI compression enabled, row-major SI ordering - * 4. SI compression enabled, column-major SI ordering - * - * If the feature is enabled, feature options may be given - * at the command line. These are optional however, so we - * will default to compressed, row-major SI blobs in the - * absence of any such options. - */ - const FeatureConfiguration::FeatureOptions * simdb_feature_opts_ = nullptr; - friend class ReportDescriptorCollection; public: @@ -294,76 +249,6 @@ namespace sparta { return enabled_; } - /*! - * \brief Check if this descriptor holds only one report - * instantiation, and that it is a timeseries report (.csv) - */ - bool isSingleTimeseriesReport() const; - - /*! - * \brief Check if this descriptor holds only one report - * instantiation, and that it is *not* a timeseries report. - * For example, .html, .json, .txt, and so on. - */ - bool isSingleNonTimeseriesReport() const; - - /*! - * \brief Switch this descriptor's timeseries report generation - * from synchronous .csv generation to asynchronous database - * persistence. - * - * The task queue object passed in is the worker thread object, - * which is shared among all report descriptors in the simulation. - * - * The simulation database passed in is the object with the - * actual connection to the physical database. This object - * is shared with the Simulation class and other descriptors. - * - * The Scheduler passed in is the one our simulation is running on, - * and the Clock passed in is the simulation's root clock. Both - * of these objects are used in order to get the "current time" - * value at each report update (current cycle, simulated time, - * etc.) - */ - void configureAsyncTimeseriesReport( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db, - const Clock & root_clk); - - /*! - * \brief Switch this descriptor's report generation from - * synchronous to asynchronous database persistence. This - * method is intended only for descriptors that have just - * one *non-timeseries* report format. - * - * Call "isSingleNonTimeseriesReport()" first before calling - * this method to be sure, otherwise this method may throw. - */ - void configureAsyncNonTimeseriesReport( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db); - - /*! - * \brief Give access to the database timeseries header. This - * will return null when this descriptor is used for any non- - * timeseries report format (json, json_reduced, txt, etc.) - * or when the "simdb" feature has been disabled. - */ - db::ReportHeader * getTimeseriesDatabaseHeader(); - - /*! - * \brief Do any post-simulation post processing steps needed. - * This is typically used for final wrap-up this descriptor - * needs to do in the simulation database, so the two inputs - * are the SimDB and the AsyncTaskEval objects that belong - * to the app::Simulation and sparta::ReportRepository objects, - * but other non-database work may need to be completed post- - * simulation as well. - */ - void doPostProcessing( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db); - /*! * \brief Provide access to the formatters we have been using * so they can coordinate with the reporting infrastructure to @@ -441,9 +326,8 @@ namespace sparta { return format; } - //! \brief When SimDB has automatic report verification enabled, - //! this descriptor may have had its dest_file changed when the - //! Simulation::setupReports() method was called. The report file + //! \brief This descriptor may have had its dest_file changed when the + //! the Simulation::setupReports() method was called. The report file //! will still end up in the dest_file that you gave the descriptor, //! but this getter is added if you need to ask this descriptor //! what its immutable dest_file was from the beginning. diff --git a/sparta/sparta/app/Simulation.hpp b/sparta/sparta/app/Simulation.hpp index fab13b933c..f058afe37f 100644 --- a/sparta/sparta/app/Simulation.hpp +++ b/sparta/sparta/app/Simulation.hpp @@ -111,43 +111,6 @@ class Simulation feature_config_ = feature_config; } - /*! - * \brief Get the database root for this simulation. - * \return Pointer to the DatabaseRoot - * - * This is a container that holds all databases the simulation is - * using. The underlying ObjectManager methods such as getTable() - * and findObject() can be accessed indirectly using the - * ObjectDatabase class (nested class inside ObjectManager). For - * example, say that we ran a simulation using the --report - * command line option, and we want to go through the DatabaseRoot - * to get the StatisticInstance / reports database records: - * - * \code - * simdb::DatabaseRoot * db_root = sim->getDatabaseRoot(); - * - * simdb::DatabaseNamespace * stats_namespace = db_root-> - * getNamespace("Stats"); - * - * simdb::ObjectManager::ObjectDatabase * stats_db = - * stats_namespace->getDatabase(); - * \endcode - * - * Once you have the ObjectDatabase for the desired namespace, - * access the table wrappers like so: - * - * \code - * std::unique_ptr ts_table = - * stats_db->getTable("Timeseries"); - * \endcode - * - * See "simdb/include/simdb/schema/DatabaseRoot.hpp" and - * "simdb/include/simdb/ObjectManager.hpp" for more info - * about using these other classes to read and write - * database records in a SimDB namespace. - */ - simdb::DatabaseRoot * getDatabaseRoot() const; - /*! * \brief There is a 1-to-1 mapping between a running simulation * and the database it is using. Some components in the simulator @@ -345,14 +308,6 @@ class Simulation */ void saveReports(); - /*! - * \brief After CommandLineSimulator runs the simulation and calls - * all post-processing APIs, this method will be called. It is the - * only thing left before our destructor gets called. Do any last- - * minute final wrap up work if needed in this method. - */ - void postProcessingLastCall(); - /*! * \brief Get a counter by its semantic (if such a counter exists). * \param sem Semantic of counter to return @@ -728,14 +683,6 @@ class Simulation */ void setupReports_(); - /*! - * \brief Right before the main sim loop, this method is called in order - * to create any SimDB triggers the simulation was configured to use. - * These triggers dictate when database namespace(s) are opened and - * closed for reads and writes via the simdb::TableProxy class. - */ - void setupDatabaseTriggers_(); - /*! * \brief In the case where a comma-separated list of file formats was * used to specify the output file format (i.e. 'csv, csv_cumulative'), @@ -1007,47 +954,6 @@ class Simulation *\brief Run controller interface */ std::unique_ptr rc_; - - /*! - * \brief This database holds things like report - * metadata and StatisticInstance values. - * - * \note This is not publicly accessible. Think of - * this pointer as a "shortcut" to the SI / reports - * namespace in the simulation database; it is used - * so often in Simulation.cpp, and passed around to - * other classes related to reports / post-processing - * that we hang onto is as a convenience. - */ - simdb::ObjectManager * stats_db_ = nullptr; - - /*! - * \brief Database root for this simulation. This is a container - * holding all database connections currently in use. The reports / - * SI database tables & records are just a subset of what can be - * accessed via the DatabaseRoot. Other databases may be available - * as well (pipeline collection DB, branch prediction DB, etc.) - */ - std::unique_ptr db_root_; - - /*! - * \brief Accessor who knows which simulation components are - * enabled for SimDB access, and which are not. - */ - std::shared_ptr sim_db_accessor_; -private: - //! At the very end of the simulation's configure() method, - //! take a first look at the feature values we were given. - void inspectFeatureValues_(); - - //! Tests may enable a post-simulation report validation - //! step, where contents of the database (SI values, metadata, - //! etc.) are compared against a baseline report. - bool isReportValidationEnabled_() const; - - //! List of report filenames which *failed* post-simulation - //! report verification. - std::set report_verif_failed_fnames_; }; } // namespace app diff --git a/sparta/sparta/app/SimulationConfiguration.hpp b/sparta/sparta/app/SimulationConfiguration.hpp index 39e9be4b60..5c70998a50 100644 --- a/sparta/sparta/app/SimulationConfiguration.hpp +++ b/sparta/sparta/app/SimulationConfiguration.hpp @@ -338,155 +338,6 @@ class SimulationConfiguration } } - /*! - * Set the location of the SimDB file produced during simulation - */ - void setSimulationDatabaseLocation(const std::string & loc) { - simdb_location_ = loc; - } - - /*! - * Add a YAML options file specifying which components are allowed - * to access the simulation database, and (optionally) when that - * access is to be granted or removed during simulation. - * - * \code - * stats: - * components: - * top.cpu.core0.rob - * root.clocks - * bpred: - * start: notif.dbaccess == 1 - * stop: notif.dbaccess == 0 - * \endcode - * - * In the above example, this would mean that for the "stats" - * database namespace, components "top.cpu.core0.rob" and - * "root.clocks" are the only ones that can access the database, - * and they can access it at any time. - * - * For the "bpred" namespace however, all components / device - * tree locations can access the database when the notification - * channel "dbaccess" has emitted a value of 1, but they are - * restricted from accessing the database when the same channel - * emits a value of 0. - * - * This YAML file could also list specific components that can - * access the database, as well as start/stop triggers for finer - * control over when the database is available for those components. - * For example: - * - * \code - * bpred: - * components: - * top.cpu.core0.rob - * start: notif.dbaccess == 1 - * stop: notif.dbaccess == 0 || notif.earlyterm == 1 - * \endcode - * - * The syntax for the "start" and "stop" database triggers - * is the same as report triggers found in YAML descriptor - * files (--report ) - * - * \note The expression parser for these YAML files does not - * support trigger tags. The following example YAML would - * fail to parse: - * - * \code - * stats: - * start: top.core0.rob.stats.total_number_retired >= 1000 - * tag: t0 - * bpred: - * stop: t0.start - * \endcode - */ - void addSimulationDatabaseAccessOptsYaml( - const std::string & opts_file) - { - simdb_enabled_components_opts_files_.emplace_back(opts_file); - } - - /*! - * Set the root directory where all of this simulation's legacy - * reports should be copied to. Say the root directory was set - * to "/tmp" and the simulation was configured to produce the - * following report files: - * - * Filename Format - * ------------- ----------- - * foo.csv csv_cumulative - * foo.json json_reduced - * bar.json json_reduced - * baz.json json_detail - * - * If the SimDB file ended up being "abcd-1234.db", then the - * simulation would *copy* (not move) the four legacy reports - * listed above into a directory structure that looks like: - * - * /tmp - * /abcd-1234 - * /csv_cumulative - * foo.csv - * /json_reduced - * foo.json - * bar.json - * /json_detail - * baz.json - * - * \param reports_root_dir Root directory where the reports - * will be copied. Directory will be created if needed. - * - * \param collected_formats Specific report formats you want - * to collect. If left empty, all formats will be collected. - */ - void setLegacyReportsCopyDir(const std::string & reports_root_dir, - const std::set & collected_formats = {}) - { - simdb_legacy_reports_copy_dir_ = reports_root_dir; - for (const auto & fmt : collected_formats) { - const utils::lowercase_string lower_fmt = fmt; - simdb_legacy_reports_collected_formats_.insert(lower_fmt.getString()); - } - } - - /*! - * Get the simulation database location configured for this - * simulation. This will return an empty string if a non-default - * location was never set at the command line. - */ - const std::string & getSimulationDatabaseLocation() const { - return simdb_location_; - } - - /*! - * Get the list of SimDB access YAML files that specify which - * components have been granted access to the simulation database, - * and (optionally) when they that access is to be enabled / disabled - * using trigger expressions. - */ - const std::vector & getDatabaseAccessOptsFiles() const { - return simdb_enabled_components_opts_files_; - } - - /*! - * Get the root directory where all of this simulation's legacy - * reports were copied to. - * \note Intended for internal / developer use. - */ - const std::string & getLegacyReportsCopyDir() const { - return simdb_legacy_reports_copy_dir_; - } - - /*! - * Get the specific legacy report formats that are being collected. - * If empty, either all formats are being collected, or the report - * collection command line option is not being used. - * \note Intended for internal / developer use. - */ - const std::set & getLegacyReportsCollectedFormats() const { - return simdb_legacy_reports_collected_formats_; - } - /*! * \brief Controls installation of signal handlers. */ @@ -749,29 +600,6 @@ class SimulationConfiguration // //////////////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////////////////////////////////// - // Simulation database - - //! Location of the SimDB file produced during simulation - std::string simdb_location_; - - //! SimDB access options are given in YAML files containing - //! component locations, such as "top.cpu.core0.rob" or - //! "root.clocks", which can also specify trigger definitions - //! that say when a component is granted access to the database. - std::vector simdb_enabled_components_opts_files_; - - //! Root directory where legacy reports generated from this - //! simulation are copied to. For internal / developer use. - std::string simdb_legacy_reports_copy_dir_; - - //! Specific formats of the legacy reports we are collecting, - //! if any. For internal / developer use. - std::set simdb_legacy_reports_collected_formats_; - - // - //////////////////////////////////////////////////////////////////////////////// }; } // namespace app diff --git a/sparta/sparta/app/SimulationInfo.hpp b/sparta/sparta/app/SimulationInfo.hpp index 74992a14a8..8e1316d942 100644 --- a/sparta/sparta/app/SimulationInfo.hpp +++ b/sparta/sparta/app/SimulationInfo.hpp @@ -30,9 +30,6 @@ #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/SpartaException.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" -namespace simdb { class ObjectManager; } - namespace sparta { @@ -117,20 +114,6 @@ class SimulationInfo std::string sparta_version_; //!< See sparta_version std::vector other_; //!< See other - //! Variables which are populated from SimDB tables: - utils::ValidValue db_elapsed_time_; - - //! The singleton SimulationInfo is the "real" object, but we - //! need to temporarily "swap in" other SimulationInfo objects - //! that were recreated from a SimDB. We'll use a stack of these - //! 'this' pointers to achieve this. - //! - //! The reason we do this is so we don't have to make - //! changes in all the report formatters that are already - //! calling SimulationInfo::getInstance() for their report - //! metadata. - static std::stack sim_inst_stack_; - static SimulationInfo sim_inst_; //!< Static simulation information public: @@ -189,20 +172,13 @@ class SimulationInfo * \brief Gets the SimulationInfo singleton instance */ static SimulationInfo& getInstance() { - if (sim_inst_stack_.empty()) { - return sim_inst_; - } - return *sim_inst_stack_.top(); + return sim_inst_; } /*! * \brief Destruction */ - ~SimulationInfo() { - if (!sim_inst_stack_.empty()) { - sim_inst_stack_.pop(); - } - } + ~SimulationInfo() = default; /*! * \brief Default Constructor @@ -307,20 +283,6 @@ class SimulationInfo } } - /*! - * \brief Recreate a SimulationInfo object from the - * provided SimInfo record with the given ObjMgrID. - * - * \note While this SimDB-created SimulationInfo object - * is in scope, calls to SimulationInfo::getInstance() - * will *not* return the singleton. It will return - * the object that you recreated using this "SimDB - * constructor". It is not recommended to use this - * constructor during an actual SPARTA simulation! - */ - SimulationInfo(const simdb::ObjectManager & sim_db, - const simdb::DatabaseID obj_mgr_db_id, - const simdb::DatabaseID report_node_id = 0); /*! * \brief Instantiate a SimulationInfo object from a json, json_reduced, * json_detail, or js_json report file. @@ -335,9 +297,7 @@ class SimulationInfo /*! * \brief Get the SPARTA version string for this SimulationInfo object. * Most of the time, this will be SimulationInfo::sparta_version (const / - * global). But there are some SimDB/report workflows that need to - * create or recreate SimulationInfo objects with a different SPARTA - * version string. + * global). */ std::string getSpartaVersion() const { if (sparta_version_.empty()) { @@ -453,20 +413,9 @@ class SimulationInfo result.emplace_back("Repro", reproduction_info); result.emplace_back("Start", start_time); - //SimDB-recreated SimulationInfo objects have their - //elapsed time values stored in the database directly. - if (db_elapsed_time_.isValid()) { - result.emplace_back("Elapsed", db_elapsed_time_.getValue()); - } - - //Normal use of the SimulationInfo::getInstance() singleton, - //which is used during live SPARTA simulations, computes the - //elapsed time value via the TimeManager. - else { - std::stringstream timestr; - timestr << TimeManager::getTimeManager().getSecondsElapsed() << 's'; - result.emplace_back("Elapsed", timestr.str()); - } + std::stringstream timestr; + timestr << TimeManager::getTimeManager().getSecondsElapsed() << 's'; + result.emplace_back("Elapsed", timestr.str()); last_captured_elapsed_time_ = result.back().second; return result; diff --git a/sparta/sparta/async/AsyncNonTimeseriesReport.hpp b/sparta/sparta/async/AsyncNonTimeseriesReport.hpp deleted file mode 100644 index e50d3ffe68..0000000000 --- a/sparta/sparta/async/AsyncNonTimeseriesReport.hpp +++ /dev/null @@ -1,247 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include - -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/ObjectManager.hpp" -#include "sparta/report/db/SingleUpdateReport.hpp" -#include "sparta/report/db/ReportNodeHierarchy.hpp" -#include "sparta/app/FeatureConfiguration.hpp" -#include "sparta/statistics/dispatch/archives/ReportStatisticsAggregator.hpp" -#include "sparta/statistics/StatisticInstance.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" -#include "sparta/report/Report.hpp" -#include "sparta/utils/SpartaAssert.hpp" - -namespace sparta { -namespace async { - -/*! - * \brief This class is used to serialize all non-timeseries - * report formats to a SimDB. It writes all SI values on a - * background thread, typically on the same thread as other - * database-related tasks. - */ -class AsyncNonTimeseriesReport -{ -public: - //! Construct with a shared worker thread / task queue, - //! a shared SimDB object, and the sparta::Report you wish - //! to write to this database. - AsyncNonTimeseriesReport( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db, - const Report & report, - const app::FeatureConfiguration::FeatureOptions * simdb_opts) : - task_queue_(task_queue), - sim_db_(sim_db), - report_(report) - { - if (simdb_opts) { - using_compression_ = simdb_opts->getOptionValue( - "compression", "enabled") == "enabled"; - } - } - - //! Write a stringized piece of metadata for this report - //! in the database. - void setStringMetadataByNameAndValue( - const std::string & name, - const std::string & value) - { - using simdb::constraints; - - //If we already have our report DB node ID, write the metadata now. - if (root_report_node_id_ > 0) { - auto meta_tbl = sim_db_->getTable("RootReportNodeMetadata"); - if (meta_tbl->updateRowValues("Value", value). - forRecordsWhere( - "ReportNodeID", constraints::equal, root_report_node_id_, - "Name", constraints::equal, name)) { - } else { - //The metadata overwrite failed, which means that this - //piece of metadata was never written to begin with. - //Create a new record for it. - meta_tbl->createObjectWithArgs( - "ReportNodeID", root_report_node_id_, - "Name", name, - "Value", value); - } - } - - //If we do not have our report DB node ID yet, save this - //metadata value for later. We'll write it when we find out - //our report DB ID. - else { - string_metadata_[name] = value; - } - } - - //! Capture our report's current SI values and write them to - //! the database on a background thread. Since this class is - //! meant to serialize non-timeseries reports (i.e. single- - //! update reports), you typically would only call this method - //! once. But you can call it as many times as you want for - //! any reason, and the SI values will just be overwritten - //! in the database with each call. - void writeCurrentValues() - { - //In order to match what the legacy formatters write into the - //reports (json, text, etc.) we will serialize all metadata - //for this report now, at the same time we write the actual - //SI values to the database. - serializeReportMetadata_(); - - const auto & all_stat_insts = si_aggregator_->getAggregatedSIs(); - for (const StatisticInstance * si : all_stat_insts) { - //These SI's are connected to their fixed spot in - //the aggregator's values vector. We just have to - //make the getValue() call. - si->getValue(); - } - - const std::vector & si_values = si_aggregator_->readFromSource(); - queueStatInstValuesOnWorker_(si_values); - } - - //! Get the root-level report node's database ID. This will - //! equal 0 (unset) until writeCurrentValues() is called, - //! which occurs at the end of simulation during the call - //! to Simulation::saveReports() - simdb::DatabaseID getRootReportNodeDatabaseID() const { - return root_report_node_id_; - } - - //! Get the SimDB object we are using. This is the same database - //! that we share with the app::Simulation. - simdb::ObjectManager * getSimulationDatabase() { - return sim_db_; - } - -private: - //! SI values writer which is invoked on a background thread. - class StatInstValuesWriter : public simdb::WorkerTask - { - public: - //! This object is only used to forward an SI values vector - //! along to the *actual* SingleUpdateReport object which - //! does the DB write. Construct with the writer object - //! and the SI values you want queued on the background - //! thread. - StatInstValuesWriter(std::shared_ptr & si_writer, - const std::vector & si_values, - const bool using_compression) : - si_values_writer_(si_writer), - si_values_(si_values), - using_compression_(using_compression) - {} - - private: - //WorkerTask implementation. Called on a background thread. - void completeTask() override; - - //Wrapper around the database record who holds our SI values. - std::shared_ptr si_values_writer_; - - //Aggregated / contiguous SI values flattened into one - //vector of doubles - const std::vector si_values_; - - //Compression is enabled by default, but can be explicitly - //disabled if desired - const bool using_compression_; - }; - - //! Put a deep copy of the incoming SI values onto the - //! background task thread to be written to the database - //! shortly. - void queueStatInstValuesOnWorker_( - const std::vector & values) - { - if (values.empty()) { - return; - } - - std::unique_ptr async_writer( - new StatInstValuesWriter(si_values_writer_, values, using_compression_)); - - if (task_queue_) { - task_queue_->addWorkerTask(std::move(async_writer)); - } else { - async_writer->completeTask(); - } - } - - //Write out the physical hierarchy of this report, including all - //subreports and all SI's, and all their metadata as well. - void serializeReportMetadata_() - { - if (root_report_node_id_ > 0) { - return; - } - - sim_db_->safeTransaction([&]() { - statistics::ReportNodeHierarchy serializer(&report_); - root_report_node_id_ = serializer.serializeHierarchy(*sim_db_); - sparta_assert(root_report_node_id_ > 0); - - si_values_writer_.reset( - new db::SingleUpdateReport(*sim_db_, root_report_node_id_)); - - si_aggregator_.reset(new statistics::ReportStatisticsAggregator(report_)); - si_aggregator_->initialize(); - - serializer.serializeReportNodeMetadata(*sim_db_); - serializer.serializeReportStyles(*sim_db_); - - for (const auto & meta : string_metadata_) { - const auto & name = meta.first; - const auto & val = meta.second; - serializer.setMetadataCommonToAllNodes(name, val, *sim_db_); - } - string_metadata_.clear(); - }); - } - - //! Shared worker thread object. We will give DB writes - //! to this task queue which will take care of them on a - //! background thread. - simdb::AsyncTaskEval * task_queue_ = nullptr; - - //! Shared database which holds all SI values. This object - //! is shared with the app::Simulation and maybe others. - simdb::ObjectManager * sim_db_ = nullptr; - - //! SI values are aggregated into one vector with - //! the help of this object. - std::unique_ptr si_aggregator_; - - //! SimDB wrapper around the tables that are used for - //! serializing single-update, non-timeseries report - //! formats. - std::shared_ptr si_values_writer_; - - //! Report from which we collect all SI values and header info - const Report & report_; - - //! ID of the root-level report node in the database - simdb::DatabaseID root_report_node_id_ = 0; - - //! Name-value pairs of metadata to be written to the database - std::map string_metadata_; - - //! This report writer supports compressed and uncompressed values - bool using_compression_ = true; -}; - -} // namespace async -} // namespace sparta - diff --git a/sparta/sparta/async/AsyncTimeseriesReport.hpp b/sparta/sparta/async/AsyncTimeseriesReport.hpp deleted file mode 100644 index 7f2fd37ab0..0000000000 --- a/sparta/sparta/async/AsyncTimeseriesReport.hpp +++ /dev/null @@ -1,743 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/statistics/db/SINodeHierarchy.hpp" -#include "sparta/statistics/dispatch/archives/ReportStatisticsAggregator.hpp" -#include "sparta/statistics/db/SIValuesBuffer.hpp" -#include "sparta/statistics/StatisticInstance.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" -#include "sparta/simulation/Clock.hpp" -#include "sparta/report/Report.hpp" -#include "sparta/app/FeatureConfiguration.hpp" -#include "sparta/report/db/Schema.hpp" -#include "sparta/utils/ValidValue.hpp" - -namespace simdb { -class ObjectManager; -} // namespace simdb - -namespace sparta::db { -class ReportHeader; -} // namespace sparta::db - -namespace sparta { -class StatisticInstance; - -namespace async { - -/*! - * \brief Use this class when you want to stream all of a report's - * StatisticInstance values (and optionally any header metadata) - * to a database. All database writes will be committed off the - * main thread. - * - * The shared AsyncTaskEval object given to the constructor is the - * one who creates and owns the worker thread. You can create just - * one worker thread, and share it among as many timeseries objects - * as you need, by doing something like this: - * - * \code - * std::shared_ptr report_thread(new ...) - * - * std::unique_ptr async_report1(new - * AsyncTimeseriesReport(report_thread, report1)); - * - * std::unique_ptr async_report2(new - * AsyncTimeseriesReport(report_thread, report2)); - * \endcode - */ -class AsyncTimeseriesReport : public simdb::Notifiable -{ -public: - //! Construct a timeseries report object with: - //! - task_queue: A background / worker thread running on a timer. - //! Used so we can push expensive DB writes off the main thread. - //! - sim_db: A shared database object, typically (primarily) owned by the - //! app::Simulation, and shared with its reports / descriptors. - //! - root_clk: The app::Simulation's root clock. - //! - report: The sparta::Report that goes with this database timeseries. - AsyncTimeseriesReport(simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db, - const Clock & root_clk, - const Report & report, - const app::FeatureConfiguration::FeatureOptions * feature_opts) : - task_queue_(task_queue), - sim_db_(sim_db), - db_timeseries_(new db::ReportTimeseries(*sim_db_)), - si_aggregator_(new statistics::ReportStatisticsAggregator(report)), - report_(report), - root_clk_(root_clk) - { - //Get the aggregator to flatten out all the leaf SI's, and - //give us that SI vector. - si_aggregator_->initialize(); - const auto & all_stat_insts = si_aggregator_->getAggregatedSIs(); - - //Figure out the compression and row/column-major ordering - const bool compress = feature_opts->getOptionValue( - "compression", "enabled") == "enabled"; - const bool row_major = feature_opts->getOptionValue( - "si-ordering", "row-major") == "row-major"; - - //We currently only support two modes: all of the SI's are - //compressed, or none of the SI's are compressed. More work - //needs to be done to design something that is flexible enough - //to put report update SI values back together when some of - //those values were compressed, and some were not. - // - //Note that this means all of the supportsCompression() methods - //throughout the stats-related classes are rendered meaningless - //until we support "half compressed, half uncompressed" report - //updates. (**We may decide against it entirely and just use - //compression for all SI's or no compression at all**) - std::vector compression_enabled_sis; - std::vector compression_disabled_sis; - for (const auto si : all_stat_insts) { - if (compress) { - //TODO: Incorporate calls to SI::supportsCompression() - compression_enabled_sis.emplace_back(si); - } else { - compression_disabled_sis.emplace_back(si); - } - } - - if (compress) { - //We'll use an SI value/blob buffer of size 1 for all of - //the SI's that said "no, I do not support compression". - //These will be pushed to the task queue for async writes - //to the SI blob table during each report update. - if (!compression_disabled_sis.empty()) { - uncompressed_si_buffer_.reset(new statistics::SIValuesBuffer( - compression_disabled_sis, - root_clk_)); - - if (row_major) { - uncompressed_si_buffer_->useRowMajorOrdering(); - } else { - uncompressed_si_buffer_->useColumnMajorOrdering(); - } - } - - //All SI's that support compression will be buffered into - //a larger SI values buffer, and after enough report updates - //have hit to fill up this larger buffer, those SI's will be - //sent to the task queue for async compression / async writes - //to the SI blob table. - if (!compression_enabled_sis.empty()) { - compressed_si_buffer_.reset(new statistics::SIValuesBuffer( - compression_enabled_sis, - root_clk_)); - - if (row_major) { - compressed_si_buffer_->useRowMajorOrdering(); - } else { - compressed_si_buffer_->useColumnMajorOrdering(); - } - - //Let's aim for a 1MB cap on the amount of memory that an - //*entire* SI blob will take up after it has been inflated. - //Start with 1MB available: - int64_t num_available_bytes_for_compression_buffers = (1 << 20); - - //Subtract away the bytes needed for *uncompressed* SI's... - num_available_bytes_for_compression_buffers -= - (compression_disabled_sis.size() * sizeof(double)); - - //Just in case we have more than 1MB of SI's in one report update... - if (num_available_bytes_for_compression_buffers < 0) { - num_available_bytes_for_compression_buffers = 0; - } - - //And calculate the number of report updates that can be - //buffered into the other SIValuesBuffer for compression. - const size_t initial_num_compression_buffers = - num_available_bytes_for_compression_buffers / - (compression_enabled_sis.size() * sizeof(double)); - - compressed_si_buffer_->initializeNumSIBuffers( - std::max(initial_num_compression_buffers, 1ul)); - - //Since we're using compression, initialize our running - //tally on the number of bytes pre- and post-compression. - total_num_bytes_sent_for_compression_ = 0; - total_num_bytes_after_compression_ = 0; - } - } - - else { - //Put all stats into the uncompressed buffer. We don't have - //enough SI's that are good candidates for compression. - uncompressed_si_buffer_.reset(new statistics::SIValuesBuffer( - all_stat_insts, - root_clk_)); - - if (row_major) { - uncompressed_si_buffer_->useRowMajorOrdering(); - } else { - uncompressed_si_buffer_->useColumnMajorOrdering(); - } - } - - //Uncompressed SI buffers just use one internal buffer. In - //other words, they don't buffer anything, and we'll send - //these SI values to the task thread on each update. - if (uncompressed_si_buffer_ != nullptr) { - //If the feature options specify "none" for "si-ordering", that means - //that we are not supposed to use in-memory buffers at all. Every report - //update will result in its own timeseries chunk/blob record. - if (feature_opts->getOptionValue("si-ordering", "none") == "none") { - uncompressed_si_buffer_->initializeNumSIBuffers(1); - } else { - //Let's aim for 1MB SI chunks - double target_num_bytes = (1 << 20); - target_num_bytes /= (compression_disabled_sis.size() * sizeof(double)); - - const size_t num_si_buffers = std::max((size_t)target_num_bytes, (size_t)1); - uncompressed_si_buffer_->initializeNumSIBuffers(num_si_buffers); - } - } - - //Register ourselves for notifications that the task queue - //is about to be flushed - task_queue_->registerForPreFlushNotifications(*this); - - //One-time population of the entire SI node hierarchy for - //this timeseries. - statistics::SINodeHierarchy serializer(*db_timeseries_, report_); - root_report_node_id_ = serializer.serializeHierarchy(*sim_db_); - } - - //! Get the root-level report node's database ID. - simdb::DatabaseID getRootReportNodeDatabaseID() const { - return root_report_node_id_; - } - - //! Destruction. Print out a message saying how much compression - //! we achieved if we have been sending compressed SI blobs to - //! the database. - ~AsyncTimeseriesReport() { - if (compressed_si_buffer_ != nullptr) { - //Note that we can now safely access our compression variables - //without acquiring the mutex since there's nothing waiting in - //the task queue that will call back into us. - const uint64_t total_num_bytes = total_num_bytes_sent_for_compression_; - const uint64_t compressed_num_bytes = total_num_bytes_after_compression_; - - const double si_compression_pct = - 1 - (static_cast(compressed_num_bytes) / - static_cast(total_num_bytes)); - - // Display as a percentage with one decimal point (89.7%) - const double pretty_print_pct = (floor(si_compression_pct * 10) / 10) * 100.0; - - std::cout << " [si-compression] Compressed SI blobs ended up being " - << pretty_print_pct << "% smaller than the raw SI values." << std::endl; - } else { - std::cout << " [si-compression] We did not perform compression " - "of SI values for this report." << std::endl; - } - } - - //! Get the sparta::Report this timeseries writer is bound to. - const Report & getReport() const { - return report_; - } - - //! Get the database header object for this timeseries writer. - //! You can use this to write (or overwrite) report metadata. - db::ReportHeader & getTimeseriesHeader() { - return db_timeseries_->getHeader(); - } - - //! Get a list of all the SI's locations in this timeseries report. - //! This is equivalent to the first row of SI information in the - //! CSV file (dest_file: out.csv) which looks something like this: - //! - //! "scheduler.ticks, scheduler.seconds, top.core0.rob.ipc, ..." - const std::vector & getStatInstLocations() const { - return si_aggregator_->getStatInstLocations(); - } - - //! Grab all current StatisticInstance values in this report - //! and queue them up in the background thread to be written - //! to disk. - void writeCurrentValues() { - if (uncompressed_si_buffer_ != nullptr) { - uncompressed_si_buffer_->bufferCurrentSIValues(); - if (uncompressed_si_buffer_->buffersAreFilled()) { - const std::vector & values = - uncompressed_si_buffer_->getBufferedSIValues(); - - uint64_t starting_picoseconds = 0; - uint64_t ending_picoseconds = 0; - uint64_t starting_cycles = 0; - uint64_t ending_cycles = 0; - - uncompressed_si_buffer_->getBeginningAndEndingTimestampsForBufferedSIs( - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Add these values to the task queue for asynchronous processing - queueStatInstValuesOnWorker_( - values, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Reset the buffer for uncompressed SI's during each report - //update. Pass in FALSE so the buffer is not initialized with - //nan's (unnecessary performance hit to do so). - uncompressed_si_buffer_->resetSIBuffers(false); - } - } - - //If we have any compression-enabled SI's, buffer their - //values right now. - if (compressed_si_buffer_ != nullptr) { - compressed_si_buffer_->bufferCurrentSIValues(); - - //If the buffer is full, give a deep copy of the raw SI - //values to a compressor object for async processing - //on the task thread. - if (compressed_si_buffer_->buffersAreFilled()) { - const std::vector & uncompressed_values = - compressed_si_buffer_->getBufferedSIValues(); - - uint64_t starting_picoseconds = 0; - uint64_t ending_picoseconds = 0; - uint64_t starting_cycles = 0; - uint64_t ending_cycles = 0; - - compressed_si_buffer_->getBeginningAndEndingTimestampsForBufferedSIs( - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - queueCompressionEnabledStatInstValuesOnWorker_( - uncompressed_values, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Reset the buffer internals. Pass in FALSE so that - //it knows not to initialize the SI values to nan. - compressed_si_buffer_->resetSIBuffers(false); - } - } - } - -private: - //! This class makes a deep copy of SI data values on the - //! main thread, and is added to the background thread / - //! worker queue for async DB writes. - class StatInstValuesWriter : public simdb::WorkerTask - { - public: - StatInstValuesWriter(const std::shared_ptr & db_timeseries, - const std::vector & values, - const db::MajorOrdering major_ordering, - const uint64_t starting_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t starting_cycles, - const uint64_t ending_cycles) : - db_timeseries_(db_timeseries), - si_values_(values), - major_ordering_(major_ordering), - starting_picoseconds_(starting_picoseconds), - ending_picoseconds_(ending_picoseconds), - starting_cycles_(starting_cycles), - ending_cycles_(ending_cycles) - {} - - private: - //WorkerTask implementation. Called on a background thread. - void completeTask() override { - db_timeseries_->writeStatisticInstValuesInTimeRange( - starting_picoseconds_, - ending_picoseconds_, - starting_cycles_, - ending_cycles_, - si_values_, - major_ordering_); - } - - //Timeseries database object. This will persist all of - //the report header / metadata and SI raw values that - //we give it in a database. - std::shared_ptr db_timeseries_; - - //Aggregated / contiguous SI values flattened into one - //vector of doubles - const std::vector si_values_; - - //Row-major or column-major ordering of SI values - const db::MajorOrdering major_ordering_; - - //Timestamps for the blob we are writing to the DB - utils::ValidValue starting_picoseconds_; - utils::ValidValue ending_picoseconds_; - utils::ValidValue starting_cycles_; - utils::ValidValue ending_cycles_; - }; - - //! This class makes a deep copy of SI data values on the - //! main thread, and is added to the background thread / - //! worker queue for async DB writes. - //! - //! The raw SI values are compressed before writing them - //! to the database. - class CompressedStatInstValuesWriter : public simdb::WorkerTask - { - public: - CompressedStatInstValuesWriter( - const std::shared_ptr & db_timeseries, - const std::vector & values, - const db::MajorOrdering major_ordering, - const uint64_t starting_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t starting_cycles, - const uint64_t ending_cycles) : - db_timeseries_(db_timeseries), - si_values_(values), - major_ordering_(major_ordering), - starting_picoseconds_(starting_picoseconds), - ending_picoseconds_(ending_picoseconds), - starting_cycles_(starting_cycles), - ending_cycles_(ending_cycles) - {} - - //! Assign a callback to be called once the compression - //! is complete. This will tell you the raw (uncompressed) - //! number of bytes that went into the compression library, - //! and the number of bytes that resulted after compression. - //! - //! Note that this callback will be called *before* the - //! compressed data is physically written to the database. - typedef uint32_t RawNumBytes; - typedef uint32_t CompressedNumBytes; - typedef std::function CompressionCallback; - - void setPostCompressionCallback(CompressionCallback cb) { - post_compression_callback_ = cb; - } - - private: - //WorkerTask implementation. Called on a background thread. - void completeTask() override; - - //Timeseries database object. This will persist all of - //the report header / metadata and SI raw values that - //we give it in a database. - std::shared_ptr db_timeseries_; - - //Aggregated / contiguous SI values flattened into one - //vector of doubles. This is given to us in *row-major* - //format. If this is the equivalent CSV: - // - // SI1 SI2 SI3 SI4 - // --- --- ---- --- - // 1.3 78 4000 3.4 - // 1.5 79 4007 3.4 - // - //Then this vector will initially be given to us as: - // - // [1.3, 1.5, 78, 79, 4000, 4007, 3.4, 3.4] - // - //We will run this vector as-is through zlib to get - //the compressed bytes that will then be written to - //the database. - const std::vector si_values_; - - //Row-major or column-major ordering of SI values - const db::MajorOrdering major_ordering_; - - //Time values for our SI blob(s). - const uint64_t starting_picoseconds_; - const uint64_t ending_picoseconds_; - const uint64_t starting_cycles_; - const uint64_t ending_cycles_; - - //Optional user callback that will be invoked after - //compressing their data, letting them know how many - //raw bytes went into the compression library, and - //how many compressed bytes came out of it. - utils::ValidValue post_compression_callback_; - }; - - //! Package up the current SI blob values, and add a new - //! worker task to the background thread's processing queue. - void queueStatInstValuesOnWorker_( - const std::vector & values, - const uint64_t starting_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t starting_cycles, - const uint64_t ending_cycles) - { - if (values.empty()) { - return; - } - - const db::MajorOrdering major_ordering = - uncompressed_si_buffer_->getMajorOrdering(); - - std::unique_ptr async_writer( - new StatInstValuesWriter( - db_timeseries_, - values, - major_ordering, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles)); - - task_queue_->addWorkerTask(std::move(async_writer)); - } - - //! Package up the current SI blob values for asynchronous - //! compression. Put this potentially expensive task on the - //! worker thread. - void queueCompressionEnabledStatInstValuesOnWorker_( - const std::vector & values, - const uint64_t starting_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t starting_cycles, - const uint64_t ending_cycles) - { - if (values.empty()) { - return; - } - - const db::MajorOrdering major_ordering = - compressed_si_buffer_->getMajorOrdering(); - - std::unique_ptr async_compressor( - new CompressedStatInstValuesWriter( - db_timeseries_, - values, - major_ordering, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles)); - - async_compressor->setPostCompressionCallback([&] - (const uint32_t num_bytes_in, const uint32_t num_bytes_out) - { - postCompressionNotification_(num_bytes_in, num_bytes_out); - }); - - task_queue_->addWorkerTask(std::move(async_compressor)); - } - - //! Post-compression callback that will let us know how - //! much we are gaining by using compression. We can use - //! this information to tweak the compression API calls - //! during simulation if needed. - void postCompressionNotification_( - const uint32_t num_bytes_pre_compression, - const uint32_t num_bytes_post_compression) - { - std::lock_guard guard(mutex_); - - total_num_bytes_sent_for_compression_ += num_bytes_pre_compression; - total_num_bytes_after_compression_ += num_bytes_post_compression; - - //TODO: This functionality is not complete yet, but we should be - //able to determine (in the sim loop) if we can get away with asking - //zlib to switch to Z_BEST_COMPRESSION if we're easily keeping up with - //the rate of incoming SI data, or Z_BEST_SPEED if we're not really - //benefitting much from attempting SI compression. In the worst case, - //we also need the ability to pull the plug on compression entirely - //and switch to Z_NO_COMPRESSION if we're struggling to keep up with - //the simulation. - // - //Note that the worker thread object would have to be changed a little - //bit to emit notifications when it is consuming work at a slower rate - //than it is receiving work packets. Before doing that, it may be useful - //to just add a stdout message during the final saveReports() call - //that says something like: - // - // Database thread has 17 things left it needs to do... - // COMPLETE - The worker thread took 1.38 seconds to flush - // the task queue at the end of simulation. - // - //If the database thread is *not* keeping up with the simulation, then - //this printout would show that we are left with more and more things - //that we still have to write to the database when we try running - //longer and longer simulations. - if (false) { - const double si_compression_pct = - 1 - (static_cast(total_num_bytes_after_compression_.getValue()) / - static_cast(total_num_bytes_sent_for_compression_.getValue())); - - (void) si_compression_pct; - - //Look at the current running "si_compression_pct" and decide - //what to do: go for more compression, go for more speed, turn - //off compression, re-enable compression, or stay the course. - } - } - - //! This callback is registered with the AsyncTaskEval to - //! let us know when a synchronous flush is being forced. - //! This gives us a chance to push any buffered data out - //! of the SIValuesBuffer objects into the worker queue. - void notifyTaskQueueAboutToFlush() override final { - pushBufferedDataToTaskQueue_(); - } - - //! Push any buffered SI values that are pending compression - //! into the task queue. This is called during synchronization - //! points like simulation pause/stop. - void pushBufferedDataToTaskQueue_() { - pushUncompressedBufferedDataToTaskQueue_(); - pushCompressedBufferedDataToTaskQueue_(); - } - - //! Push any buffered data from the uncompressed SI values - //! containers. - void pushUncompressedBufferedDataToTaskQueue_() { - if (uncompressed_si_buffer_ == nullptr) { - return; - } - if (uncompressed_si_buffer_->buffersAreEmpty()) { - return; - } - - const std::vector & values = - uncompressed_si_buffer_->getBufferedSIValues(); - - uint64_t starting_picoseconds = 0; - uint64_t ending_picoseconds = 0; - uint64_t starting_cycles = 0; - uint64_t ending_cycles = 0; - - uncompressed_si_buffer_->getBeginningAndEndingTimestampsForBufferedSIs( - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Add these values to the task queue for asynchronous processing - queueStatInstValuesOnWorker_( - values, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Reset the buffer for uncompressed SI's during each report - //update. Pass in FALSE so the buffer is not initialized with - //nan's (unnecessary performance hit to do so). - uncompressed_si_buffer_->resetSIBuffers(false); - } - - //! Push any buffered data from the compressed SI values - //! containers. - void pushCompressedBufferedDataToTaskQueue_() { - if (compressed_si_buffer_ == nullptr) { - return; - } - if (compressed_si_buffer_->buffersAreEmpty()) { - return; - } - - //If there is any pending SI data for the compressor, - //hand it over for async processing on the task thread. - const std::vector & uncompressed_values = - compressed_si_buffer_->getBufferedSIValues(); - - uint64_t starting_picoseconds = 0; - uint64_t ending_picoseconds = 0; - uint64_t starting_cycles = 0; - uint64_t ending_cycles = 0; - - compressed_si_buffer_->getBeginningAndEndingTimestampsForBufferedSIs( - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - queueCompressionEnabledStatInstValuesOnWorker_( - uncompressed_values, - starting_picoseconds, - ending_picoseconds, - starting_cycles, - ending_cycles); - - //Reset the buffer internals. Pass in FALSE so that - //it knows not to initialize the SI values to nan. - compressed_si_buffer_->resetSIBuffers(false); - } - - //! Shared worker thread object. We will give DB writes to - //! this "task queue" to handle for us in the background. - simdb::AsyncTaskEval * task_queue_ = nullptr; - - //! Shared database which holds all SI values. This object - //! is shared with the app::Simulation. - simdb::ObjectManager * sim_db_ = nullptr; - - //! Wrapper around the timeseries database table(s). - std::shared_ptr db_timeseries_; - - //! SI values are aggregated into one vector with - //! the help of this object. This makes DB writes easier. - std::unique_ptr si_aggregator_; - - //! SI values buffer which contains uncompressed statistics values. - std::unique_ptr uncompressed_si_buffer_; - - //! SI values buffer which contains SI values that will be sent - //! for async compression when it becomes full. - std::unique_ptr compressed_si_buffer_; - - //! Keep a running tally on the number of bytes that went into - //! the compressor, and the number of bytes that resulted after - //! compression. We'll use this info to tweak the size of the - //! compression buffer during simulation if needed. - utils::ValidValue total_num_bytes_sent_for_compression_; - utils::ValidValue total_num_bytes_after_compression_; - - //! Mutex to protect the compression-related variables. These - //! are accessed on the worker thread when we are asynchronously - //! told how the latest SI blob compression went. - mutable std::mutex mutex_; - - //! Report from which we collect all SI values and header info - const Report & report_; - - //! The simulation's root clock. Used in order to get the current - //! "time values" when we are asked to write the SI blobs into the - //! database. - const Clock & root_clk_; - - //! ID of the root-level report node in the database - simdb::DatabaseID root_report_node_id_ = 0; -}; - -} // namespace async -} // namespace sparta - diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp deleted file mode 100644 index 0e34713450..0000000000 --- a/sparta/sparta/collection/Collectable.hpp +++ /dev/null @@ -1,894 +0,0 @@ -// -*- C++ -*- - -/** - * \file Collectable.hpp - * - * \brief Implementation of the Collectable class that allows - * a user to collect an object into an pipeViewer pipeline file - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "sparta/collection/PipelineCollector.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/events/PayloadEvent.hpp" -#include "sparta/events/EventSet.hpp" -#include "sparta/events/SchedulingPhases.hpp" -#include "sparta/pairs/SpartaKeyPairs.hpp" -#include "sparta/utils/Utils.hpp" -#include "sparta/utils/MetaStructs.hpp" - -#include "boost/numeric/conversion/converter.hpp" - -namespace sparta{ - namespace collection - { - - /** - * \class Collectable - * \brief Class used to either manually or auto-collect an Annotation String - * object in a pipeline database - * \tparam DataT The DataT of the collectable being collected - * \tparam collection_phase The phase collection will occur. If - * sparta::SchedulingPhase::Collection, - * collection is done prior to Tick. - * - * Auto-collection will occur only if a Collectable is - * constructed with a collected_object. If no object is - * provided, it assumes a manual collection and the - * scheduling phase is ignored. - */ - - template - class Collectable : public CollectableTreeNode - { - public: - static constexpr uint64_t BAD_DISPLAY_ID = 0x1000; - - /** - * \brief Construct the Collectable, no data object associated, part of a group - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - Collectable(sparta::TreeNode* parent, - const std::string& name, - const std::string& group, - uint32_t index, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - CollectableTreeNode(sparta::notNull(parent), name, group, index, desc), - event_set_(this), - ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", - CREATE_SPARTA_HANDLER_WITH_DATA(Collectable, closeRecord, bool)) - { - sparta_assert(getNodeUID() < ~(decltype(transaction_t::location_ID))0, - "Tree has at least " << getNodeUID() << " in it. Because pipeouts " - "use these locations and only have 32b location IDs, the pipeline " - "collection system imposes a limit of " << ~(decltype(transaction_t::location_ID))0 - << " nodes in the tree. This is expected to be an unattainable number " - "without some kind of runaway allocation bug. If you are sure you " - << " nodes in the tree. This is expected to be an unattainable number " - "without some kind of runaway allocation bug. If you are sure you " - "need more nodes than this, contact the SPARTA team. In the meantime, " - "it is probably safe to comment out this assertion if you do not " - "plan to use the pipeline collection functionality"); - - argos_record_.location_ID = boost::numeric::converter::convert(getNodeUID()); - argos_record_.control_Process_ID = 0; // what is this for? - argos_record_.parent_ID = parentid; - argos_record_.flags = is_Annotation; - argos_record_.time_Start = 0; - argos_record_.time_End = 1; - - - } - - /** - * \brief Construct the Collectable - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param collected_object Pointer to the object to collect during the "COLLECT" phase - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - Collectable(sparta::TreeNode* parent, - const std::string& name, - const DataT * collected_object, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - Collectable(parent, name, - TreeNode::GROUP_NAME_NONE, - TreeNode::GROUP_IDX_NONE, - parentid, - desc) - { - collected_object_ = collected_object; - - // Get an initial value, if available - if(collected_object) { - std::ostringstream ss; - ss << *collected_object; - prev_annot_ = ss.str(); - } - } - - /** - * \brief Construct the Collectable, no data object associated - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - Collectable(sparta::TreeNode* parent, - const std::string& name, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - Collectable(parent, name, nullptr, parentid, desc) - { - // Can't auto collect without setting collected_object_ - setManualCollection(); - } - - //! Virtual destructor -- does nothing - virtual ~Collectable() {} - - /** - * \brief For manual collection, provide an initial value - * \param val The value to initial the record with - */ - void initialize(const DataT & val) { - std::ostringstream ss; - ss << val; - prev_annot_ = ss.str(); - } - - //! Explicitly/manually collect a value for this collectable, ignoring - //! what the Collectable is currently pointing to. - void collect(const DataT & val) - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - std::ostringstream ss; - ss << val; - if((ss.str() != prev_annot_) && !record_closed_) - { - // Close the old record (if there is one) - closeRecord(); - } - - // Remember the new string for a new record and start - // a new record if not empty. - prev_annot_ = ss.str(); - if(!prev_annot_.empty() && record_closed_) { - startNewRecord_(); - record_closed_ = false; - } - } - } - - /*! - * \brief Explicitly collect a value for the given duration - * \param duration The amount of time in cycles the value is available - * - * Explicitly collect a value for this collectable for the - * given amount of time. - * - * \warn No checks are performed if a new value is collected - * within the previous duration! - */ - void collectWithDuration(const DataT & val, sparta::Clock::Cycle duration) - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(duration != 0) { - ev_close_record_.preparePayload(false)->schedule(duration); - } - collect(val); - } - } - - //! Virtual method called by - //! CollectableTreeNode/PipelineCollector when a user of the - //! TreeNode requests this object to be collected. - void collect() override final { - // If pointer has become nullified, close the record - if(nullptr == collected_object_) { - closeRecord(); - return; - } - collect(*collected_object_); - } - - /*! - * \brief Calls collectWithDuration using the internal collected_object_ - * specified at construction. - * \pre Must have constructed wit ha non-null collected object - */ - void collectWithDuration(sparta::Clock::Cycle duration) { - // If pointer has become nullified, close the record - if(nullptr == collected_object_) { - closeRecord(); - return; - } - collectWithDuration(*collected_object_, duration); - } - - //! For heartbeat collections, existing records will need to - //! be closed and reopened - void restartRecord() override final { - // Do not get a new ID when we're doing a heartbeat - if(!record_closed_) { - argos_record_.flags |= CONTINUE_FLAG; - writeRecord_(); - argos_record_.flags &= ~CONTINUE_FLAG; - startNewRecord_(); - } - } - - //! Force close a record. This will close the record - //! immediately and clear the field for the next cycle - void closeRecord(const bool & simulation_ending = false) override final - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(!record_closed_ && writeRecord_(simulation_ending)) { - prev_annot_.clear(); - } - record_closed_ = true; - } - } - - //! \brief Do not perform any automatic collection - //! The SchedulingPhase is ignored - void setManualCollection() { - auto_collect_ = false; - } - - protected: - - //! \brief Get a reference to the internal event set - //! \return Reference to event set -- used by DelayedCollectable - EventSet & getEventSet_() { - return event_set_; - } - - private: - - //! Return true if the annotation was written; false otherwise - bool writeRecord_(bool simulation_ending = false) - { - sparta_assert(pipeline_col_, - "Must startCollecting_ on this Collectable " - << getLocation() << " before a record can be written"); - - // This record hasn't changed, don't write it - if(SPARTA_EXPECT_FALSE(argos_record_.time_Start == pipeline_col_->getScheduler()->getCurrentTick())) { - return false; - } - - // Set a new transaction ID since we're starting anew - argos_record_.transaction_ID = pipeline_col_->getUniqueTransactionId(); - - // Notice that we make the length +1 in order to null - // terminate the data. - argos_record_.length = static_cast(prev_annot_.size()); - - // We should just change the transaction structure to take - // an std::string instead of char* in the future. - argos_record_.annt = prev_annot_; - - // Capture the end time - argos_record_.time_End = - pipeline_col_->getScheduler()->getCurrentTick() + (simulation_ending ? 1 : 0);; - - // If this assert fires, then a user is trying to collect - // a record multiple times in their cycle window or - // something else really bad has happened. - sparta_assert(argos_record_.time_Start < argos_record_.time_End); - - // Write the old record to the file - pipeline_col_->writeRecord(argos_record_); - - return true; - } - - //! Start a new record - void startNewRecord_() { - // Set up the new start time for the new record - argos_record_.time_Start = pipeline_col_->getScheduler()->getCurrentTick(); - } - - //! Virtual method called by CollectableTreeNode when - //! collection is enabled on the TreeNode - void setCollecting_(bool collect, Collector * collector) override final - { - pipeline_col_ = dynamic_cast(collector); - sparta_assert(pipeline_col_ != nullptr, - "Collectables can only added to PipelineCollectors... for now"); - - if(collect && !prev_annot_.empty()) { - // Set the start time for this transaction to be - // moment collection is enabled. - startNewRecord_(); - } - - // If the collected object is null, this Collectable - // object is to be explicitly collected - if(collected_object_ && auto_collect_) { - if(collect) { - // Add this Collectable to the PipelineCollector's - // list of objects requiring collection - pipeline_col_->addToAutoCollection(this, collection_phase); - } - else { - // Remove this Collectable from the - // PipelineCollector's list of objects requiring - // collection - pipeline_col_->removeFromAutoCollection(this); - } - } - - if(!collect && !record_closed_) { - // Force the record to be written - closeRecord(); - } - } - - // The annotation object to be collected - const DataT * collected_object_ = nullptr; - - // The live transaction record - annotation_t argos_record_; - - // Store the result of the previous annotation, the - // annotation_t struct holds a pointer to this - std::string prev_annot_; - - // Ze Collec-tor - PipelineCollector * pipeline_col_ = nullptr; - - // For those folks that want a value to automatically - // disappear in the future - sparta::EventSet event_set_; - sparta::PayloadEvent ev_close_record_; - - // Is this collectable currently closed? - bool record_closed_ = true; - - // Should we auto-collect? - bool auto_collect_ = true; - }; - - /** - * \class UniquePairGenerator - * \brief A class which provides every new instantiation of the Templated class Collectable - * with a new unique value. This value is used by all the instantiations of a certain type - * of a Collectable as its own unique PairId which differentiates itself from other Collectable - * instatiations templated on some other type of class. - */ - class UniquePairIDGenerator { - public: - - // We need to make Collectable a friend of this class, as it will directly call - // the static private function from within itself. - template - friend class Collectable; - private: - - // Any point this funtion is invoked, it releases a new 64bit integer which is unique. - // So, we need to limit the access of this function. - static uint64_t getUniquePairID_() { - static uint64_t id = 0; - return ++id; - } - }; - - /** - * \class Collectable - * \brief Class used to either manually or auto-collect a Name Value Pair - * object in a pipeline database - * \tparam DataT The DataT of the Pair Definition of the collectable being collected - * \tparam collection_phase The phase collection will occur. If - * sparta::SchedulingPhase::Collection, - * collection is done prior to Tick. - * - * Auto-collection will occur only if a Collectable is - * constructed with a collected_object. If no object is - * provided, it assumes a manual collection and the - * scheduling phase is ignored. - * - * We are templatizing the Collectable Class on the actual collectable to be collected. - * So, the modeler does not have to maintain two different ways to call Collectable. - * The modeler will always templatize Collectable on - * the actual DataType or on a pointer to it, exactly the way they do now. - * To enable this template overload, we need to check if the DataT the modeler - * has templatized on Collectable class, - * is actually a sparta::Pair type or not. - * The only way to check this is to find out if this DataT type has a type alias inside - * its namespace which is derived from sparta::PairDefinition class which in turn must be templatized - * on the actual Collectable class or the DataT type itself, that we want to collect. - * That is why we have a type named "SpartaPairDefinitionType" inside the Actual Collectable class which refers to its - * PairDefinition type. - * For the same reason, we have a type named "SpartaPairDefinitionType" inside the Pair Definition class which refers - * to its Actual Collectable type. - * The std::enable_if template switching basically checks if the DataType has a type named "SpartaPairDefinitionType", - * which is actually a PairDefinition of itself, a Pair Collectable Entity, or not. - * The only way to check if DataType has a type "SpartaPairDefinitionType" which is Pair Definition of itself, a - * Collectable Entity or not, is by using SFINAE(Substitution Failure Is Not An Error). - * It checks if DataType has a type "SpartaPairDefinitionType" in its namespace which derives from sparta::PairDefinition - * which is templatized on the the DataType itself. - * If this exact piece of code is substituted to form a well-formed code, this template overload is - * selected by compiler. If this does not work, then the compiler creates an ill-formed code. - * Any ill-formed code during Template Parameter Substitution is not considered an error. - * It is rather thrown away, and the compiler moves to find a more generic Template Overload. - * - * \note This Template Overload is switched on only for Pair-Collectable User Defined DataTypes. - * \note The Modeler might also pass a Shared Pointer to the actual DataT object - * instead of the actual Object. - * \note So, we handle that case by using remove_shared_ptr templated struct. - */ - template - class Collectable>, - typename MetaStruct::remove_any_pointer_t::SpartaPairDefinitionType>::value>> : - public sparta::PairCollector::SpartaPairDefinitionType>, public CollectableTreeNode - { - // Aliasing the actual Datatype of the collectable being collected as Data_t - typedef MetaStruct::remove_any_pointer_t Data_t; - - // Aliasing the Datatype of the Pair Definition of the collectable being collected as PairDef_t - typedef typename Data_t::SpartaPairDefinitionType PairDef_t; - - // Making a bunch of APIs of PairCollector class being available to us with the using directive - using PairCollector::getNameStrings; - using PairCollector::getDataVector; - using PairCollector::getStringVector; - using PairCollector::getFormatVector; - using PairCollector::getSizeOfVector; - using PairCollector::getPEventLogVector; - using PairCollector::getArgosFormatGuide; - using PairCollector::collect_; - using PairCollector::isCollecting; - - public: - - /** - * \brief Construct the Collectable, no data object associated, part of a group - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - Collectable(sparta::TreeNode* parent, - const std::string& name, - const std::string& group, - uint32_t index, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - sparta::PairCollector(), - CollectableTreeNode(sparta::notNull(parent), name, group, index, desc), - argos_record_(pair_t(0, 1, parentid, 0, BAD_DISPLAY_ID, - boost::numeric::converter::convert(getNodeUID()), - is_Pair, 0)), - event_set_(this), - ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", - CREATE_SPARTA_HANDLER_WITH_DATA(Collectable, closeRecord, bool)) { - static constexpr auto MAX_UID = std::numeric_limits::max(); - sparta_assert(getNodeUID() < MAX_UID, - "Tree has at least " << getNodeUID() << " in it. Because pipeouts " - "use these locations and only have 32b location IDs, the pipeline " - "collection system imposes a limit of " << MAX_UID - << " nodes in the tree. This is expected to be an unattainable number " - "without some kind of runaway allocation bug. If you are sure you " - "need more nodes than this, contact the SPARTA team. In the meantime, " - "it is probably safe to comment out this assertion if you do not " - "plan to use the pipeline collection functionality"); - } - - /** - * \brief Construct the Collectable - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param collected_object Pointer to the object to collect during the "COLLECT" phase - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - - Collectable(sparta::TreeNode* parent, - const std::string& name, - const DataT * collected_object, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - Collectable(parent, name, - TreeNode::GROUP_NAME_NONE, - TreeNode::GROUP_IDX_NONE, - parentid, - desc) - { - collected_object_ = collected_object; - - // Get an initial value, if available - if(collected_object) { - initialize(*collected_object); - } - } - - /** - * \brief Construct the Collectable, no data object associated - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - Collectable(sparta::TreeNode* parent, - const std::string& name, - uint64_t parentid = 0, - const std::string & desc = "Collectable ") : - Collectable(parent, name, nullptr, parentid, desc) - { - // Can't auto collect without setting collected_object_ - setManualCollection(); - } - - //! Virtual destructor -- does nothing - virtual ~Collectable() {} - - /** - * \brief For manual collection, provide an initial value - * \param val The value to initialze the record with - */ - template - MetaStruct::enable_if_t::value, void> - initialize(const T & val) { - if(SPARTA_EXPECT_FALSE(isCollected())) { - collect(val); - } - } - - /** - * \brief For manual collection, provide an initial value - * \param val A reference to a pointer pointing to the value to initialze the record with - */ - template - MetaStruct::enable_if_t::value, void> - initialize(const T & val){ - if(SPARTA_EXPECT_FALSE(isCollected())) { - collect(*val); - } - } - - //! Explicitly/manually collect a value for this collectable, ignoring - //! what the Collectable is currently pointing to. - //! Here we pass the actual object of the collectable type we are collecting. - template - MetaStruct::enable_if_t::value, void> - collect(const T & val) - { - collect_(val); - - // if the value pairs are not the same as the - // previous value pairs and the previous record is still open - if(!isSameRecord() && !record_closed_) - { - //Close the old record (if there is one) - closeRecord(); - } - - // Remember the new value pairs for a new record and start - // a new record if not empty. - updateLastRecord_(); - - // If the new Value pairs are not empty and current record is closed, we start a new record. - if(record_closed_ && hasData_()){ - startNewRecord_(); - record_closed_ = false; - } - } - - //! Explicitly/manually collect a value for this collectable, ignoring - //! what the Collectable is currently pointing to. - //! Here we pass the shared pointer to the actual object of the collectable type we are collecting. - template - MetaStruct::enable_if_t::value, void> - collect(const T & val){ - // If pointer has become nullified, close the record - if(nullptr == val) { - closeRecord(); - return; - } - collect(*val); - } - - /*! - * \brief Explicitly collect a value for the given duration - * \param duration The amount of time in cycles the value is available - * - * Explicitly collect a value for this collectable for the - * given amount of time. - * - * \warn No checks are performed if a new value is collected - * within the previous duration! - */ - template - MetaStruct::enable_if_t::value, void> - collectWithDuration(const T & val, sparta::Clock::Cycle duration){ - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(duration != 0) { - ev_close_record_.preparePayload(false)->schedule(duration); - } - collect(val); - } - } - - /*! - * \brief Explicitly collect a value from a shared pointer for the given duration - * \param duration The amount of time in cycles the value is available - * - * Explicitly collect a value for this collectable passed as a shared pointer for the - * given amount of time. - * - * \warn No checks are performed if a new value is collected - * within the previous duration! - */ - template - MetaStruct::enable_if_t::value, void> - collectWithDuration(const T & val, sparta::Clock::Cycle duration){ - // If pointer has become nullified, close the record - if(nullptr == val) { - closeRecord(); - return; - } - collectWithDuration(*val, duration); - } - - //! Virtual method called by - //! CollectableTreeNode/PipelineCollector when a user of the - //! TreeNode requests this object to be collected. - void collect() override final { - // If pointer has become nullified, close the record - if(nullptr == collected_object_) { - closeRecord(); - return; - } - collect(*collected_object_); - } - - /*! - * \brief Calls collectWithDuration using the internal collected_object_ - * specified at construction. - * \pre Must have constructed wit ha non-null collected object - */ - void collectWithDuration(sparta::Clock::Cycle duration) { - // If pointer has become nullified, close the record - if(nullptr == collected_object_) { - closeRecord(); - return; - } - collectWithDuration(*collected_object_, duration); - } - - //! For heartbeat collections, existing records will need to - //! be closed and reopened - void restartRecord() override final { - - // Do not get a new ID when we're doing a heartbeat - if(!record_closed_) { - argos_record_.flags |= CONTINUE_FLAG; - writeRecord_(); - argos_record_.flags &= ~CONTINUE_FLAG; - startNewRecord_(); - } - } - - //! Force close a record. This will close the record - //! immediately and clear the field for the next cycle - void closeRecord(const bool & simulation_ending = false) override final - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(!record_closed_ && writeRecord_(simulation_ending)) { - - // Clear the previous vector containing the Name Value pairs. - argos_record_.valueVector.clear(); - argos_record_.stringVector.clear(); - } - record_closed_ = true; - } - } - - //! \brief Do not perform any automatic collection - //! The SchedulingPhase is ignored - void setManualCollection() { - auto_collect_ = false; - } - - //! \brief Before writing a record to file, we need to check - // if any of the old Values have changed or not. - bool isSameRecord() const { - return (argos_record_.valueVector == getDataVector()) && (argos_record_.stringVector == getStringVector()); - } - - //! \brief Strictly a Debug/Testing API. - //! Never to be called in real modeler's code. - const std::string & dumpNameValuePairs(const DataT & val) { - collect_(val); - log_string_.clear(); - std::ostringstream ss; - for(const auto & pairs : getPEventLogVector()){ - ss << pairs.first << "(" << pairs.second << ") "; - } - log_string_ = ss.str(); - return log_string_; - } - - protected: - - //! \brief Get a reference to the internal event set - //! \return Reference to event set -- used by DelayedCollectable - inline EventSet & getEventSet_() { - return event_set_; - } - - //! \brief Collectable is inheriting from PairCollector Class - // and this is a pure virtual function in base - //! \brief We need to redefine and override this function, - // else Collectable will also become Abstract. - virtual void generateCollectionString_() override {} - - private: - - /** - * \brief A static function which sets the Unique PairId of a certain instantiation of Collectable. - * Function contains a static variable which tells us if this templated instantiation of Collectable - * already has been instantiated anytime before, or this is the first time we are instantiating it. - * Checks the value of the static variable to see if 0 or not. If it is 0, it means this is the very - * first time we are creating an instance of this class, Set the variable id to the getuniquePairID - * function of the UniquePairIDGenerator class generates, guranteed to be unique - * and not shared by any other Pair class. - */ - inline static uint64_t getUniquePairID_() { - static uint64_t id = 0; - if(id){ return id; } - id = sparta::collection::UniquePairIDGenerator::getUniquePairID_(); - return id; - } - - //! \brief Update the vector containing the Name Value Pairs - // we collected from the last collection - //! \brief Fill it with the new Name Value pairs from - // this cycle of Collection. - void updateLastRecord_(){ - // These values will change every time the transaction changes - argos_record_.valueVector = getDataVector(); - argos_record_.stringVector = getStringVector(); - argos_record_.sizeOfVector = getSizeOfVector(); - - if (has_display_id_field_) { - argos_record_.display_ID = argos_record_.valueVector[0].first & 0x0fff; - } - } - - //! \brief Does this transaction contain any data yet? - bool hasData_() const { - return !(argos_record_.valueVector.empty() && argos_record_.stringVector.empty()); - } - - //! \brief Send the Pair Structure Record to Outputter - // for writing to the Transaction Database File. - bool writeRecord_(bool simulation_ending = false) - { - sparta_assert(pipeline_col_, - "Must startCollecting_ on this Collectable " - << getLocation() << " before a record can be written"); - - // This record hasn't changed, don't write it - if(SPARTA_EXPECT_FALSE(argos_record_.time_Start == pipeline_col_->getScheduler()->getCurrentTick())) { - return false; - } - - // Set a new transaction ID since we're starting anew - argos_record_.transaction_ID = pipeline_col_->getUniqueTransactionId(); - - argos_record_.pairId = getUniquePairID_(); - - // Capture the end time - argos_record_.time_End = - pipeline_col_->getScheduler()->getCurrentTick() + (simulation_ending ? 1 : 0); - - // If this assert fires, then a user is trying to collect - // a record multiple times in their cycle window or - // something else really bad has happened. - sparta_assert(argos_record_.time_Start < argos_record_.time_End); - - // Write the old record to the file - pipeline_col_->writeRecord(argos_record_); - return true; - } - - //! Start a new record - void startNewRecord_() { - - // Set up the new start time for the new record - argos_record_.time_Start = pipeline_col_->getScheduler()->getCurrentTick(); - } - - //! collection is enabled on the TreeNode - void setCollecting_(bool collect, Collector * collector) override final - { - pipeline_col_ = dynamic_cast(collector); - sparta_assert(pipeline_col_ != nullptr, - "Collectables can only added to PipelineCollectors... for now"); - - // These fields only need to be set once - they define the format of the collectable - // and remain constant for the duration of the simulation - argos_record_.nameVector = getNameStrings(); - argos_record_.length = argos_record_.nameVector.size(); - has_display_id_field_ = (argos_record_.nameVector[0] == "DID"); - for(const auto formatter: getFormatVector()) { - argos_record_.delimVector.emplace_back(formatter); - } - - if(collect && hasData_()){ - - // Set the start time for this transaction to be - // moment collection is enabled. - startNewRecord_(); - } - - // If the collected object is null, this Collectable - // object is to be explicitly collected - if(collected_object_ && auto_collect_) { - if(collect) { - - // Add this Collectable to the PipelineCollector's - // list of objects requiring collection - pipeline_col_->addToAutoCollection(this, collection_phase); - } - else { - - // Remove this Collectable from the - // PipelineCollector's list of objects requiring - // collection - pipeline_col_->removeFromAutoCollection(this); - } - } - } - // The pair collectable object to be collected - const DataT * collected_object_ = nullptr; - - // The live transaction record - pair_t argos_record_; - - // Ze Collec-tor - PipelineCollector * pipeline_col_ = nullptr; - - // For those folks that want a value to automatically - // disappear in the future - sparta::EventSet event_set_; - sparta::PayloadEvent ev_close_record_; - - // Is this collectable currently closed? - bool record_closed_ = true; - - // Should we auto-collect? - bool auto_collect_ = true; - - // Is the first field the display ID (DID)? - bool has_display_id_field_ = false; - - std::string log_string_; - }; - }//namespace collection -}//namespace sparta diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index ca87578941..687e5ebae3 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -1,134 +1,121 @@ -// -*- C++ -*- +#pragma once +#include "sparta/simulation/TreeNode.hpp" +#include "sparta/collection/CollectionPoints.hpp" -/** - *\file CollectableTreeNode.hpp - * - * - *\brief define a CollectableTreeNode type TreeNode. - */ +namespace sparta { +namespace collection { -#pragma once +class CollectionPoints; -#include -#include +class CollectableTreeNode : public TreeNode +{ +public: + CollectableTreeNode(TreeNode* parent, const std::string& name, const std::string& desc = "CollectableTreeNode ") + : TreeNode(parent, name, desc) + { + markHidden(); + } -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/collection/Collector.hpp" + virtual void addCollectionPoint(CollectionPoints & collection_points) = 0; +}; -namespace sparta{ -namespace collection +template +class Collectable : public CollectableTreeNode { +public: + using value_type = typename MetaStruct::remove_any_pointer::type; - /** - * \class CollectableTreeNode - * \brief An abstract type of TreeNode that - * has virtual calls to start collection on this node, - * and stop collection on this node. - */ - class CollectableTreeNode : public sparta::TreeNode + Collectable(TreeNode* parent, const std::string& name, const CollectableT* collectable, const std::string& desc = "Collectable ") + : CollectableTreeNode(parent, name, desc) + , collectable_(collectable) { - public: - /** - * \brief Construct. - * \param parent a pointer to the parent treenode - * \param name the name of this treenode - * \param group the name of the group for this treenode - * \param index the index within the group - * \param desc A description for this treenode. - */ - CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, - const std::string& group, uint32_t index, - const std::string& desc = "CollectableTreeNode ") : - sparta::TreeNode(parent, name, group, index, desc) - { - markHidden(); // Mark self as hidden from the default - // printouts (to reduce clutter) - } + } - /** - * \brief Construct. - * \param parent a pointer to the parent treenode - * \param name the name of this treenode - * \param desc Description of this CollectableTreeNode - */ - CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, - const std::string& desc= "CollectableTreeNode ") : - CollectableTreeNode(parent, name, sparta::TreeNode::GROUP_NAME_NONE, - sparta::TreeNode::GROUP_IDX_NONE, desc) - {} - - //!Virtual destructor - virtual ~CollectableTreeNode() - {} - - /** - * \brief Method that tells this treenode that is now running - * collection. - * \param collector The collector that is performing the collection - * - * This method should instantiate any necessary values for the - * treenode necessary to collection as well as flip the - * is_collecting boolean. - */ - void startCollecting(Collector * collector) { - is_collected_ = true; - setCollecting_(true, collector); + void addCollectionPoint(CollectionPoints & collection_points) override + { + auto is_pod = std::integral_constant::value>(); + addCollectionPoint_(collection_points, is_pod); + } + +private: + void addCollectionPoint_(CollectionPoints & collection_points, std::true_type) + { + if (collectable_) { + collection_points.addStat(getLocation(), getClock(), collectable_); } + } - /** - * \brief Method that tells this treenode that is now not - * running collection. - */ - void stopCollecting(Collector * collector) - { - setCollecting_(false, collector); - is_collected_ = false; + void addCollectionPoint_(CollectionPoints & collection_points, std::false_type) + { + sparta_assert(!collectable_); + //TODO cnyce collection_points.add + } + + const CollectableT* collectable_; +}; + +template <> +class Collectable : public CollectableTreeNode +{ +public: + using value_type = int32_t; + + Collectable(TreeNode* parent, const std::string& name, const bool* collectable, const std::string& desc = "Collectable ") + : CollectableTreeNode(parent, name, desc) + , collectable_(collectable) + { + } + + void addCollectionPoint(CollectionPoints & collection_points) override + { + std::function get_bool_as_int = [this]() { return getBoolAsInt_(); }; + collection_points.addStat(getLocation(), getClock(), get_bool_as_int, simdb::Format::boolalpha); + } + +private: + value_type getBoolAsInt_() const + { + return *collectable_ ? 1 : 0; + } + + const bool* collectable_; +}; + +template +class IterableCollector : public CollectableTreeNode +{ +public: + IterableCollector(TreeNode* parent, const std::string& name, const ContainerT* container, const size_t capacity, const std::string& desc = "IterableCollector ") + : CollectableTreeNode(parent, name, desc) + , container_(container) + , capacity_(capacity) + { + for (size_t i = 0; i < capacity_; ++i) { + bins_.emplace_back(new IterableCollectorBin(this, name + std::to_string(i))); } + } - /** - * \brief Determine whether or not this node has collection - * turned on or off. - */ - bool isCollected() const { return is_collected_; } - - //! Pure virtual method used by deriving classes to be - //! notified when they can perform their collection - virtual void collect() = 0; - - //! Pure virtual method used by deriving classes to re-start a - //! record if necessary. This is mostly used by - //! PipelineCollector where it will insert a heartbeat index - //! by closing all records and opening them again. - virtual void restartRecord() {} - - //! Pure virtual method used by deriving classes to force - //! close a record. This is useful for simulation end where - //! each collectable is allowed its final say - //! \param simulation_ending If true, the simulation is - //! terminating and the "end cycle" is not really the - //! true end. Implementers of this method should a +1 - //! to their end cycle in their records to ensure it - //! does not get closed out. - virtual void closeRecord(const bool & simulation_ending = false) { (void) simulation_ending; } - - protected: - - /** - * \brief Indicate to sub-classes that collection has flipped - * \param collect true if collection is enabled; false otherwise - */ - virtual void setCollecting_(bool collect, Collector *) { (void) collect; } - - private: - - /** - * \brief A value that represents whether or not this TreeNode is - * being collected by a collector - */ - bool is_collected_ = false; + void addCollectionPoint(CollectionPoints & collection_points) override + { + collection_points.addContainer(getLocation(), getClock(), container_, capacity_); + } +private: + class IterableCollectorBin : public TreeNode + { + public: + IterableCollectorBin(TreeNode* parent, const std::string& name, const std::string& desc = "IterableCollectorBin ") + : TreeNode(parent, name, desc) + { + markHidden(); + } }; -} -} + const ContainerT* container_; + const size_t capacity_; + std::vector> bins_; +}; + +} // namespace collection +} // namespace sparta diff --git a/sparta/sparta/collection/CollectionPoints.hpp b/sparta/sparta/collection/CollectionPoints.hpp new file mode 100644 index 0000000000..37cd925667 --- /dev/null +++ b/sparta/sparta/collection/CollectionPoints.hpp @@ -0,0 +1,236 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/collection/Scalars.hpp" +#include "simdb/collection/Structs.hpp" +#include "simdb/collection/IterableStructs.hpp" +#include "simdb/collection/TimeseriesCollector.hpp" +#include "sparta/simulation/Clock.hpp" +#include "sparta/utils/MetaStructs.hpp" + +#include +#include // For __cxa_demangle +#include + +namespace sparta +{ + inline std::string demangle(const char* mangled_name) { + int status = 0; + char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, nullptr, &status); + if (status == 0 && demangled_name) { + std::string result(demangled_name); + std::free(demangled_name); + return result; + } + return mangled_name; + } + + template + inline std::string demangled_type() { + return demangle(typeid(T).name()); + } +} + +namespace sparta { +namespace collection { + +class CollectionPoints +{ +public: + template + typename std::enable_if::value && std::is_standard_layout::value, void>::type + addStat(const std::string& location, const Clock* clk, const StatT* back_ptr, const simdb::Format format = simdb::Format::none) + { + auto demangled = demangled_type(); + auto& instantiator = instantiators_[demangled]; + if (!instantiator) { + instantiator.reset(new StatInstantiator()); + } + + dynamic_cast*>(instantiator.get())->addStat(location, clk, back_ptr, format); + } + + template + typename std::enable_if::value && std::is_standard_layout::value, void>::type + addStat(const std::string& location, const Clock* clk, std::function func_ptr, const simdb::Format format = simdb::Format::none) + { + auto demangled = demangled_type(); + auto& instantiator = instantiators_[demangled]; + if (!instantiator) { + instantiator.reset(new StatInstantiator()); + } + + dynamic_cast*>(instantiator.get())->addStat(location, clk, func_ptr, format); + } + + template + typename std::enable_if::value, void>::type + addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) + { + auto demangled = demangled_type() + (Sparse ? "Sparse" : "Contig"); + auto& instantiator = instantiators_[demangled]; + if (!instantiator) { + instantiator.reset(new IterStructInstantiator()); + } + + dynamic_cast*>(instantiator.get())->addContainer(location, clk, container, capacity); + } + + template + typename std::enable_if::value, void>::type + addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) + { + // TODO cnyce + (void)location; + (void)clk; + (void)container; + (void)capacity; + } + + void createCollections(simdb::Collections* collections) + { + std::unordered_map clk_periods; + for (const auto& kvp : instantiators_) { + kvp.second->getClockPeriods(clk_periods); + } + + std::unordered_map clk_names_by_location; + for (const auto& kvp : instantiators_) { + kvp.second->getClockNamesByLocation(clk_names_by_location); + } + + for (const auto& kvp : clk_periods) { + collections->addClock(kvp.first, kvp.second); + } + + for (const auto& kvp : clk_names_by_location) { + collections->setClock(kvp.first, kvp.second); + } + + size_t idx = 0; + for (auto& kvp : instantiators_) { + const auto collection_prefix = "Collection" + std::to_string(idx); + kvp.second->createCollections(collections, collection_prefix); + ++idx; + } + + instantiators_.clear(); + } + +private: + class CollectableInstantiator + { + public: + virtual ~CollectableInstantiator() = default; + virtual void getClockPeriods(std::unordered_map& clk_periods) const = 0; + virtual void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const = 0; + virtual void createCollections(simdb::Collections* collections, const std::string& collection_prefix) = 0; + }; + + template + class StatInstantiator : public CollectableInstantiator + { + public: + void addStat(const std::string& location, const Clock* clk, const StatT* back_ptr, simdb::Format format = simdb::Format::none) + { + simdb::ScalarValueReader reader(back_ptr); + simdb::Stat stat(location, reader, format); + stats_.emplace_back(location, clk, stat); + } + + void addStat(const std::string& location, const Clock* clk, std::function func_ptr, simdb::Format format = simdb::Format::none) + { + simdb::ScalarValueReader reader(func_ptr); + simdb::Stat stat(location, reader, format); + stats_.emplace_back(location, clk, stat); + } + + void getClockPeriods(std::unordered_map& clk_periods) const override + { + for (const auto& tup : stats_) { + const auto clk = std::get<1>(tup); + clk_periods[clk->getName()] = clk->getPeriod(); + } + } + + void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override + { + for (const auto& tup : stats_) { + const auto location = std::get<0>(tup); + const auto clk = std::get<1>(tup); + clk_names_by_location[location] = clk->getName(); + } + } + + void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override + { + using CollectionT = simdb::StatCollection; + const auto collection_name = collection_prefix + "_" + demangled_type(); + std::unique_ptr collection(new CollectionT(collection_name)); + + for (const auto& tup : stats_) { + const Clock* clk = std::get<1>(tup); + const simdb::Stat& stat = std::get<2>(tup); + collection->addStat(stat, clk->getName()); + } + + collections->addCollection(std::move(collection)); + } + + private: + std::vector>> stats_; + }; + + template + class IterStructInstantiator : public CollectableInstantiator + { + public: + void addContainer(const std::string& location, const Clock* clk, const ContainerT* obj, const size_t capacity) + { + containers_.emplace_back(location, clk, obj, capacity); + } + + void getClockPeriods(std::unordered_map& clk_periods) const override + { + for (const auto& tup : containers_) { + const auto clk = std::get<1>(tup); + clk_periods[clk->getName()] = clk->getPeriod(); + } + } + + void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override + { + for (const auto& tup : containers_) { + const auto location = std::get<0>(tup); + const auto clk = std::get<1>(tup); + clk_names_by_location[location] = clk->getName(); + } + } + + void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override + { + using CollectionT = simdb::IterableStructCollection; + + for (size_t idx = 0; idx < containers_.size(); ++idx) { + const std::string &location = std::get<0>(containers_[idx]); + const ContainerT *obj = std::get<2>(containers_[idx]); + const size_t capacity = std::get<3>(containers_[idx]); + + const auto collection_name = collection_prefix + "_" + demangled_type() + "_" + std::to_string(idx); + std::unique_ptr collection(new CollectionT(collection_name)); + + collection->addContainer(location, obj, capacity); + collections->addCollection(std::move(collection)); + } + } + + private: + std::vector> containers_; + }; + + std::unordered_map> instantiators_; +}; + +} // namespace collection +} // namespace sparta diff --git a/sparta/sparta/collection/Collector.hpp b/sparta/sparta/collection/Collector.hpp deleted file mode 100644 index 0a80236857..0000000000 --- a/sparta/sparta/collection/Collector.hpp +++ /dev/null @@ -1,52 +0,0 @@ -// -*- C++ -*- - -/** - * \file Collector.hpp - * - * \brief Define a base Collector class. - */ - -#pragma once - -#include - -namespace sparta{ -namespace collection -{ - /** - * \class Collector - * - * \brief A non-templated base class that all Collectors should - * inherit from. - */ - class Collector - { - public: - /** - * \brief Construct a collector. - * \param parent the parent tree node for this collector. - * \param name the name of the collector. - * \param group the TreeNode group name. - * \param group_idx the TreeNode group index. - */ - Collector(const std::string& name) : - name_(name) - { } - - //! Enable polymorphism and also be nice - virtual ~Collector() {} - - std::string getName() const { - return name_; - } - - protected: - /*! - * The name of the collector. This will be used by many - * collectors to create child collectable objects - */ - std::string name_; - }; -}//namespace collection -}//namespace sparta - diff --git a/sparta/sparta/collection/DelayedCollectable.hpp b/sparta/sparta/collection/DelayedCollectable.hpp deleted file mode 100644 index 3cae60d31f..0000000000 --- a/sparta/sparta/collection/DelayedCollectable.hpp +++ /dev/null @@ -1,183 +0,0 @@ -// -*- C++ -*- - -/** - * \file DelayedCollectable.hpp - * - * \brief Implementation of the DelayedCollectable class that allows - * a user to collect an object into an pipeViewer pipeline file - */ - -#pragma once - -#include -#include -#include "sparta/collection/Collectable.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/events/EventSet.hpp" -#include "sparta/events/PayloadEvent.hpp" - -namespace sparta { - -namespace collection -{ - /** - * \class DelayedCollectable - * \brief Class used to record data in pipeline collection, but - * recorded in a delayed fashion - * - * DelayedCollectable is useful for delivering a collected chunk - * of data to the PipelineCollector in the future. Such an - * example is SyncPort where data can be sent with a delay of N - * and the view should show this data only on cycle N for - * delivery. - */ - template - class DelayedCollectable : public Collectable - { - using CollectableTreeNode::isCollected; - - struct DurationData - { - DurationData() {} - - DurationData(const DataT & dat, - sparta::Clock::Cycle duration) : - data(dat), duration(duration) - {} - - DataT data; - sparta::Clock::Cycle duration; - }; - - public: - - // Promote the collect method from Collectable - using Collectable::collect; - - /** - * \brief Construct the DelayedCollectable, no data object associated, part of a group - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param group The group for which to create this object as a child sparta::TreeNode - * \param index The index within the group for this collectable - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - DelayedCollectable(sparta::TreeNode* parent, - const std::string& name, - const std::string& group, - uint32_t index, - uint64_t parentid = 0, - const std::string & desc = "DelayedCollectable ") : - Collectable(parent, name, group, index, desc) - { - (void) parentid; - } - - /** - * \brief Construct the DelayedCollectable - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param collected_object Pointer to the object to collect during the "COLLECT" phase - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - DelayedCollectable(sparta::TreeNode* parent, - const std::string& name, - const DataT * collected_object, - uint64_t parentid = 0, - const std::string & desc = "DelayedCollectable ") : - Collectable(parent, name, collected_object, parentid, desc) - {} - - /** - * \brief Construct the DelayedCollectable, no data object associated - * \param parent A pointer to a parent treenode. Must not be null - * \param name The name for which to create this object as a child sparta::TreeNode - * \param parentid The transaction id of a parent for this collectable; 0 for no parent - * \param desc A description for the interface - */ - DelayedCollectable(sparta::TreeNode* parent, - const std::string& name, - uint64_t parentid = 0, - const std::string & desc = "DelayedCollectable ") : - DelayedCollectable(parent, name, nullptr, parentid, desc) - { - // Can't auto collect without setting iterable_object_ - Collectable::setManualCollection(); - } - - /*! - * \brief Explicitly collect a value in the future - * \param val The value to collect in the future - * \param delay The delay before recording this value to file - * - * Explicitly collect a value for this collectable in the - * future, ignoring what the Collectable is currently pointing - * to. - */ - void collect(const DataT & val, sparta::Clock::Cycle delay) - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(delay != 0) { - ev_collect_.schedule(val, delay); - } - else { - Collectable::collect(val); - } - } - } - - /*! - * \brief Explicitly collect a value in the future - * \param val The value to collect in the future - * \param delay The delay before recording this value to file - * \param duration The amount of time in cycles the value is available - * - * Explicitly collect a value for this collectable in the - * future, ignoring what the Collectable is currently pointing - * to. - */ - void collectWithDuration(const DataT & val, - sparta::Clock::Cycle delay, - sparta::Clock::Cycle duration) - { - if(SPARTA_EXPECT_FALSE(isCollected())) - { - if(delay != 0) { - ev_collect_duration_.preparePayload({val, duration})->schedule(delay); - } - else { - Collectable::collectWithDuration(val, duration); - } - } - } - - private: - - // Called from collectWithDuration where the data needs to be - // delivered at a given delayed time, but only for a short - // duration. - void collectWithDuration_(const DurationData & dur_dat) { - Collectable::collectWithDuration(dur_dat.data, dur_dat.duration); - } - - // For those folks that want collection to appear in the - // future - sparta::PayloadEvent ev_collect_{ - &Collectable::getEventSet_(), "delayedpipelinecollectable_event", - CREATE_SPARTA_HANDLER_WITH_DATA(Collectable, collect, DataT)}; - - // For those folks that want collection to appear in the - // future with a duration - sparta::PayloadEvent ev_collect_duration_{ - &Collectable::getEventSet_(), "delayedpipelinecollectable_duration_event", - CREATE_SPARTA_HANDLER_WITH_DATA(DelayedCollectable, - collectWithDuration_, DurationData)}; - - - }; -}//namespace collection -}//namespace sparta - diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp deleted file mode 100644 index bb8b96f0bd..0000000000 --- a/sparta/sparta/collection/IterableCollector.hpp +++ /dev/null @@ -1,352 +0,0 @@ -// -*- C++ -*- - -/* - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include "sparta/utils/Utils.hpp" -#include "sparta/log/MessageSource.hpp" -#include "sparta/events/SchedulingPhases.hpp" -#include "sparta/collection/Collectable.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" -#include "sparta/collection/PipelineCollector.hpp" - -namespace sparta { -namespace collection { - -/** - * \class IterableCollector - * \brief A collector of any iterable type (std::vector, std::list, sparta::Buffer, etc) - * - * \tname IterableType The type of the collected object - * \tname collection_phase The phase collection will occur. - * Collection happens automatically in this - * phase, unless it is disabled by a call to - * setManualCollection() - * \tname sparse_array_type Set to true if the iterable type is - * sparse, meaning iteration will occur on - * the entire iterable object, but each - * iterator might not be valid to - * de-reference. When this is true, it is - * expected that the iterator returned from - * the IterableType can be queried for - * validity (by a call to itr->isValid()). - * - * This collector will iterable over an std::array type, std::list - * type, std::vector type, sparta::Buffer, sparta::Queue, sparta::Array, or - * even a simple C-array type. The class needs to constructed with an - * expected capacity of the container, and the container should never - * grow beyond this expected capacity. If so, during collection, the - * class will output a warning message (only once). - * - * How collection is performed: - * - * - During the phase given by the template argument, this class will - * be given an opportunity to collect the object (convert the - * container's objects to a string using operator<<). - * - * - The main collection loop always iterates from 0 to the expected - * capacity, even if the \i size of the container is smaller. For - * every element in the collected object (begin() -> end()[size]), - * the object is collected. From end()[size] -> expected capacity, - * records are closed: - * - * 
- *   |  collected   |     closed      |
- *   | begin -> end | end -> capacity |
- *
- * - * \note This Template Overload is switched on only for POD Data Types and/or Non-Pair-Collectable User Defined DataTypes. - */ -template -class IterableCollector : public CollectableTreeNode -{ -public: - typedef typename IterableType::size_type size_type; - - /** - * \brief constructor - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param group Group this collector is part of - * \param index the index within the group - * \param desc Description of this node - * \param iterable Pointer to the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const std::string & group, - const uint32_t index, - const std::string & desc, - const IterableType * iterable, - const size_type expected_capacity) : - CollectableTreeNode(parent, name, group, index, desc), - iterable_object_(iterable), - expected_capacity_(expected_capacity), - event_set_(this), - ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", - CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)) - { - for (size_type i = 0; i < expected_capacity_; ++i) - { - std::stringstream name_str; - name_str << name << i; - positions_.emplace_back(new CollectableT(this, name_str.str(), group, i)); - } - } - - - /** - * \brief constructor - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param group Group this collector is part of - * \param index the index within the group - * \param desc Description of this node - * \param iterable the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const std::string & group, - const uint32_t index, - const std::string & desc, - const IterableType & iterable, - const size_type expected_capacity) : - IterableCollector(parent, name, group, index, desc, &iterable, expected_capacity) - {} - - /** - * \brief constructor - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param desc Description of this node - * \param iterable Pointer to the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const std::string & desc, - const IterableType * iterable, - const size_type expected_capacity) : - IterableCollector(parent, name, name, 0, desc, iterable, expected_capacity) - {} - - /** - * \brief constructor - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param desc Description of this node - * \param iterable the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const std::string & desc, - const IterableType & iterable, - const size_type expected_capacity) : - IterableCollector(parent, name, name, 0, desc, &iterable, expected_capacity) - {} - - /** - * \brief constructor with no description - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param iterable Pointer to the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const IterableType * iterable, - const size_type expected_capacity) : - IterableCollector (parent, name, name + " Iterable Collector", - iterable, expected_capacity) - { - // Delegated constructor - } - - /** - * \brief constructor with no description - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param iterable the iterable object to collect - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const IterableType & iterable, - const size_type expected_capacity) : - IterableCollector (parent, name, name + " Iterable Collector", - &iterable, expected_capacity) - { - // Delegated constructor - } - - /** - * \brief constructor with no iterable object associated - * \param parent the parent treenode for the collector - * \param name the name of the collector - * \param expected_capacity The maximum size this item should grow to - */ - IterableCollector (TreeNode * parent, - const std::string & name, - const size_type expected_capacity) : - IterableCollector (parent, name, name + " Iterable Collector", - nullptr, expected_capacity) - { - // Can't auto collect without setting iterable_object_ - setManualCollection(); - } - - //! Collect the contents of the iterable object. This function - //! will walk starting from index 0 -> expected_capacity, clearing - //! out any records where the iterable object does not contain - //! data. - void collect(const IterableType * iterable_object) - { - // If pointer has become nullified, close the records - if(nullptr == iterable_object) { - closeRecord(); - } - else if (SPARTA_EXPECT_FALSE(isCollected())) - { - if(SPARTA_EXPECT_FALSE(iterable_object->size() > expected_capacity_)) - { - if(SPARTA_EXPECT_FALSE(warn_on_size_)) - { - sparta::log::MessageSource::getGlobalWarn() - << "WARNING! The collected object '" - << getLocation() << "' has grown beyond the " - << "expected capacity (given at construction) for collection. " - << "Expected " << expected_capacity_ << " but grew to " - << iterable_object->size() - << " This is your first and last warning."; - warn_on_size_ = false; - } - } - collectImpl_(iterable_object, std::integral_constant()); - } - } - - //! Collect the contents of the associated iterable object - void collect() override { - collect(iterable_object_); - } - - //! Force close all records for this iterable type. This will - //! close the record immediately and clear the field for the next - //! cycle - void closeRecord(const bool & simulation_ending = false) override { - for (size_type i = 0; i < positions_.size(); ++i) { - positions_[i]->closeRecord(simulation_ending); - } - } - - //! Schedule event to close all records for this iterable type. - void scheduleCloseRecord(sparta::Clock::Cycle cycle) { - if(SPARTA_EXPECT_FALSE(isCollected())) { - ev_close_record_.preparePayload(false)->schedule(cycle); - } - } - - //! \brief Do not perform any automatic collection - //! The SchedulingPhase is ignored - void setManualCollection() { - auto_collect_ = false; - } - - //! \brief Perform a collection, then close the records in the future - //! \param duration The time to close the records, 0 is not allowed - void collectWithDuration(sparta::Clock::Cycle duration) { - if(SPARTA_EXPECT_FALSE(isCollected())) { - collect(); - if(duration != 0) { - ev_close_record_.preparePayload(false)->schedule(duration); - } - } - } - - //! Reattach to a new iterable object (used for moves) - void reattach(const IterableType * obj) { - iterable_object_ = obj; - } - -private: - typedef Collectable::value_type> CollectableT; - // Standard walk of iterable types - void collectImpl_(const IterableType * iterable_object, std::false_type) - { - sparta_assert(nullptr != iterable_object, - "Can't collect iterable_object because it's a nullptr! How did we get here?"); - auto itr = iterable_object->begin(); - auto eitr = iterable_object->end(); - for (uint32_t i = 0; i < expected_capacity_; ++i) - { - if (itr != eitr) { - positions_[i]->collect(*itr); - ++itr; - } else { - positions_[i]->closeRecord(); - } - } - } - - // Full iteration walk, checking validity of the iterator. This - // is used for Pipe and Array where the iterator points to valid - // and not valid entries in the component - void collectImpl_(const IterableType * iterable_object, std::true_type) - { - sparta_assert(nullptr != iterable_object, - "Can't collect iterable_object because it's a nullptr! How did we get here?"); - uint32_t s = 0; - auto itr = iterable_object->begin(); - for (uint32_t i = 0; i < expected_capacity_; ++i, ++s) - { - if (itr.isValid()) { - positions_[i]->collect(*itr); - } else { - positions_[i]->closeRecord(); - } - ++itr; - } - } - - //! Virtual method called by CollectableTreeNode when collection - //! is enabled on the TreeNode - void setCollecting_(bool collect, Collector * collector) override - { - PipelineCollector * pipeline_col = dynamic_cast(collector); - sparta_assert(pipeline_col != nullptr); - - if(collect && auto_collect_) { - // Add this Collectable to the PipelineCollector's - // list of objects requiring collection - pipeline_col->addToAutoCollection(this, collection_phase); - } - else { - closeRecord(); - } - } - - const IterableType * iterable_object_; - std::vector> positions_; - const size_type expected_capacity_ = 0; - bool auto_collect_ = true; - bool warn_on_size_ = true; - - // For those folks that want a value to automatically - // disappear in the future - sparta::EventSet event_set_; - sparta::PayloadEvent ev_close_record_; - -}; -} // namespace collection -} // namespace sparta diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 8995789bf0..a1e44fe505 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -15,649 +15,171 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/collection/CollectableTreeNode.hpp" -#include "sparta/collection/Collector.hpp" +#include "sparta/collection/CollectionPoints.hpp" #include "sparta/events/EventSet.hpp" #include "sparta/events/UniqueEvent.hpp" #include "sparta/events/GlobalOrderingPoint.hpp" #include "sparta/kernel/Scheduler.hpp" - -#include "sparta/pipeViewer/Outputter.hpp" -#include "sparta/pipeViewer/ClockFileWriter.hpp" -#include "sparta/pipeViewer/LocationFileWriter.hpp" #include "sparta/simulation/TreeNodePrivateAttorney.hpp" -namespace sparta{ +#include "simdb/collection/Scalars.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" + +namespace sparta { namespace collection { - /** - * \class PipelineCollector - * - * \brief A class that facilitates all universal pipeline - * collection operations such as outputting finalized - * records, generating unique transaction Id's, maintaining - * heartbeat functionality, writing the location file, - * writing the clock file. - * - * The class must be initialized with - * PipelineCollector::getPipelineCollector()->init(...) before - * collection is to occure. This method is required to have - * important parameters required during pipeline collection. - * - * This class operates on a specific scheduler specified at construction. - * It's implementation should not access the sparta Scheduler singleton - * directly. - * - * destroy() should also be called at the end of the programs life - * to perform post collection maintenance. Any transactions alive - * at this point will have their current data written to disk with - * the end time as the end time of simulation. If the singleton - * was never initialized, destroy will have no effect. - * - * Once the singleton is created and initialized with init - * pipeline collection still needs to be switched on. This can be - * done via the startCollection() at the treenode level. This - * method recursively turns on collection at and below the - * treenode pointer passed in. - * - * Likewise collection can be turned off via stopCollection at any - * time. - */ - class PipelineCollector : public Collector + class PipelineCollector { - class CollectablesByClock - { - public: - CollectablesByClock(const Clock * clk, - SchedulingPhase collection_phase) : - ev_set_(nullptr) - { - switch(collection_phase) - { - case SchedulingPhase::Trigger: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_trigger", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::Update: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_update", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::PortUpdate: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_portupdate", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::Flush: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_flush", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::Collection: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_collection", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::Tick: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_tick", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::PostTick: - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_posttick", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); - break; - case SchedulingPhase::__last_scheduling_phase: - sparta_assert(!"Should not have gotten here"); - break; - // NO DEFAULT! Allows for compiler errors if the enum - // class is updated. - } - ev_collect_->setScheduleableClock(clk); - ev_collect_->setScheduler(clk->getScheduler()); - ev_collect_->setContinuing(false); - } - - void enable(CollectableTreeNode * ctn) { - enabled_ctns_.insert(ctn); - // Schedule collect event in the next cycle in case - // this is called in an unavailable pphase. - ev_collect_->schedule(sparta::Clock::Cycle(1)); - } - - void disable(CollectableTreeNode * ctn) { - enabled_ctns_.erase(ctn); - } - - bool anyCollected() const { - return !enabled_ctns_.empty(); - } - - void performCollection() { - // std::for_each(enabled_ctns_.begin(), - // enabled_ctns_.end(), [&](CollectableTreeNode * it) { - // std::cout << it->getName() << " " << ev_collect_.getClock()->currentCycle() << std::endl; - // }); - - for(auto & ctn : enabled_ctns_) { - if(ctn->isCollected()) { - ctn->collect(); - } - } - if(!enabled_ctns_.empty()) { - ev_collect_->schedule(); - } - } - - void print() { - for(auto ctn : enabled_ctns_) { - std::cout << '\t' << ctn->getName() << std::endl; - } - } - private: - EventSet ev_set_; - - std::unique_ptr ev_collect_; - std::set enabled_ctns_; - }; - - // A map of the clock pointer and the structures that - // represent collectables on that clock. - std::map, - sparta::NUM_SCHEDULING_PHASES>> clock_ctn_map_; - - // Registered collectables - std::set registered_collectables_; - public: - - /** - * \brief Instantiate the collector with required parameters before - * pipeline collection can occur. - * \param filepath The relative path to the output directory or file prefix. - * \param heartbeat_interval The interval offset at which to write an - * index file (in Ticks). If 0, the heartbeat will be derived from the - * known clocks in the simulation. - * \param root_clk A pointer to the clock that maintains the hyper - * cycle time. - * \param root A pointer to the root sparta::TreeNode during - * simulation which will be walked when producing - * a location map file that maps location id - * numbers with the sparta tree location for - * collected objects. - * \param scheduler Scheduler on which this collector operates. If null, - * uses the Scheduler belonging to root_clk. This allows us to - * run different pipeline collectors on different schedulers - * which is useful for standalone analyzers which run multiple - * instances of some model in parallel over the same trace data - * - * \warning If filepath is a directory, the directory must - * already exist. - * \pre The sparta tree must be finalized. - * \note This method does NOT start collection. To start - * collection, call startCollection - * - */ - PipelineCollector(const std::string& filepath, Scheduler::Tick heartbeat_interval, - const sparta::Clock* root_clk, const sparta::TreeNode* root, - Scheduler* scheduler=nullptr) : - Collector("PipelineCollector"), - scheduler_(scheduler != nullptr ? scheduler : root_clk->getScheduler()), - collector_events_(nullptr), - ev_heartbeat_(&collector_events_, Collector::getName() + "_heartbeat_event", - CREATE_SPARTA_HANDLER(PipelineCollector, performHeartBeat_), 0) + PipelineCollector(const std::string& simdb_filename, + const std::set& enabled_nodes, + const size_t heartbeat, + sparta::RootTreeNode * rtn, + const sparta::CounterBase * insts_retired_counter) + : db_mgr_(simdb_filename, true) + , filename_(simdb_filename) + , root_(rtn) + , ev_set_(nullptr) + , insts_retired_counter_(insts_retired_counter) { - // Sanity check - pipeline collection cannot occur without a scheduler - sparta_assert(scheduler_); - - ev_heartbeat_.setScheduleableClock(root_clk); - ev_heartbeat_.setScheduler(scheduler_); - - // We need to reverse the dependency order for the PostTick GOP and ev_heartbeat_ so that every other - // event happens *before* ev_heartbeat_. - // Doing this ensures that we don't accidentally mark a 1-cycle transaction as continued. - DAG* dag = scheduler_->getDAG(); // Get a handle to the DAG - sparta_assert(dag); - Vertex* post_tick_gop = dag->getGOPoint("PostTick"); // Get a handle to the PostTick GOP - sparta_assert(post_tick_gop); - - dag->unlink(ev_heartbeat_.getVertex(), post_tick_gop); // Undo the ev_heartbeat_ >> heartbeat_gop link - post_tick_gop->precedes(ev_heartbeat_); // Set post_tick_gop >> heartbeat_gop - - sparta_assert(root != nullptr, "Pipeline Collection will not be able to create location file because it was passed a nullptr root treenode."); - sparta_assert(root->isFinalized(), "Pipeline collection cannot be constructed until the sparta tree has been finalized."); - // Assert that we got valid pointers necessary for pipeline collection. - sparta_assert(root_clk != nullptr, "Cannot construct PipelineCollector because root clock is a nullptr"); - sparta_assert(scheduler_->isFinalized() == false, "Pipeline Collection cannot be instantiated after scheduler finalization -- it creates events"); - - // Initialize the clock/collectable map and find the fastest clock - const sparta::Clock* fastest_clk = nullptr; - std::function addClks; - addClks = [&addClks, this, &fastest_clk] (const sparta::Clock* clk) - { - if(clk != nullptr){ - auto & u_p = clock_ctn_map_[clk]; - for(uint32_t i = 0; i < NUM_SCHEDULING_PHASES; ++i) { - u_p[i].reset(new CollectablesByClock(clk, static_cast(i))); - } - for(const sparta::TreeNode* child : sparta::TreeNodePrivateAttorney::getAllChildren(clk)) { - const sparta::Clock* child_clk = dynamic_cast(child); - if(child_clk){ - auto clk_period = child_clk->getPeriod(); - // If this clock has a non-1 period (i.e. not the root clock) - // AND there is either - // (A) no fastest clock yet - // or - // (B) this clock is a higher frequency than the fastest clock. - // then choose this as the fastest - if(clk_period != 1 && (!fastest_clk || (clk_period < fastest_clk->getPeriod()))) { - fastest_clk = child_clk; - } - addClks(child_clk); - } - } - } - }; - addClks(root_clk); - if(fastest_clk == nullptr){ - fastest_clk = root_clk; - } - - // A multiple to multiply against the fastest clock when no heartbeat was set. - //! \todo The multiplier should also be scaled slightly by the number of locations - //! registered in order to better estimate the ideal heartbeat size. - static const uint32_t heartbeat_multiplier = 200; - // round heartbeat using a multiple of the fastest clock if one was not set. - if (heartbeat_interval == 0) - { - sparta_assert(fastest_clk->getPeriod() != 0); - heartbeat_interval = fastest_clk->getPeriod() * heartbeat_multiplier; //round up to the nearest multiple of 100 - heartbeat_interval = heartbeat_interval + (100 - (heartbeat_interval % 100)); - // pipeViewer requires that intervals be a multiple of 100. - sparta_assert(heartbeat_interval % 100 == 0) + // Note that we only care about the collection data and have + // no need for any other tables, aside from the tables that the + // DatabaseManager adds automatically to support this feature. + simdb::Schema schema; + db_mgr_.createDatabaseFromSchema(schema); + + std::function func_ptr = std::bind(&PipelineCollector::getCollectionTick_, this); + db_mgr_.getCollectionMgr()->useTimestampsFrom(func_ptr); + + if (insts_retired_counter_) { + std::function ipc_func = std::bind(&PipelineCollector::getIPC_, this); + db_mgr_.getCollectionMgr()->enableArgosIPC(ipc_func); } - // We are gonna subtract one from the heartbeat_interval - // later.. Better be greater than one. - sparta_assert(heartbeat_interval > 1); - // Initialize some values. - filepath_ = filepath; - heartbeat_interval_ = heartbeat_interval; - closing_time_ = heartbeat_interval; - root_clk_ = root_clk; - - ev_heartbeat_.setContinuing(false); // This event does not keep simulation going + db_mgr_.getCollectionMgr()->setHeartbeat(heartbeat); + createCollections_(enabled_nodes); } ~PipelineCollector() { - // XXX This is a little goofy looking. Should we make sparta_abort that takes conditional - // be sparta_abort_unless()? - sparta_abort(collection_active_ != true, - "The PipelineCollector was not torn down properly. Before " - "tearing down the simulation tree, you must call " - "destroy() on the collector"); + db_mgr_.closeDatabase(); } - /*! - * \brief Teardown the pipeline collector - * - * Tear down the PipelineCollector. Should be called before - * Tree teardown to close all open transactions. - */ - void destroy() - { - if(collection_active_) { - sparta_assert(writer_ != nullptr, "Somehow collection is active, but we have a null writer"); - for(auto & ctn : registered_collectables_) { - if(ctn->isCollected()) { - ctn->closeRecord(true); // set true for simulation termination - } - } - } - registered_collectables_.clear(); - writer_.reset(); - collection_active_ = false; + //! \return the pipeout file path + const std::string & getFilePath() const { + return filename_; } - void reactivate(const std::string& filepath) - { - sparta_assert(!collection_active_, - "You can only reactivate the PipelineCollector after you destroy it"); - filepath_ = filepath; + void startCollecting() { + // Schedule collect event in the next cycle in case + // this is called in an unavailable phase. + ev_collect_->schedule(sparta::Clock::Cycle(1)); } - /** - * \brief Turn on collection for everything below a TreeNode. - * Recursively transverse the tree and turn on child - * nodes for pipeline collection. - * - * \param starting_node TreeNode to start collection on. This - * TreeNode will try to start collection - * as well as any node below it. - * - * \note The Scheduler MUST be finalized before this method is - * called - */ - void startCollection(sparta::TreeNode* starting_node) - { - if(collection_active_ == false) - { - // Create the outputter used for writing transactions to disk. - writer_.reset(new pipeViewer::Outputter(filepath_, heartbeat_interval_)); - - // We need to write an index on the start BEFORE any transactions have been written. - writer_->writeIndex(); - - // Write the clock information out - writeClockFile_(); - - // Open the locations file - location_writer_.reset(new pipeViewer::LocationFileWriter(filepath_)); - - // The reader needs heartbeat indexes up to the current - // collection point. This can happen on delayed pipeline - // collection. Start the heartbeats at the first interval - // as the Outputter class handles hb 0 - last_heartbeat_ = 0; - const uint64_t num_hb = scheduler_->getCurrentTick()/heartbeat_interval_; - uint64_t cnt = 0; - while(cnt != num_hb) { - // write an index - writer_->writeIndex(); - last_heartbeat_ += heartbeat_interval_; - ++cnt; - } + void stopCollecting() { + db_mgr_.onPipelineCollectorClosing(); + db_mgr_.getConnection()->getTaskQueue()->stopThread(); + db_mgr_.closeDatabase(); + } - // Schedule a heartbeat at the next interval, offset from - // the current tick. For example, if the scheduler is at - // 6,600,456, then we want to schedule at 7,000,000 if the - // interval is 1M and the num_hb is 6 - ev_heartbeat_.scheduleRelativeTick(((num_hb + 1) * heartbeat_interval_) - - scheduler_->getCurrentTick(), scheduler_); + private: + uint64_t getCollectionTick_() const { + return scheduler_->getCurrentTick() - 1; + } - collection_active_ = true; - } + void createCollections_(const std::set& enabled_nodes) { + CollectionPoints collectables; + recurseAddCollectables_(root_, collectables, enabled_nodes); + collectables.createCollections(db_mgr_.getCollectionMgr()); + db_mgr_.finalizeCollections(); - *(location_writer_.get()) << (*starting_node); - - // Recursively collect the start node and children - std::function recursiveCollect; - recursiveCollect = [&recursiveCollect, this] (sparta::TreeNode* starting_node) - { - // First turn on this node if it's actually a CollectableTreeNode - CollectableTreeNode* c_node = dynamic_cast(starting_node); - if(c_node != nullptr) { - c_node->startCollecting(this); - registered_collectables_.insert(c_node); - } + recurseFindFastestCollectableClock_(root_, fastest_clk_, enabled_nodes); + sparta_assert(fastest_clk_, "No clock found! Are there any collectables?"); - // Recursive step. Go through the children and turn them on as well. - for(sparta::TreeNode* node : sparta::TreeNodePrivateAttorney::getAllChildren(starting_node)) - { - recursiveCollect(node); - } - }; + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(fastest_clk_)->getName() + "_auto_collection_event_collection", + CREATE_SPARTA_HANDLER(PipelineCollector, performCollection_), 1)); - recursiveCollect(starting_node); + ev_collect_->setScheduleableClock(fastest_clk_); + ev_collect_->setScheduler(fastest_clk_->getScheduler()); + ev_collect_->setContinuing(false); + scheduler_ = fastest_clk_->getScheduler(); + sparta_assert(scheduler_, "Cannot run pipeline collection without a scheduler"); } - /** - * \brief Stop pipeline collection on only those - * CollectableTreeNodes given - * \param starting_node The node to shut collection down on - * - */ - void stopCollection(sparta::TreeNode* starting_node) + void recurseAddCollectables_(sparta::TreeNode * node, + CollectionPoints & collectables, + const std::set& enabled_nodes) { - std::function recursiveStopCollect; - recursiveStopCollect = [&recursiveStopCollect, this] (sparta::TreeNode* starting_node) { - // First turn off this node if it's actually a CollectableTreeNode - CollectableTreeNode* c_node = dynamic_cast(starting_node); - if(c_node != nullptr) - { - c_node->stopCollecting(this); - registered_collectables_.erase(c_node); - } - - // Recursive step. Go through the children and turn them on as well. - for(sparta::TreeNode* node : sparta::TreeNodePrivateAttorney::getAllChildren(starting_node)) - { - recursiveStopCollect(node); + if (auto c = dynamic_cast(node)) { + if (enabled_nodes.empty() || enabled_nodes.count(c->getLocation())) { + c->addCollectionPoint(collectables); } - }; - recursiveStopCollect(starting_node); - - bool still_active = !registered_collectables_.empty(); - // for(auto & cp : clock_ctn_map_) { - // if(cp.second->anyCollected()) { - // still_active = true; - // break; - // } - // } - collection_active_ = still_active; - } - - /** - * \brief Stop pipeline collection on only those - * CollectableTreeNodes that this PipelineCollector - * was started with - */ - void stopCollection() { - for(auto & col : registered_collectables_) { - col->stopCollecting(this); } - registered_collectables_.clear(); - } - - /** - * \brief Add the CollectableTreeNode to auto collection - * \param ctn The CollectableTreeNode that is to be collected - * \param collection_phase The phase to collect the object in - * - * Enable collection on the given CollectableTreeNode. This - * is a runtime call. There are some rules here: - * - * #. The Scheduler must be finialized and simulation started - * #. The clock that the CollectableTreeNode belongs to must - * have been registered with the PipelineCollector at init time. - */ - void addToAutoCollection(CollectableTreeNode * ctn, - SchedulingPhase collection_phase = SchedulingPhase::Tick) - { - auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); - sparta_assert(ccm_pair != clock_ctn_map_.end()); - ccm_pair->second[static_cast(collection_phase)]->enable(ctn); - } - - /** - * \brief Remove the given CollectableTreeNode from collection - * \param ctn The CollectableTreeNode that is to be removed from collection - * - * Disable collection on the given CollectableTreeNode. This - * is a runtime call. There are some rules here: - * - * #. The Scheduler must be finialized and simulation started - * #. The clock that the CollectableTreeNode belongs to must - * have been registered with the PipelineCollector at init time. - */ - void removeFromAutoCollection(CollectableTreeNode * ctn) - { - auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); - sparta_assert(ccm_pair != clock_ctn_map_.end()); - for(auto & u_p : ccm_pair->second) { - u_p->disable(ctn); + for (auto & child : node->getChildren()) { + recurseAddCollectables_(child, collectables, enabled_nodes); } } - /** - * \brief Return a unique transaction id using a dummy counter - */ - uint64_t getUniqueTransactionId() - { - // make sure we are not going to overflow our int, - // if we did overflow our id's are no longer unique! - sparta_assert(last_transaction_id_ < (~(uint64_t)0)); - return ++last_transaction_id_; - } - - /** - * \brief Output a finized transaction to our Outputter class. - * \param dat The transaction to be outputted. - * \param R_Type the type of transaction struct - */ - template - void writeRecord(const R_Type& dat) - { - sparta_assert(collection_active_, "The pipeline head must be running in order to write a transaction"); - - // Make sure it's within the heartbeat window - sparta_assert(dat.time_End <= (last_heartbeat_ + heartbeat_interval_)); - sparta_assert(dat.time_Start >= last_heartbeat_); - - - // Make sure transactions are exclusive. - // transaction [4999-5000] should be written BEFORE the index is written for transactions - // starting at 5000 - sparta_assert(closing_time_ < heartbeat_interval_ // Ignore first heartbeat - || dat.time_Start >= closing_time_ - heartbeat_interval_, - "Attempted to write a pipeout record with exclusive start =(" - << dat.time_Start << "), less than closing of previous interval" - << closing_time_ - heartbeat_interval_ ); - - // std::cout << "writing annt. " << "loc: " << dat.location_ID << " start: " - // << dat.time_Start << " end: " << dat.time_End - // << " parent: " << dat.parent_ID << std::endl; - - writer_->writeTransaction(dat); - ++transactions_written_; - } - - /** - * \brief Return the number of transactions that this singleton - * has passed to it's output. This is useful for testing purposes. - */ - uint64_t numTransactionsWritten() const - { - return transactions_written_; - } - - /** - * \brief Return true if the collector is actively collecting - * - * Will be true if there are any registered collectables that - * are being collected on any clock. - * - * \note This method should not be used to determine whether - * pipeline collection is running at a specific tree - * node location. - */ - bool isCollectionActive() const { - return collection_active_; - } - - void printMap() { - //std::cout << "Printing Map Not Supported" << std::endl; - // for(auto & p : clock_ctn_map_) { - // std::cout << "\nClock : " << p.first->getName() - // << "\nAuto collectables: " << std::endl; - // p.second->print(); - // } - } - - //! \return the pipeout file path - const std::string & getFilePath() const { - return filepath_; - } - - //! \return the scheduler for this collector - Scheduler* getScheduler() const { - return scheduler_; - } - - private: - - /** - * \brief Write the clock file based off of a pointer to the root clock, - * that was established in the parameters of startCollection - */ - void writeClockFile_() - { - // We only need the ClockFileWriter to exist during the writing of the clock file. - // there for it was created on the stack. - //std::cout << "Writing Pipeline Collection clock file. " << std::endl; - pipeViewer::ClockFileWriter clock_writer(filepath_); - clock_writer << (*root_clk_); - } - - //! Perform a heartbeat on the collector. This is required to - //! enable the writing of an index file used by the pipeout - //! reader for fast access - void performHeartBeat_() + void recurseFindFastestCollectableClock_(const sparta::TreeNode * node, + const Clock *& fastest_clk, + const std::set& enabled_nodes) { - if(collection_active_) { - // Close all transactions - for(auto & ctn : registered_collectables_) { - if(ctn->isCollected()) { - ctn->restartRecord(); + if (auto c = dynamic_cast(node)) { + if (!enabled_nodes.empty() && !enabled_nodes.count(c->getLocation())) { + return; + } + if (c->getClock() != nullptr) { + if (fastest_clk == nullptr) { + fastest_clk = c->getClock(); + } else { + if (c->getClock()->getPeriod() < fastest_clk->getPeriod()) { + fastest_clk = c->getClock(); + } } } + } - // write an index - writer_->writeIndex(); - - // Remember the last time we recorded a heartbeat - last_heartbeat_ = scheduler_->getCurrentTick(); - - // Schedule another heartbeat - ev_heartbeat_.schedule(heartbeat_interval_); + for (auto & child : node->getChildren()) { + recurseFindFastestCollectableClock_(child, fastest_clk, enabled_nodes); } } - //! A pointer to the outputter class used for writing - //! transactions to physical disk. - std::unique_ptr writer_; - - //! A pointer to the root sparta TreeNode of the - //! simulation. Important for writing the location map file. - std::unique_ptr location_writer_; - //sparta::TreeNode* collected_treenode_ = nullptr; - - //! Pointer to the root clock. This clock is considered the - //! hyper-clock or the clock with the hypercycle - const sparta::Clock * root_clk_ = nullptr; - - //! The filepath/prefix for writing pipeline collection files too - std::string filepath_; + void performCollection_() { + db_mgr_.getCollectionMgr()->collectAll(); + ev_collect_->schedule(); + } - //! Keep track of the last transaction id given to ensure that - //! each transaction is written with a unique id - uint64_t last_transaction_id_ = 0; + double getIPC_() const + { + auto tick = scheduler_->getCurrentTick(); + auto cycle = fastest_clk_->getCycle(tick); + auto num_retired = insts_retired_counter_->get(); + return static_cast(num_retired) / static_cast(cycle); + } - //! The number of transactions written to disk so far - uint64_t transactions_written_ = 0; + //! The SimDB database + simdb::DatabaseManager db_mgr_; - //! The number of ticks between heart beats. Also the offset - //! between index pointer writes in the Outputter - uint64_t heartbeat_interval_ = 0; + //! The SimDB filename e.g. "pipeline.db" + std::string filename_; - //! The last heartbeat we recorded - Scheduler::Tick last_heartbeat_ = 0; + //! The root tree node + sparta::RootTreeNode * root_ = nullptr; - //! Scheduler on which this collector operates - Scheduler * scheduler_; + //! The event set for the performCollection_() callback event + EventSet ev_set_; - //! Event and EventSet for performing heartbeats - EventSet collector_events_; - UniqueEvent ev_heartbeat_; + //! The unique event for the performCollection_() callback + std::unique_ptr ev_collect_; - //! The time that the next heartbeat will occur - uint64_t closing_time_ = 0; + //! The simulation scheduler + const sparta::Scheduler * scheduler_; - //! Is collection enabled on at least one node? - bool collection_active_ = false; + //! The "total instruction retired" counter + const sparta::CounterBase * insts_retired_counter_; + //! Fastest clock across all collectable tree nodes + const Clock * fastest_clk_ = nullptr; }; }// namespace collection diff --git a/sparta/sparta/control/TemporaryRunController.hpp b/sparta/sparta/control/TemporaryRunController.hpp index 34b8da1eff..c2c6b118ac 100644 --- a/sparta/sparta/control/TemporaryRunController.hpp +++ b/sparta/sparta/control/TemporaryRunController.hpp @@ -4,7 +4,6 @@ #include "sparta/kernel/SpartaHandler.hpp" #include "sparta/kernel/Scheduler.hpp" -#include "simdb_fwd.hpp" namespace sparta { class Clock; @@ -17,10 +16,6 @@ namespace sparta { class StreamController; } - namespace async { - class AsynchronousTaskEval; - } - namespace control { /*! @@ -56,8 +51,6 @@ class TemporaryRunControl { std::shared_ptr & getStreamController(); - void setDatabaseTaskThread(simdb::AsyncTaskEval * db_thread); - uint64_t getCurrentCycle(const std::string& clk_name) const; uint64_t getCurrentCycle(const sparta::Clock* clk=nullptr) const; @@ -117,11 +110,6 @@ class TemporaryRunControl { //! Statistics stream controller. Used for starting/stopping listener //! objects, and forcing data flushes. std::shared_ptr stream_controller_; - - //! Database task thread. This is given to us so that we can - //! force synchronization points during simulation pause (end - //! of a runi/runc command, etc.) - simdb::AsyncTaskEval * db_task_thread_ = nullptr; }; } // namespace control diff --git a/sparta/sparta/kernel/Scheduler.hpp b/sparta/sparta/kernel/Scheduler.hpp index 24d7d84590..7379afddd0 100644 --- a/sparta/sparta/kernel/Scheduler.hpp +++ b/sparta/sparta/kernel/Scheduler.hpp @@ -45,6 +45,10 @@ namespace sparta { class GlobalTreeNode; + +namespace collection { + class PipelineCollector; +} } // namespace sparta diff --git a/sparta/sparta/pairs/SpartaKeyPairs.hpp b/sparta/sparta/pairs/SpartaKeyPairs.hpp index c212f22e88..62502aff42 100644 --- a/sparta/sparta/pairs/SpartaKeyPairs.hpp +++ b/sparta/sparta/pairs/SpartaKeyPairs.hpp @@ -30,8 +30,6 @@ #include "sparta/utils/MetaTypeList.hpp" #include "sparta/utils/MetaStructs.hpp" #include "sparta/utils/DetectMemberUtils.hpp" -#include "simdb/utils/CompatUtils.hpp" - namespace sparta { @@ -1693,7 +1691,7 @@ namespace sparta { template MetaStruct::enable_if_t< std::is_integral>::value && - simdb::utils::is_pod>::value && + MetaStruct::is_pod>::value && !MetaStruct::is_bool>::value, void> updateValueInCache_( diff --git a/sparta/sparta/pipeViewer/ClockFileWriter.hpp b/sparta/sparta/pipeViewer/ClockFileWriter.hpp deleted file mode 100644 index d6c3b16d01..0000000000 --- a/sparta/sparta/pipeViewer/ClockFileWriter.hpp +++ /dev/null @@ -1,132 +0,0 @@ -// -*- C++ -*- - -/** - * \file ClockfileWriter.hpp - * - * \brief Contains Clockfilewriter class - */ - -#pragma once - -#include -#include -#include - -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/simulation/Clock.hpp" -#include "sparta/simulation/TreeNodePrivateAttorney.hpp" -namespace sparta{ -namespace pipeViewer -{ - /** - * \brief Object capable of writing file with clock info entries for a given - * device tree for consumption by the pipeViewer viewer - * - * File format is a version line: - * \verbatim - * - * \endverbatim - * (where version is ClockFileWriter::VERSION) - * - * followed by 1 line showing hypercycle (tick) frequency in Hz: - * \verbatim - * - * \endverbatim - * - * Followed by any number of single-line entries; 1 per each clock in the - * the simulation (in no particular order) each having the form: - * \verbatim - * ,,,,\n - * \endverbatim - * Note that the newline (\n) will be present on every line - * - * Lines beginning with '#' as the first character are comments - */ - class ClockFileWriter - { - public: - - /*! - * \brief Default Constructor - * \param prefix Prefix of file to which data will be written. - */ - ClockFileWriter(const std::string& prefix, - const std::string& fn_extension = "clock.dat", - const uint32_t fmt_version = 1) : - file_(prefix + fn_extension, std::ios::out) - { - if(!file_.is_open()){ - throw sparta::SpartaException("Failed to open clock file \"") - << prefix + fn_extension << "\" for write"; - } - - // Throw on write failure - file_.exceptions(std::ostream::eofbit | std::ostream::badbit | std::ostream::failbit | std::ostream::goodbit); - - file_ << fmt_version << " # Version Number" << std::endl; - } - - - /*! - * \brief Writes content of an entire clock tree with the given root to - * this clocks file. - * \param root Root node of clock tree to write to the file - * \warning There is no need to write each node individually. - * \warnings Any nodes inserted with this operator more than once will - * cause repeat entries - */ - ClockFileWriter& operator<<(const sparta::Clock& clk) { - - //! \todo Need to extract a frequency (in Hz) from clk - file_ << 1 << " # Tick frequency" << std::endl; - recursWriteClock_(&clk); - file_.flush(); - return *this; - } - - private: - - /*! - * \brief Recursively writes an entry for the given clock and all - * children in pre-order. - * \param node Node for which entriy should be written. Children will - * then be iterated. - * \post May write any number of lines to the output file - * depending on number of nodes and each node's name, - * group info, and aliases. - */ - void recursWriteClock_(const sparta::Clock* clk) - { - sparta_assert(clk != nullptr); - - file_ << clk->getNodeUID() << ',' - << clk->getName() << ',' - << clk->getPeriod() << ',' - << clk->getRatio().getNumerator() << ',' - << clk->getRatio().getDenominator() << std::endl; - - // Recurse into children - for(const sparta::TreeNode* child : sparta::TreeNodePrivateAttorney::getAllChildren(clk)) { - const sparta::Clock* child_clk = dynamic_cast(child); - if(child_clk){ - recursWriteClock_(child_clk); - } - } - } - - /*! - * \brief Filename with which this file was actually opened (includes - * prefix given at construction) - */ - std::string filename_; - - /*! - * \brief File to which clock data will be written. - */ - std::ofstream file_; - - }; // class ClockFileWriter - -}//namespace pipeViewer -}//namespace sparta - diff --git a/sparta/sparta/pipeViewer/InformationWriter.hpp b/sparta/sparta/pipeViewer/InformationWriter.hpp deleted file mode 100644 index 0613e26b93..0000000000 --- a/sparta/sparta/pipeViewer/InformationWriter.hpp +++ /dev/null @@ -1,79 +0,0 @@ -// -*- C++ -*- - -/** - * \file InformationWriter.hpp - * \brief Define the Pipeline Collection Information outputter. - */ -#pragma once - -#include -#include "sparta/utils/SpartaException.hpp" - -namespace sparta{ - /** - * \class InformationWriter - * \brief A class that allows the simulation developer to - * write data to an information file stored near pipeline collection - * output files about the simulation. - * - * Human readible data can be written to file via the << operator or - * public writeLine() methods. - */ - class InformationWriter - { - public: - - /** - * \brief construct a InformationWriter stream. - * \param file The name of the file + path to use as the output stream. - */ - InformationWriter(const std::string& file) : - file_(file, std::ios::out) - { - if(!file_.is_open()){ - throw SpartaException("Failed to open InformationWriter file for file: " + file); - } - - // Throw on write failure - file_.exceptions(std::ostream::eofbit | std::ostream::badbit | std::ostream::failbit | std::ostream::goodbit); - } - virtual ~InformationWriter() - { - file_.close(); - } - - /** - * \brief Allow appending to the file via the << operator. - * \param object the data to write to the file - */ - template - InformationWriter& operator << (const T& object) - { - file_ << object; - return *this; - } - /** - * \brief Allow the user to write a string line to the file. - * \param str the string to be written as a line in the file. - */ - template - void writeLine(const T& str) - { - file_ << str << std::endl; - } - /** - * \brief Allow the user to write some stuff to the file. - * This will not break at the end of the line. - */ - template - void write(const T& str) - { - file_ << str; - } - - private: - std::ofstream file_; /*!< an outstream to write to file with */ - - }; - -} diff --git a/sparta/sparta/pipeViewer/LocationFileWriter.hpp b/sparta/sparta/pipeViewer/LocationFileWriter.hpp deleted file mode 100644 index c6c25bfdce..0000000000 --- a/sparta/sparta/pipeViewer/LocationFileWriter.hpp +++ /dev/null @@ -1,191 +0,0 @@ -// -*- C++ -*- - -/** - * \file LocationFileWriter.hpp - * - * \brief Contains LocationFileWriter class - */ - -#pragma once - -#include -#include -#include - -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/simulation/Clock.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" -#include "sparta/simulation/TreeNodePrivateAttorney.hpp" -namespace sparta{ -namespace pipeViewer -{ - /** - * \brief Object capable of writing a file with Location entries for a given - * device tree for consumption by the pipeViewer viewer - * - * File format is a version line: - * \verbatim - * - * \endverbatim - * (where version is LocationFileWriter::VERSION) - * - * followed by any number of single-line entries (1 or more for each node - * in the simualation) having the form: - * \verbatim - * ,,\n - * \endverbatim - * Note that the newline (\n) will be present on every line - * - * For nodes with no clock association, NO_CLOCK_ID is used as \a clock_id - * - * Lines beginning with '#' as the first character are comments - * - * Multiple entries can be written for the same node if that node has - * aliases or group information. This is intended to make that node - * easier to identify in the viewer. However, this is only done for - * the rightmost object in each entry location string. Ancestor - * nodes of the node whose entry is being writtenwill be printed using the - * - * \todo Support all alias and group identifiers for each node in a - * location, not just the last (rightmost) object in each location - * printed - * \todo Only write locations being collected to the file - */ - class LocationFileWriter - { - /*! - * \brief ID to write to disk when a location has no associated clock. - */ - const int NO_CLOCK_ID = -1; - - public: - - /*! - * \brief Default Constructor - * \param prefix Prefix of file to which data will be written. - */ - LocationFileWriter(const std::string& prefix, - const std::string& fn_extension = "location.dat", - const uint32_t fmt_version = 1) : - filename_(prefix + fn_extension), - file_(filename_, std::ios::out) - { - if(!file_.is_open()){ - throw sparta::SpartaException("Failed to open location file \"") - << filename_ << "\" for write"; - } - - // Throw on write failure - file_.exceptions(std::ostream::eofbit | std::ostream::badbit | std::ostream::failbit | std::ostream::goodbit); - - file_ << fmt_version << " # Version Number" << std::endl; - } - - /*! - * \brief Writes content of an entire tree with the given root to this - * location file. - * \param root Root node of tree to write to the file - * \warning There is no need to write each node individually. - * \warnings Any nodes added more than once will cause repeat entries - */ - LocationFileWriter& operator<<(const sparta::TreeNode& root) { - recursWriteNode_(&root); - file_.flush(); - return *this; - } - - private: - - /*! - * \brief Writes an 1-line entry for 1 node using the given location - * string (which might not math node->getLocation) - * \param node Node to write an entry for. UID and clock association are - * extracted from this - * \param location Location string to write to file. This should - * identify the node but might be an alias, the actual name, or a - * group[index] string. Must be fully qualified (e.g. - * top.node0.node1.thisnode) - */ - void writeNodeEntry_(const sparta::TreeNode* node, const std::string& location) { - sparta_assert(node != nullptr); - - file_ << node->getNodeUID() << ',' << location << ','; - - const sparta::Clock* clk = node->getClock(); - if(clk){ - file_ << clk->getNodeUID(); - }else{ - file_ << NO_CLOCK_ID; // No clock - } - - file_ << "\n"; - } - - /*! - * \brief Recursively writes entries for all nodes in the subtree - * including this \a node in pre-order - * \param node Node for which entries should be written. Children will - * then be iterated. - * \post May write any number of lines to the output file depending on - * number of nodes and each node's name, group info, and aliases. - */ - void recursWriteNode_(const sparta::TreeNode* node) { - sparta_assert(node != nullptr); - - // Write node only if it is a collectable - if(dynamic_cast(node)){ - - // Compute a valid location representing the parent - std::string parent_loc; - if(node->getParent() != nullptr){ - parent_loc = node->getParent()->getDisplayLocation(); - } - - // Write Node with its given name string - if(node->getName() != ""){ - writeNodeEntry_(node, node->getLocation()); - } - - // Write node with its group info if appropriate: "group_name[idx]" - if(node->getGroup() != "" && node->getGroup() != sparta::TreeNode::GROUP_NAME_BUILTIN) { - std::stringstream group_el_ident; - group_el_ident << parent_loc - << sparta::TreeNode::LOCATION_NODE_SEPARATOR_ATTACHED - << node->getGroup() << "[" << node->getGroupIdx() << "]"; - - writeNodeEntry_(node, group_el_ident.str()); - } - - // Write node with each alias this node has - for(const std::string& alias : node->getAliases()){ - std::stringstream alias_ident; - alias_ident << parent_loc - << sparta::TreeNode::LOCATION_NODE_SEPARATOR_ATTACHED - << alias; - - writeNodeEntry_(node, alias_ident.str()); - } - } // if(dynamic_cast(node)) - - // Recurse into children regardless of whether this node was collectable - for(const sparta::TreeNode* child : sparta::TreeNodePrivateAttorney::getAllChildren(node)){ - recursWriteNode_(child); - } - } - - /*! - * \brief Filename with which this file was actually opened (includes - * prefix given at construction) - */ - std::string filename_; - - /*! - * \brief File to which location data will be written. - */ - std::ofstream file_; - - }; // class LocationFileWriter - -}//namespace pipeViewer -}//namespace sparta - diff --git a/sparta/sparta/pipeViewer/Outputter.hpp b/sparta/sparta/pipeViewer/Outputter.hpp deleted file mode 100644 index 6d40210c88..0000000000 --- a/sparta/sparta/pipeViewer/Outputter.hpp +++ /dev/null @@ -1,262 +0,0 @@ -// -*- C++ -*- - -/** - * \file Outputter.hpp - ** \brief Outputs Transactions to record file and builds index file while running * - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/TupleHashCompute.hpp" - -namespace sparta::pipeViewer -{ - - - /** - * \class Outputter - * @ brief A class that facilitates taking in Record objects and writing - * them to the record file, and building the index as it goes. - * - * The index file is a list of uint64_t pointers into the record - * file for the first transaction that ended at a multiple of a - * standard offset, such as there is an index for every "interval" - * of cycles. - * - * (Note) the first entry in the index file will always be the interval amount - * (Note) the last entry in the index file will always point to the last record - * written to file. - * - */ - class Outputter - { - /** - * \brief Safely write data to a file with error checking. - */ - void writeData_(std::ofstream& ss, const char* const data, const std::size_t size) - { - ss.write(data, size); - } - - template - void writeData_(std::ofstream& ss, const T* const data, const std::size_t size = sizeof(T)) - { - writeData_(ss, reinterpret_cast(data), size); - } - - template - void writeData_(std::ofstream& ss, const T& data) - { - writeData_(ss, &data, sizeof(T)); - } - - public: - - /*! - * \brief File format version written by this outputter. - * This must be incremented on any change to the transaction type - * \note If you are incrementing this, be sure pipeViewer::Reader is up to - * date and backward compatible - */ - static constexpr uint32_t FILE_VERSION = 2; - - /** - * \brief Construct an Outputter - * \param file_path the path to the folder to store output files. - * \param interval The number of cycles between indexes - */ - Outputter(const std::string& filepath, const uint64_t interval); - - /** - * \brief Close the Record file and the index file. - * also writes an index pointing to the end of the record file. - */ - virtual ~Outputter(); - - /** - * \brief Write a generic transaction to the record file, and update the index file. - * \param dat The transaction to be written. - */ - template - void writeTransaction(const R_Type& dat) - { - last_record_pos_ = record_file_.tellp(); -#ifdef PIPELINE_DBG - std::cout << "writing transaction at: " << last_record_pos_ << " TMST: " - << dat.time_Start << " TMEN: " << dat.time_End << std::endl; -#endif - writeData_(record_file_, dat); - } - - /** - * \brief A method that marks a pointer to the record file's current location. - * This method will likely be set to run on the schedular on a given interval. - */ - void writeIndex(); - private: - std::ofstream record_file_; /*!< The record file contains the actual transaction data */ - std::ofstream index_file_; /*!< The file stream for index file being created */ - std::ofstream map_file_; /*!< The file stream for map file which maps Location ID to Pair ID */ - std::ofstream data_file_;/*! The file stream for data file which contains Name, Size, Pair number */ - std::ofstream string_file_;/*! The file stream for string_map file which has the String representation */ - std::ofstream display_format_file_; - - uint64_t last_record_pos_; /*!< A pointer to the last record written */ - }; - - /*! - * \brief Write an annotation transaction to the record file. - * notice that an annotation needs special attention in order - * to take care of outputting ANNT data itself properly - * \warning this deletes the memory for ANNT in the dat passed - */ - template<> - inline void Outputter::writeTransaction(const annotation_t& dat) - { - - writeTransaction(static_cast(dat)); - writeData_(record_file_, dat.length); - writeData_(record_file_, dat.annt.data(), dat.length); - } - - /*! - * \brief Write a pair transaction to the record file. - */ - template<> - inline void Outputter::writeTransaction(const pair_t & dat){ - - // A static Unordered Set which contains unique location id of the records. - static std::unordered_set locIDSet; - - // A static Unordered Set which contains unique pair id of the records. - static std::unordered_set pairIDSet; - - using multiIndex = std::tuple; - - // A static Unordered Map with a Tuple of 3 Integers as Key and a String as Value. - // This map is used to store the String mappings from the Intermediate Integer values. - // We use integers as this makes the database smaller and also very fast to write to Binary File. - // The first Integer in the May Key is the unique Pair they bleong to. - // The Second Integer is the Field number they belong to. - // The Third Integer is the actual Integral value which corresponds to the String value. - static std::unordered_map> stringMap; - - // If we find a Location ID that we have not seen before, we store it in the Location ID set. - if(locIDSet.emplace(dat.location_ID).second) { - // We add the Location Id followed by the Pair Id of that record in the map file. - map_file_ << dat.location_ID << ':' << dat.pairId << '\n'; - } - - // If we find a Pair ID we have not seen before, we store it in the Pair ID set. - if(pairIDSet.emplace(dat.pairId).second) { - // We write the Pair ID to the data file followed by the Number of pairs - // this kind of pair collectable contains. - // The first pair of every pair record is its PairID, so we do not add that to the database. - data_file_ << dat.pairId << ':' << dat.length; - - // We write the generic transaction structure to the record file. - writeTransaction(static_cast(dat)); - - // We iterate over all the name value pairs of the current pair record. - for(std::size_t i = 0; i < dat.length; ++i) { - data_file_ << ':'; - if(dat.valueVector[i].second){ - // We add the Name String followed by the Sizeof in the data file. - data_file_ << dat.nameVector[i] << ':' << dat.sizeOfVector[i] << ":0"; - - // We write the Value for field "i" and only write as much Bytes - // as it needs to by checking Sizes[i]. - writeData_(record_file_, - &dat.valueVector[i].first, - dat.sizeOfVector[i]); - - // We check if the value at field "i" has any String Representation. - // If it has, then its corresponding string vector field will not be empty. - if(!dat.stringVector[i].empty()){ - // We check if we have seen this exact pair, field and value before or not. - if(const auto& [val, str] = std::tie(dat.valueVector[i].first, dat.stringVector[i]); - stringMap.emplace(std::piecewise_construct, std::forward_as_tuple(dat.pairId, i, val), std::forward_as_tuple(str)).second) { - // We add this mapping into out String Map file which we will - // use when reading back from the database. - string_file_ << dat.pairId - << ':' << i - << ':' << val - << ':' << str << '\n'; - } - } - } - else{ - // We add the Name String followed by the Sizeof in the data file. - data_file_ << dat.nameVector[i] << ":0:1"; - - // We write the Value for field "i" and only write as much Bytes - // as it needs to by checking Sizes[i]. - const auto& str = dat.stringVector[i]; - const uint16_t length = str.size(); - writeData_(record_file_, length); - writeData_(record_file_, str.data(), length); - } - } - data_file_ << '\n'; - display_format_file_ << dat.pairId; - for(const auto& fmt: dat.delimVector) { - display_format_file_ << ':' << static_cast(fmt); - } - display_format_file_ << '\n'; - } - - // If we find a Pair ID we have seen before, - // we do not store it in the Pair ID set and move forward with just writing it to file. - else{ - - // We write the generic transaction structure to the record file. - writeTransaction(static_cast(dat)); - - // We iterate over all the name value pairs of the current pair record. - for(std::size_t i = 0; i < dat.length; ++i){ - if(dat.valueVector[i].second){ - // We write the Value for field "i" and only write as much Bytes - // as it needs to by checking Sizes[i]. - writeData_(record_file_, - &dat.valueVector[i].first, - dat.sizeOfVector[i]); - - // We check if the value at field "i" has any String Representation. - // If it has, then its corresponding string vector field will not be empty. - if(!dat.stringVector[i].empty()){ - // We check if we have seen this exact pair, field and value before or not. - if(const auto& [val, str] = std::tie(dat.valueVector[i].first, dat.stringVector[i]); - stringMap.emplace(std::piecewise_construct, std::forward_as_tuple(dat.pairId, i, val), std::forward_as_tuple(str)).second) { - // We add this mapping into out String Map file which we will - // use when reading back from the database. - string_file_ << dat.pairId - << ':' << i - << ':' << val - << ':' << str << '\n'; - } - } - } - else{ - // We write the Value for field "i" and only write as much Bytes - // as it needs to by checking Sizes[i]. - const auto& str = dat.stringVector[i]; - const uint16_t length = str.size(); - writeData_(record_file_, length); - writeData_(record_file_, str.data(), length); - } - } - } - } -}//namespace sparta::pipeViewer diff --git a/sparta/sparta/pipeViewer/transaction_structures.hpp b/sparta/sparta/pipeViewer/transaction_structures.hpp deleted file mode 100644 index 9a0c38827e..0000000000 --- a/sparta/sparta/pipeViewer/transaction_structures.hpp +++ /dev/null @@ -1,149 +0,0 @@ -/** - * \file transaction_structures.hpp - * - */ - -#pragma once - -#include -#include -#include -#include - -#include "sparta/pairs/PairFormatter.hpp" - -#define is_Annotation 0x1 -#define is_Instruction 0x2 -#define is_MemoryOperation 0x3 -#define is_Pair 0x4 -#define TYPE_MASK 0x7//!< Mask used for extracting type ID portion from transaction flags -#define CONTINUE_FLAG 0x10 //!< Flag used to indicate if this transaction should be considered a continuation of the previous transaction - -static constexpr uint64_t BAD_DISPLAY_ID = 0x1000; - -static constexpr std::string_view HEADER_PREFIX = "sparta_pipeout_version:"; -static constexpr int VERSION_LENGTH = 4; -static constexpr size_t HEADER_SIZE = HEADER_PREFIX.size() + VERSION_LENGTH + 1; // prefix + number + newline - -/*! - * \brief Generic transaction event, packed for density on disk - * \warning Since this is written as a chunk, it is endian-dependent and - * possibly compiler dependent - * \todo Should consider removing packing attribute since it is mainly related - * to serialization. Create custom packed serialization code instead. - */ -struct __attribute__ ((aligned(8))) transaction_t { - uint64_t time_Start = 0; //! Event Start Time 8 Bytes - uint64_t time_End = 0; //! Event End Time 8 Bytes - uint64_t parent_ID = 0; //! Parent Transaction ID 8 Bytes - uint64_t transaction_ID = 0; //! Transaction ID 8 Bytes - - // Any value above 0x0fff is an invalid value for this field - uint64_t display_ID = BAD_DISPLAY_ID; //! Display ID 8 Bytes - - uint32_t location_ID = 0; //! Location 4 Bytes - uint16_t flags = 0; //! Flags/Trans Type 2 Bytes - uint16_t control_Process_ID = 0; //! Control Process ID 2 Bytes - - transaction_t() = default; - - //! Parameterized Constructor - transaction_t(uint64_t time_Start, uint64_t time_End, uint64_t parent_ID, - uint64_t transaction_ID, uint64_t display_ID, uint32_t location_ID, uint16_t flags, - uint16_t control_Process_ID) : - time_Start(time_Start), time_End(time_End), parent_ID(parent_ID), - transaction_ID(transaction_ID), display_ID(display_ID), location_ID(location_ID), flags(flags), - control_Process_ID(control_Process_ID) {} - -}; - -// Instruction Event -struct instruction_t : public transaction_t { - uint32_t operation_Code = 0; //! Operation Code 4 Bytes - uint64_t virtual_ADR = 0; //! Virtual Address 8 Bytes - uint64_t real_ADR = 0; //! Real Address 8 Bytes - - instruction_t() = default; -}; - -// Memory Operation Event -struct memoryoperation_t : public transaction_t { - uint64_t virtual_ADR = 0; //! 8 Bytes - uint64_t real_ADR = 0; //! 8 Bytes - - memoryoperation_t() = default; -}; - -// Annotation Event (Catch-All) -struct annotation_t : public transaction_t { - uint16_t length = 0; //! Annotation Length 2 Bytes - std::string annt; //! Pointer to Annotation Start - - annotation_t() = default; - - explicit annotation_t(transaction_t&& rhs) : - transaction_t(std::move(rhs)) - { - } -}; - -// Name Value Pair Event -struct pair_t : public transaction_t { - // Number of pairs contained in this record 2 Bytes. - uint16_t length {0}; - - // The Unique pair id for every Name-Value class collected. - uint16_t pairId {0}; - - // Vector of 2 Byte unsigned ints which contains the - // sizeofs of every different pair value in a record - std::vector sizeOfVector; - - // Vector of 8 Byte unsigned ints which contains the - // actual value or the Integral representation of the - // actual values of every Name string in a record. - // We only store these values in the database. - using IntT = uint64_t; - using ValidPair = std::pair; - std::vector valueVector; - - // Vector of the different Name Strings in a record. - std::vector nameVector; - - // Vector of the actual String Values which we need to - // lookup while displaying in pipeViewer - // If a field value has no string representation, - // the enum vector field is empty at that position. - std::vector stringVector; - - sparta::PairFormatterVector delimVector; - - // The default constructor suffices for this structure. - // No Move Constructor needed for this structure as there - // is no older version of such a structure. - pair_t() = default; - - explicit pair_t(transaction_t&& rhs) : - transaction_t(std::move(rhs)) - { - } - - //! Parameterized Constructor - pair_t(const uint64_t time_Start, - const uint64_t time_End, - const uint64_t parent_ID, - const uint64_t transaction_ID, - const uint64_t display_ID, - const uint32_t location_ID, - const uint16_t flags, - const uint16_t control_Process_ID) : - transaction_t( - time_Start, - time_End, - parent_ID, - transaction_ID, - display_ID, - location_ID, - flags, - control_Process_ID) {} -}; diff --git a/sparta/sparta/ports/DataPort.hpp b/sparta/sparta/ports/DataPort.hpp index 5e2c364908..56bd02e802 100644 --- a/sparta/sparta/ports/DataPort.hpp +++ b/sparta/sparta/ports/DataPort.hpp @@ -18,7 +18,6 @@ #include "sparta/events/PayloadEvent.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/Scheduleable.hpp" -#include "sparta/collection/Collectable.hpp" namespace sparta { @@ -288,7 +287,8 @@ namespace sparta class DataInPort final : public InPort, public DataContainer { // Pipeline collection type - typedef collection::Collectable CollectorType; + // TODO cnyce + // typedef collection::Collectable CollectorType; public: @@ -445,8 +445,7 @@ namespace sparta * \param node The TreeNode to add the collector */ void enableCollection(TreeNode* node) override { - collector_.reset(new CollectorType(node, Port::name_, 0, - "Data being received on this DataInPort")); + (void) node; } private: @@ -530,8 +529,9 @@ namespace sparta //! The receiving clock const Clock * receiver_clock_ = nullptr; - /// Pipeline collection - std::unique_ptr collector_; + //! Pipeline collection + //! TODO cnyce + //! std::unique_ptr collector_; //! Data receiving point void receivePortData_(const DataT & dat) @@ -540,11 +540,6 @@ namespace sparta if(SPARTA_EXPECT_TRUE(explicit_consumer_handler_)) { explicit_consumer_handler_((const void*)&dat); } - if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { - if(SPARTA_EXPECT_FALSE(collector_->isCollected())) { - collector_->collect(dat); - } - } } //! This port's additional delay for receiving the data diff --git a/sparta/sparta/ports/SyncPort.hpp b/sparta/sparta/ports/SyncPort.hpp index 3a64f08237..29064b9af9 100644 --- a/sparta/sparta/ports/SyncPort.hpp +++ b/sparta/sparta/ports/SyncPort.hpp @@ -72,7 +72,6 @@ #include "sparta/simulation/TreeNode.hpp" #include "sparta/utils/DataContainer.hpp" -#include "sparta/collection/DelayedCollectable.hpp" #include "sparta/ports/Port.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/EventSet.hpp" @@ -111,7 +110,8 @@ namespace sparta template class SyncOutPort final : public OutPort { - typedef collection::DelayedCollectable CollectorType; + // TODO cnyce + // typedef collection::DelayedCollectable CollectorType; public: //! A typedef for the type of data this port passes. @@ -303,10 +303,6 @@ namespace sparta uint64_t sched_delay_ticks = sync_in_port_->send_(dat, clk_, send_delay_cycles, allow_slide, is_fwd_progress); - if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { - collector_->collectWithDuration(dat, send_delay_cycles, 1); - } - sparta_assert(send_cycle > prev_data_send_cycle_ || prev_data_send_cycle_ == PREV_DATA_SEND_CYCLE_INIT, //init tick 0 getLocation() << ": trying to send at cycle " @@ -375,8 +371,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - collector_.reset(new CollectorType(node, Port::name_, 0, - "Data being sent out on this SyncOutPort")); + (void) node; } private: @@ -387,8 +382,9 @@ namespace sparta /// The in-port all data will be sent to SyncInPort * sync_in_port_ = nullptr; - /// Pipeline collection: TODO - figure out how this works w/ sync ports - std::unique_ptr collector_; + /// Pipeline collection + /// TODO cnyce + /// std::unique_ptr collector_; /// Last cycle any data was sent Clock::Cycle PREV_DATA_SEND_CYCLE_INIT = 0xffffffffffffffff; //init tick 0 @@ -410,7 +406,9 @@ namespace sparta template class SyncInPort final : public InPort, public DataContainer { - typedef collection::Collectable CollectorType; + //! TODO cnyce + //! typedef collection::Collectable CollectorType; + public: //! Expected typedef for DataT typedef DataT DataType; @@ -593,8 +591,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - collector_.reset(new CollectorType(node, Port::name_, 0, - "Data being recirculated on this SyncInPort")); + (void) node; } //! Set the ready state for the port before simulation begins @@ -809,12 +806,6 @@ namespace sparta explicit_consumer_handler_((const void*)&dat); } - // Show the data that has arrived on this OutPort that - // the receiver now sees - if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { - collector_->collectWithDuration(dat, 1); - } - if (SPARTA_EXPECT_FALSE(info_logger_)) { info_logger_ << "DELIVERING @" << receiver_clock_->currentCycle() << "(" << num_in_flight_ << ") " @@ -1060,8 +1051,9 @@ namespace sparta //! only call setReady() once per cycle. Scheduler::Tick set_ready_tick_ = 0; - //! Pipeline collection. TODO: See if this works for syncports - std::unique_ptr collector_; + //! Pipeline collection + //! TODO cnyce + //! std::unique_ptr collector_; /// loggers sparta::log::MessageSource info_logger_; diff --git a/sparta/sparta/report/DatabaseInterface.hpp b/sparta/sparta/report/DatabaseInterface.hpp deleted file mode 100644 index 7522bf9e27..0000000000 --- a/sparta/sparta/report/DatabaseInterface.hpp +++ /dev/null @@ -1,387 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "sparta/app/Simulation.hpp" -#include "sparta/simulation/RootTreeNode.hpp" -#include "sparta/simulation/TreeNode.hpp" -#include "simdb/schema/DatabaseRoot.hpp" -#include "simdb_fwd.hpp" - -#include - -namespace sparta { -namespace trigger { - class ExpressionTrigger; -} -class SubContainer; - -/*! - * \class DatabaseAccessor - * \brief There is a 1-to-1 mapping between a running simulation - * and the database it is using. Some components in the simulator - * may have database access, while others are not intended to use - * the database. This is controlled via command line arguments, and - * the simulation's DatabaseAccessor knows which components are DB- - * enabled, and which are not. - */ -class DatabaseAccessor -{ -public: - //! Simulations typically will instantiate their DatabaseAccessor - //! relative to the root tree node. During simulation, TreeNode's - //! which ask the question "am I enabled for database access" will - //! always be answered FALSE if they are not a child under this - //! RootTreeNode. - //! - //! \param rtn RootTreeNode at the top of a device tree owned by - //! an app::Simulation object - explicit DatabaseAccessor(RootTreeNode * rtn) : - root_(rtn) - { - if (all_simulation_accessors_.insert(rtn).second) { - if (static_simdb_accessor_invoked_ && - all_simulation_accessors_.size() > 1) - { - throw SpartaException("More than one DatabaseAccessor has been ") - << "created, which indicates there may be more than one " - << "simulation running concurrently. When this is the case, " - << "the GET_DB_FROM_CURRENT_SIMULATION macro cannot be used."; - } - } - } - - //! Check enabled status for any TreeNode - //! \param tn TreeNode 'this' pointer at the call site where - //! database access is being queried - bool isEnabled( - const std::string & db_namespace, - const TreeNode * tn) const - { - if (!tn) { - return false; - } - - const utils::lowercase_string dbns = db_namespace; - auto & enabled_components = enabled_components_[dbns]; - auto & implicitly_disabled_components = implicitly_disabled_components_[dbns]; - if (implicitly_disabled_components.count(tn) > 0) { - return false; - } - - auto mark_implicitly_disabled = [&implicitly_disabled_components, tn](const bool enabled) { - if (!enabled) { - implicitly_disabled_components.insert(tn); - } - return enabled; - }; - - TreeNode * child = nullptr; - TreeNode * search_node = nullptr; - const std::string loc = tn->getLocation(); - if (root_->hasChild(loc)) { - child = root_->getChild(loc); - search_node = root_; - } else if (root_->getSearchScope()->hasChild(loc)) { - child = root_->getSearchScope()->getChild(loc); - search_node = root_->getSearchScope(); - } else { - return mark_implicitly_disabled(false); - } - - return mark_implicitly_disabled( - expandEnabledComponentsWildcards_( - search_node, child, enabled_components)); - } - - //! Check enabled status for any simulation - //! \param sim Simulation 'this' pointer at the call site where - //! database access is being queried - bool isEnabled(const app::Simulation * sim) const { - if (!sim) { - return false; - } - if (root_ != sim->getRoot()) { - return false; - } - return (sim->getDatabaseRoot() != nullptr); - } - - //! Check enabled status from any call site which does - //! not fit one of the other isEnabled() overloads. - //! Returns the SimDB object if enabled, nullptr if - //! not. - static simdb::ObjectManager::ObjectDatabase * isEnabled( - const std::string & db_namespace) - { - static_simdb_accessor_invoked_ = true; - if (all_simulation_accessors_.size() == 1) { - if (auto sim = (*all_simulation_accessors_.begin())->getSimulation()) { - if (auto db_root = sim->getDatabaseRoot()) { - if (auto root_namespace = db_root->getNamespace(db_namespace)) { - return root_namespace->getDatabase(); - } - } - } - } - return nullptr; - } - -private: - //! Command line arguments pick and choose which components - //! should be database-enabled, which results in calls into - //! this method before simulation starts. This method is - //! intended to only be callable by the Simulation object. - //! - //! \param loc Location of the TreeNode where database access - //! has been enabled ("top.core0.rob", etc.) - void enableComponentAtLocation_( - const std::string & db_namespace, - const std::string & loc) - { - std::string dbns_(db_namespace); - boost::trim(dbns_); - - std::string loc_(loc); - boost::trim(loc_); - enabled_components_[dbns_].insert(loc_); - } - - //! Expand any simdb-enabled-components that were given - //! with wildcards in them. For example, say we had the - //! following: - //! - //! \code - //! TreeNode * search_node = _global - //! TreeNode * requesting_node = top.cpu.core0.rob - //! enabled_components = { "top.cpu.core0.r*" } - //! \endcode - //! - //! Strictly using string comparison would not find any - //! matches, since "top.cpu.core0.rob" != "top.cpu.core0.r*" - //! but if we ask the search node to find all of its - //! children (not necessarily *immediate* children) - //! for anything matching "top.cpu.core0.r*" we would - //! end up with this (at the time of this writing, - //! sparta_core_example only had these matches): - //! - //! \code - //! enabled_components = { - //! "top.cpu.core0.regs", - //! "top.cpu.core0.rename", - //! "top.cpu.core0.rob" - //! } - //! \endcode - //! - //! Let's update the simdb-enabled-components list to - //! account for any wildcards now. - bool expandEnabledComponentsWildcards_( - TreeNode * search_node, - TreeNode * requesting_node, - std::unordered_set & enabled_components) const - { - if (!requesting_node) { - return false; - } - - if (enabled_components.count(requesting_node->getLocation())) { - return true; - } - - if (!search_node) { - return false; - } - - std::vector all_matching_nodes; - for (const std::string & component : enabled_components) { - std::vector matching_nodes; - search_node->findChildren(component, matching_nodes); - - all_matching_nodes.insert(all_matching_nodes.end(), - matching_nodes.begin(), - matching_nodes.end()); - } - - const std::unordered_set unique_matching_nodes( - all_matching_nodes.begin(), - all_matching_nodes.end()); - - std::unordered_set expanded_components; - for (auto tn : unique_matching_nodes) { - expanded_components.insert(tn->getLocation()); - } - - std::swap(expanded_components, enabled_components); - return enabled_components.count(requesting_node->getLocation()) > 0; - } - - //! \class AccessTrigger - //! \brief Utility class which turns trigger expressions - //! into invocable SpartaHandler's, and informs the owning - //! DatabaseAccessor object when a schema namespace has - //! just become available or unavailable for reads and - //! writes via the TableProxy class objects. - class AccessTrigger - { - public: - AccessTrigger( - DatabaseAccessor * db_accessor, - const std::string & db_namespace, - const std::string & start_expr, - const std::string & stop_expr, - RootTreeNode * rtn, - std::shared_ptr & sub_container); - - private: - void grantAccess_() { - db_accessor_->grantAccess_(db_namespace_); - } - - void revokeAccess_() { - db_accessor_->revokeAccess_(db_namespace_); - } - - std::shared_ptr start_; - std::shared_ptr stop_; - DatabaseAccessor * db_accessor_ = nullptr; - std::string db_namespace_; - }; - - //! This method is called when a SimDB namespace has - //! just become available for reads and writes via - //! the TableProxy class objects. - void grantAccess_(const std::string & db_namespace) { - if (auto db = isEnabled(db_namespace)) { - db->grantAccess(); - } - } - - //! This method is called when a SimDB namespace has - //! just become unavailable for reads and writes via - //! the TableProxy class objects. - void revokeAccess_(const std::string & db_namespace) { - if (auto db = isEnabled(db_namespace)) { - db->revokeAccess(); - } - } - - //! Set various access options from a YAML file specifying: - //! : - //! components: - //! - //! - //! ... - //! : - //! components: - //! - //! - //! ... - //! start: - //! stop: - void setAccessOptsFromFile_(const std::string & opt_file); - - friend class AccessTrigger; - friend class app::Simulation; - RootTreeNode * root_ = nullptr; - - //Member variables for isEnabled() calls from component - //code which has a 'this' pointer compatible with the - //various isEnabled() overloads. - mutable std::unordered_map< - std::string, - std::unordered_set> enabled_components_; - mutable std::unordered_map< - std::string, - std::unordered_set> implicitly_disabled_components_; - std::vector> access_triggers_; - std::shared_ptr sub_container_; - - //Member variables for isEnabled() calls from component - //code which does not have a 'this' pointer matching one - //of the isEnabled() overloads. - static std::unordered_set all_simulation_accessors_; - static bool static_simdb_accessor_invoked_; -}; - -/*! - * \brief Base struct for DatabaseInterface template class - */ -template -struct DatabaseInterface; - -/*! - * \brief All TreeNode objects or TreeNode subclass objects use - * this method to determine if they are database-enabled. - */ -template -struct DatabaseInterface::value>::type> -{ - static simdb::ObjectManager::ObjectDatabase * dbEnabled( - const std::string & db_namespace, - T * tn) - { - if (!tn) { - return nullptr; - } - if (auto sim = tn->getSimulation()) { - if (auto db_accessor = sim->getSimulationDatabaseAccessor()) { - if (db_accessor->isEnabled(db_namespace, tn)) { - if (auto db_root = sim->getDatabaseRoot()) { - if (auto root_namespace = db_root->getNamespace(db_namespace)) { - return root_namespace->getDatabase(); - } - } - } - } - } - return nullptr; - } -}; - -/*! - * \brief All Simulation objects and Simulation subclass objects - * use this method to determine if they are database-enabled. - */ -template -struct DatabaseInterface::value>::type> -{ - static simdb::ObjectManager::ObjectDatabase * dbEnabled( - const std::string & db_namespace, - T * sim) - { - if (!sim) { - return nullptr; - } - if (auto db_accessor = sim->getSimulationDatabaseAccessor()) { - if (db_accessor->isEnabled(sim)) { - if (auto db_root = sim->getDatabaseRoot()) { - if (auto root_namespace = db_root->getNamespace(db_namespace)) { - return root_namespace->getDatabase(); - } - } - } - } - return nullptr; - } -}; - -//! This macro lets callers request the SimDB object from a variety -//! of simulation contexts. -#define GET_DB_FOR_COMPONENT(db_namespace, thisptr) \ - sparta::DatabaseInterface::type>::dbEnabled(#db_namespace, thisptr) - -//! This is similar to the macro above, but is intended to be used -//! from a simulation context that does not have an appropriate 'this' -//! pointer for the GET_DB_FOR_COMPONENT macro. It can also be used -//! in code that *does* subclass from TreeNode/etc. but for that -//! particular call site, you *always* want SimDB access. -//! -//! This is effectively singleton access, with safety checks in place -//! to ensure that the static getter can be called safely. -#define GET_DB_FROM_CURRENT_SIMULATION(db_namespace) \ - sparta::DatabaseAccessor::isEnabled(#db_namespace) - -} - diff --git a/sparta/sparta/report/Report.hpp b/sparta/sparta/report/Report.hpp index 7e79336738..a41e2b7658 100644 --- a/sparta/sparta/report/Report.hpp +++ b/sparta/sparta/report/Report.hpp @@ -22,7 +22,6 @@ #include "sparta/utils/SpartaAssert.hpp" #include "sparta/simulation/TreeNodePrivateAttorney.hpp" #include "sparta/trigger/ExpressionTrigger.hpp" -#include "simdb_fwd.hpp" namespace sparta { @@ -34,11 +33,6 @@ namespace sparta } } - class StatInstRowIterator; - namespace db { - class DatabaseContextCounter; - } - /*! * \brief Collection of optionally-named StatisticInstances and other * (sub)reports. @@ -210,10 +204,7 @@ namespace sparta start_tick_(rhp.start_tick_), end_tick_(rhp.end_tick_), info_string_(rhp.info_string_), - sub_statistics_(rhp.sub_statistics_), - si_row_iterator_(rhp.si_row_iterator_), - report_node_id_(rhp.report_node_id_), - si_node_ids_(rhp.si_node_ids_) + sub_statistics_(rhp.sub_statistics_) { // Update parent pointers of all subreports for(auto& sr : subreps_){ @@ -242,12 +233,8 @@ namespace sparta start_tick_(rhp.start_tick_), end_tick_(rhp.end_tick_), info_string_(std::move(rhp.info_string_)), - sub_statistics_(std::move(rhp.sub_statistics_)), - si_row_iterator_(std::move(rhp.si_row_iterator_)), - report_node_id_(rhp.report_node_id_), - si_node_ids_(std::move(rhp.si_node_ids_)) + sub_statistics_(std::move(rhp.sub_statistics_)) { - rhp.report_node_id_ = 0; // Update parent pointers of all subreports for(auto& sr : subreps_){ sr.setParent_(this); @@ -284,9 +271,6 @@ namespace sparta end_tick_ = rhp.end_tick_; info_string_ = rhp.info_string_; sub_statistics_ = rhp.sub_statistics_; - si_row_iterator_ = rhp.si_row_iterator_; - report_node_id_ = rhp.report_node_id_; - si_node_ids_ = rhp.si_node_ids_; return *this; } @@ -896,49 +880,6 @@ namespace sparta return sub_statistics_; } - /*! - * \brief Mapping from StatisticInstance's to their sub-StatisticInstance's - * (supports ContextCounter pseudo-recreation from SimDB records after - * simulation, where we do not actually have StatisticDef's, or even - * TreeNode's of any kind). - * - * To illustrate what this data structure represents, say that we had - * the following hierarchy in the original SPARTA simulation: - * - * Report - * SI (wraps a ContextCounter) - * internal_counters_[0] (wraps a CounterBase) - * internal_counters_[1] (wraps a CounterBase) - * - * The equivalent hierarchy when recreating the same report *after* - * a simulation looks like this: - * - * Report - * SI (is the root node of a DatabaseContextCounter) - * SI (is the first sub-statistic under it) - * SI (is the second sub-statistic under it) - */ - typedef - // ...the root node of a DatabaseContextCounter - std::unordered_map, - // ...the list of sub-statistics under it - std::vector> - > DBSubStatisticInstances; - - /*! - * \brief Get a SimDB-recreated Report's mapping from SI's to sub-statistic - * instances, if any - */ - const DBSubStatisticInstances & getDBSubStatistics() const { - if (report_node_id_ == 0) { - sparta_assert(db_sub_statistics_.empty()); - } - return db_sub_statistics_; - } - //////////////////////////////////////////////////////////////////////// //! @} @@ -1227,25 +1168,6 @@ namespace sparta } } - /*! - * \brief This utility can be used to generate a formatted - * report from a root-level ReportNodeHierarchy record in - * the provided database (ObjectManager). - * - * The given report_hier_node_id must have ParentNodeID=0 - * in the ReportNodeHierarchy table, or this method will - * throw an exception. - */ - static bool createFormattedReportFromDatabase( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID report_hier_node_id, - const std::string & filename, - const std::string & format, - const Scheduler * scheduler); - - //////////////////////////////////////////////////////////////////////// - //! @} - private: /*! @@ -1336,83 +1258,6 @@ namespace sparta void legacyDelayedStart_(const trigger::CounterTrigger * trigger); void legacyDelayedEnd_(const trigger::CounterTrigger * trigger); - /*! - * \brief Reconstruct a Report node from a database record ID in - * the provided SimDB. Throws if the given report hierarchy node - * ID is not found in this database. - * - * This private constructor is not meant to be invoked directly - * from the outside world. This would typically be called from - * from SimDB-related static sparta::Report methods. - */ - Report(const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr, - const Scheduler * scheduler); - - /*! - * \brief When we recreate a sparta::Report object from SimDB - * records, we need to put a few things in place which help - * us *directly* get our SI values from the database blob, - * since we are not even running an actual simulation. For - * the most part, SimDB-created SI's do not have any internals - * such as CounterBase/ParameterBase/StatisticDef pointers. - * They get their SI values from "StatInstValueLookup" objects, - * which are tied to "StatInstRowIterator" objects. The SI's - * own the value lookup objects, while the reports/subreports - * own the row iterator objects. Both of these objects work - * together like this: - * - * 1. Advance the row iterator to the next row of SI values. - * - * 2a. Ask the value lookup for your specific SI value. - * It knows your SI index, so it knows the element - * offset into the SI double vector, and it gives - * you the value. - * - * 2b. The value lookup objects are all bound to the row - * iterator's vector, which is itself bound - * to an ObjectQuery against one of the SimDB tables... - * - * 3. Call StatInstRowIterator::getNext() to advance the - * row iterator one more row in the database. This - * calls ObjectQuery::getNext(), which memcpy's the - * next SI blob into the row iterator's vector, - * decompressing the blob if needed. - * - * 4. All StatInstValueLookup objects that were bound to this - * row iterator object will be "updated automatically", since - * they are just using indirection to point somewhere else - * which actually has the current values. - */ - bool prepareForSIDatabaseIteration_( - const simdb::ObjectManager & obj_mgr); - - /*! - * \brief Starting at 'this' report node, recursively get - * all mappings from Report/SI database node ID to the - * Report or SI that lives at each node. - */ - void recursGetReportAndSINodeDatabaseIDs_( - std::unordered_map & report_nodes_by_id, - std::unordered_map & si_nodes_by_id); - - /*! - * \brief Starting at 'this' report node, find the first - * StatInstRowIterator member variable we encounter while - * traversing in a depth-first fashion. - */ - std::shared_ptr - recursFindTopmostSIRowIteratorPlaceholder_(); - - /*! - * \brief Set/reset/unset the StatInstRowIterator that is - * given to us. Passing in a null row iterator is the same - * thing as resetting this report's row iterator; it will - * not reject a null iterator object. - */ - void recursSetSIRowIterator_( - std::shared_ptr & si_row_iterator); - /*! * \brief Schedler associated with this report (for time-elapsed information) */ @@ -1508,41 +1353,6 @@ namespace sparta * \brief Flag for enabling auto-expansion of ContextCounter stats (off by default) */ bool auto_expand_context_counter_stats_ = false; - - //! \name SimDB-related variables - //! @{ - //////////////////////////////////////////////////////////////////////// - - /*! - * \brief Mapping from DB-recreated StatisticInstance's to their - * sub-statistic instances, if any. - */ - DBSubStatisticInstances db_sub_statistics_; - - /*! - * \brief This row iterator object is a wrapper around an - * ObjectQuery. It is used to get report/SI data values - * out of a SimDB, and into a formatted report (json_detail, - * html, text, etc.) - */ - std::shared_ptr si_row_iterator_; - - /*! - * \brief Cached database ID. Equals 0 for all Report objects - * created during simulation. Will be non-zero for Report objects - * recreated after a simulation from SimDB record(s). - */ - simdb::DatabaseID report_node_id_ = 0; - - /*! - * \brief Cached database ID(s) of SimDB-recreated StatisticInstance's - * that belong to this Report object. Will be empty for Report objects - * created during simulation. - */ - std::vector si_node_ids_; - - //////////////////////////////////////////////////////////////////////// - //! @} }; //! \brief Report stream operator diff --git a/sparta/sparta/report/db/DatabaseContextCounter.hpp b/sparta/sparta/report/db/DatabaseContextCounter.hpp deleted file mode 100644 index 3d1d86367b..0000000000 --- a/sparta/sparta/report/db/DatabaseContextCounter.hpp +++ /dev/null @@ -1,155 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include - -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/statistics/InstrumentationNode.hpp" - -namespace sparta { - -class Report; -class StatisticInstance; - -namespace db { - -/*! - * \brief This class mimics sparta::ContextCounter but is - * only used outside of a running simulation when producing - * report files from database records. - * - * Objects like Report and StatisticInstance are recreated - * during SimDB-driven report generation, *without* needing - * to actually build TreeNode's of any kind. Unlike simple - * StatisticInstance's which wrap Counters or Parameters, - * this class is more involved due to the fact that reports - * have very specific rules for how they write out their - * ContextCounter's to the formatted report file. - * - * In other words, DatabaseContextCounter is only relatively - * complex since its counterpart sparta::ContextCounter - * has relatively complex formatting rules, and SimDB-driven - * reports need to exactly match simulation-driven reports. - */ -class DatabaseContextCounter -{ -public: - //! This class will take care of writing out the contents - //! of the sub-statistic SI's. The caller who is using - //! one of these objects needs to be able to ask if a - //! particular SI has already been serialized. - using UnprintableSIs = std::unordered_set; - - //! Construction with the ContextCounter root SI and its - //! sub-SI's that live under the root. Here is what the - //! original simulation may have had: - //! - //! SI (StatisticDef) - //! SI (CounterBase) - //! SI (CounterBase) - //! - //! The corresponding DatabaseContextCounter would then be: - //! - //! SI (cc_node) - //! SI (unprintable_sis[*]) - //! SI (unprintable_sis[*]) - //! - //! Since we don't have a tree outside of a simulation, we - //! cannot create StatisticDef's or CounterBase's. Or at - //! least we *should not* have to recreate parts of the - //! original tree, since everything we need to know about - //! report/SI hierarchy, along with all metadata and SI - //! values, is right there in the database. Reconstructing - //! a tree using "real" components like StatisticDef / - //! StatisticSet / InstrumentationNode / ContextCounter - //! is very unnecessary overhead, and very cumbersome code - //! to maintain. - DatabaseContextCounter( - const StatisticInstance * cc_node, - std::shared_ptr & unprintable_sis); - - //! Analogous to TreeNode::getName() - const std::string & getName() const; - - //! Analogous to InstrumentationNode::groupedPrinting() - bool groupedPrinting( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const; - - //! Analogous to InstrumentationNode::groupedPrintingReduced() - bool groupedPrintingReduced( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const; - - //! Analogous to InstrumentationNode::groupedPrintingDetail() - bool groupedPrintingDetail( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const; - -private: - //! Analogous to ContextCounter::ContextCounterInfo - struct ContextCounterInfo { - std::string name_; - std::string desc_; - InstrumentationNode::visibility_t vis_ = 0; - double val_ = 0; - const void * ctx_addr_ = nullptr; - }; - mutable std::vector ctx_info_; - - //! Analogous to ContextCounter (StatisticDef) 'this' pointer - const StatisticInstance * cc_node_ = nullptr; - - //! Analogous to ContextCounter::internal_counters_ - std::shared_ptr unprintable_sis_; - - //! Analogous to TreeNode::getDesc() - std::string cc_desc_; - - //! Analogous to TreeNode::getName() - std::string cc_name_; - - //! Mimics ContextCounter::extractCtxInfo_ - void extractCtxInfo_( - const std::vector & sub_stats) const; - - //! Mimics __groupedPrinting() free function in sparta/ContextCounter.h - bool groupedPrinting_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info, - const std::string & aggregate_desc, - const InstrumentationNode::visibility_t aggregate_vis) const; - - //! Mimics __groupedPrintingReduced() free function in sparta/ContextCounter.h - bool groupedPrintingReduced_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info) const; - - //! Mimics __groupedPrintingDetail() free function in sparta/ContextCounter.h - bool groupedPrintingDetail_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info) const; - - //! At the end of the various "grouped printing" methods, tack on - //! any "unprintable SI(s)" into the "dont_print_these" sets. - void appendUnprintablesToSet_( - std::set & dont_print_these) const; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/ReportHeader.hpp b/sparta/sparta/report/db/ReportHeader.hpp deleted file mode 100644 index 2f3df812c7..0000000000 --- a/sparta/sparta/report/db/ReportHeader.hpp +++ /dev/null @@ -1,116 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include - -#include "simdb_fwd.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" - -namespace sparta { -namespace db { - -class ReportTimeseries; - -/*! - * \brief Wrapper around a database record (ObjectRef) - * which provides user-friendly API's to read and write - * report metadata in the database. - */ -class ReportHeader -{ -public: - //! Create a ReportHeader wrapper around an *existing* database record - explicit ReportHeader(std::unique_ptr obj_ref); - - //! Create a new ReportHeader object in the database - ReportHeader(const simdb::ObjectManager & obj_mgr); - - //! Get this record's unique database ID. Returns the - //! ID of the underlying ObjectRef this ReportHeader - //! wraps. - uint64_t getId() const; - - //! Get this record's ObjectRef. This is the same record you - //! would get if you asked us 'getId()', and gave that ID to - //! the ObjectManager to get that record for the "ReportHeader" - //! table. - const simdb::ObjectRef & getObjectRef() const; - simdb::ObjectRef & getObjectRef(); - - //! Some report headers are standalone database - //! records, but timeseries reports always have - //! header metadata that go with them. Calling - //! this method will make the table-to-table - //! connection (primary key / foreign key) that - //! will let you do this: - //! - //! ReportHeader header(...) - //! header.setReportStartTime(1500) - //! - //! ReportTimeseries timeseries(...) - //! header.setOwningTimeseries(timeseries) - //! ... - //! - //! timeseries.getHeader().getReportStartTime() - //! ^^^ Returns 1500, which was read from the - //! physical database, NOT from any member - //! variable in any object. - void setOwningTimeseries(const ReportTimeseries & ts); - - //! METADATA SETTERS --------------------------------- - void setReportName( - const std::string & report_name); - - void setReportStartTime( - const uint64_t start_time); - - void setReportEndTime( - const uint64_t end_time); - - void setSourceReportDescDestFile( - const std::string & fname); - - void setCommaSeparatedSILocations( - const std::string & si_locations); - - void setSourceReportNumStatInsts( - const uint32_t num_stat_insts); - - void setStringMetadata( - const std::string & name, - const std::string & value); - - //! METADATA GETTERS --------------------------------- - //! - //! - Note that none of these getters return const &, - //! since this object is just a wrapper requesting - //! data from the database. It does not actually - //! store anything in memory. - std::string getReportName() const; - - uint64_t getReportStartTime() const; - - uint64_t getReportEndTime() const; - - std::string getSourceReportDescDestFile() const; - - std::string getCommaSeparatedSILocations() const; - - std::string getStringMetadata(const std::string & name) const; - - std::map getAllStringMetadata() const; - - std::map getAllHiddenStringMetadata() const; - -private: - std::unique_ptr obj_ref_; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/ReportNodeHierarchy.hpp b/sparta/sparta/report/db/ReportNodeHierarchy.hpp deleted file mode 100644 index 82a12bff7f..0000000000 --- a/sparta/sparta/report/db/ReportNodeHierarchy.hpp +++ /dev/null @@ -1,484 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "sparta/report/Report.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" - -namespace sparta { -namespace statistics { - -/*! - * \brief This class serializes a sparta::Report's entire - * report tree (report names, subreport names, etc.) into - * the ReportNodeHierarchy table in the database object - * you provide. - */ -class ReportNodeHierarchy -{ -public: - //! Construct with a report object you want to serialize. - explicit ReportNodeHierarchy(const Report * report) : - report_(report) - { - if (report_ == nullptr) { - throw SpartaException("Null report given to ReportNodeHierarchy"); - } - } - - //! Serialize this object's report to the provided SimDB. - //! Returns the database ID corresponding to the root-level - //! Report node in this hierarchy. - simdb::DatabaseID serializeHierarchy(const simdb::ObjectManager & obj_mgr) - { - obj_mgr.safeTransaction([&]() { - //This walks the SI/report tree in a depth-first - //fashion, creating report nodes in the database - //table(s) along the way as it serializes the - //hierarchy. - int leftmost_si_index = 0; - const bool is_leaf = false; - - std::unique_ptr root_report_node = - createReportNode_(report_->getName(), 0, - leftmost_si_index, - is_leaf, - obj_mgr); - - report_node_ids_.emplace_back(report_, root_report_node->getId()); - unordered_report_node_ids_[report_] = root_report_node->getId(); - - recursCreateSubreportNode_(*report_, - *root_report_node, - leftmost_si_index, - obj_mgr); - - root_report_node_db_id_ = root_report_node->getId(); - for (const auto & meta : root_report_metadata_) { - serializeReportGlobalMetadata_( - root_report_node_db_id_, meta.first, meta.second, obj_mgr); - } - root_report_metadata_.clear(); - - //Create any sub-statistics hierarchies that exist in this report. - //This will be used later to generate report files for reports - //that had ContextCounter's in them. - recursCreateSubStatisticsNodeHierarchy_(report_, obj_mgr); - - //Store some information in a separate table that will let - //the database report recreation code path know when to - //skip over certain sub-statistics. This lets database- - //regenerated reports exactly match simulation-produced - //reports when ContextCounters are found in the simulator. - markSubStatisticNodesAsUnprintable_(obj_mgr); - - //Create a 1-to-1 link between this root-level report - //node record and the ObjectManager it came from. This - //will be needed later on when we are asked to recreate - //a formatted report from a report database ID. - // - //For example, we may need to get the exact SimulationInfo - //record that was created by the original simulation. Those - //records live in a table with an ObjMgrID field, which is - //how we tie these two tables together with low overhead. - auto report_obj_mgr_linker = obj_mgr.getTable("RootReportObjMgrIDs"); - auto report_obj_mgr_link = report_obj_mgr_linker->createObjectWithArgs( - "RootReportNodeID", root_report_node_db_id_, - "ObjMgrID", obj_mgr.getId()); - }); - - return root_report_node_db_id_; - } - - //! Write report metadata to the provided SimDB. This includes - //! things like report start/stop times. - void serializeReportNodeMetadata(const simdb::ObjectManager & obj_mgr) const - { - obj_mgr.safeTransaction([&]() { - for (const auto & reports_by_id : report_node_ids_) { - const Report * report = reports_by_id.first; - const simdb::DatabaseID report_hier_node_id = reports_by_id.second; - serializeReportNodeMetadata_(*report, report_hier_node_id, obj_mgr); - } - }); - } - - //! Write any style information this object's report had at - //! the time of the original simulation. Some reports may have - //! no style metadata at all; it is not an error to call this - //! method anyway. In those cases, no ReportStyle record will - //! be created in the database for this report. - void serializeReportStyles(const simdb::ObjectManager & obj_mgr) const - { - obj_mgr.safeTransaction([&]() { - for (const auto & reports_by_id : report_node_ids_) { - const Report * report = reports_by_id.first; - const simdb::DatabaseID report_hier_node_id = reports_by_id.second; - serializeReportStyle_(*report, report_hier_node_id, obj_mgr); - } - }); - } - - //! Add generic name-value pairs of string metadata that applies - //! to every report/subreport/SI node we are serializing. - void setMetadataCommonToAllNodes( - const std::string & name, - const std::string & value, - const simdb::ObjectManager & obj_mgr) - { - if (root_report_node_db_id_ > 0) { - serializeReportGlobalMetadata_( - root_report_node_db_id_, name, value, obj_mgr); - } else { - root_report_metadata_[name] = value; - } - } - -private: - //! All report/SI nodes are written to the ReportNodeHierarchy - //! table through this method. - std::unique_ptr createReportNode_( - const std::string & name, - const simdb::DatabaseID node_id, - int & leftmost_si_index, - const bool is_leaf, - const simdb::ObjectManager & obj_mgr) const - { - std::unique_ptr hier_tbl = obj_mgr.getTable("ReportNodeHierarchy"); - std::unique_ptr node_ref = hier_tbl->createObjectWithArgs( - "Name", name, - "ParentNodeID", node_id, - "IsLeafSI", is_leaf ? 1 : 0, - "LeftmostSIIndex", leftmost_si_index); - - if (is_leaf) { - ++leftmost_si_index; - } - return node_ref; - } - - //! Some extra metadata only applies to leaf SI's, and not other - //! hierarchy nodes. Things like SI::getLocation(), SI::getDesc(), - //! etc. are all written to the database through this method. - void createLeafSIMetadata_( - const StatisticInstance * si, - const simdb::ObjectRef & report_hier_node_ref, - const simdb::ObjectManager & obj_mgr) const - { - std::unique_ptr si_metadata_tbl = obj_mgr.getTable("SIMetadata"); - if (si_metadata_tbl == nullptr) { - throw SpartaException("Unable to locate SIMetadata table in SimDB"); - } - - std::unique_ptr si_metadata_ref = si_metadata_tbl->createObjectWithArgs( - "ReportNodeID", report_hier_node_ref.getId(), - "Location", si->getLocation(), - "Desc", si->getDesc(false), - "ExprString", si->getExpressionString(), - "ValueSemantic", (int)si->getValueSemantic(), - "Visibility", (int)si->getVisibility(), - "Class", (int)si->getClass()); - - auto sdef = si->getStatisticDef(); - std::map written_metadata; - if (sdef) { - const auto & sdef_metadata = sdef->getMetadata(); - if (!sdef_metadata.empty()) { - si_metadata_tbl = obj_mgr.getTable("RootReportNodeMetadata"); - for (const auto & meta : sdef_metadata) { - si_metadata_tbl->createObjectWithArgs( - "ReportNodeID", report_hier_node_ref.getId(), - "Name", meta.first, - "Value", meta.second); - written_metadata[meta.first] = meta.second; - } - } - } - - const auto & si_metadata = si->getMetadata(); - if (!si_metadata.empty()) { - si_metadata_tbl = obj_mgr.getTable("RootReportNodeMetadata"); - for (const auto & meta : si_metadata) { - auto iter = written_metadata.find(meta.first); - sparta_assert(iter == written_metadata.end() || - iter->second == meta.second); - - si_metadata_tbl->createObjectWithArgs( - "ReportNodeID", report_hier_node_ref.getId(), - "Name", meta.first, - "Value", meta.second); - written_metadata[meta.first] = meta.second; - } - } - } - - //! Report metadata such as start/end tick, author, etc. are written - //! to the database through this method. - void serializeReportNodeMetadata_( - const Report & report_at_node, - const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr) const - { - std::unique_ptr metadata_tbl = - obj_mgr.getTable("ReportNodeMetadata"); - - std::unique_ptr metadata_ref = metadata_tbl->createObject(); - metadata_ref->setPropertyInt32("ReportNodeID", report_hier_node_id); - metadata_ref->setPropertyUInt64("StartTick", report_at_node.getStart()); - - if (report_at_node.getEnd() == Scheduler::INDEFINITE) { - auto sched = report_at_node.getScheduler(); - if (sched) { - metadata_ref->setPropertyUInt64("EndTick", sched->getCurrentTick()); - } else { - metadata_ref->setPropertyUInt64("EndTick", report_at_node.getEnd()); - } - } else { - metadata_ref->setPropertyUInt64("EndTick", report_at_node.getEnd()); - } - - const auto & author = report_at_node.getAuthor(); - if (!author.empty()) { - metadata_ref->setPropertyString("Author", author); - } - - const auto & info_str = report_at_node.getInfoString(); - if (!info_str.empty()) { - metadata_ref->setPropertyString("InfoString", info_str); - } - } - - //! Report style metadata is written to the database through this - //! method. Some reports have no style metadata at all, and for - //! those reports, no style records will be added to the database. - void serializeReportStyle_( - const Report & report_at_node, - const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr) const - { - const auto & styles = report_at_node.getAllStyles(); - if (styles.empty()) { - return; - } - - std::unique_ptr style_tbl = obj_mgr.getTable("ReportStyle"); - for (const auto & nv : styles) { - const std::string & style_name = nv.first; - const std::string & style_value = nv.second; - style_tbl->createObjectWithArgs("StyleName", style_name, - "StyleValue", style_value, - "ReportNodeID", report_hier_node_id); - } - } - - //! The SI/report tree is recursively serialized to the database - //! as we walk the tree in a depth-first fashion. This method is - //! called once for every top- and mid-level report node we encounter. - void recursCreateSubreportNode_( - const Report & subreport, - const simdb::ObjectRef & parent_node_ref, - int & leftmost_si_index, - const simdb::ObjectManager & obj_mgr) - { - std::set sub_stat_internal_pointers; - const Report::SubStaticticInstances & sub_stats = subreport.getSubStatistics(); - - for (const auto & stat : subreport.getStatistics()) { - const std::string name = !stat.first.empty() ? - stat.first : stat.second->getLocation(); - - const StatisticInstance * stat_inst = stat.second.get(); - const StatisticDef * def = stat_inst->getStatisticDef(); - const CounterBase * ctr = stat_inst->getCounter(); - const ParameterBase * prm = stat_inst->getParameter(); - - auto sub_stat_iter = sub_stats.find(def); - const bool valid_stat_def = (def != nullptr); - const bool has_valid_sub_stats = - (valid_stat_def && sub_stat_iter != sub_stats.end()); - - if (has_valid_sub_stats) { - //Gather up the 'this' pointers of the sub-statistics' counters - //or parameters. We'll use this information to mark certain sub- - //statistic nodes as "unprintable", which means that the database- - //driven report regeneration code will know when to skip over - //certain sub-statistics when it is making reports from a SimDB - //dataset. - for (const auto sub_stat : sub_stat_iter->second) { - if (sub_stat->getCounter() != nullptr) { - sub_stat_internal_pointers.insert(sub_stat->getCounter()); - } else if (sub_stat->getParameter() != nullptr) { - sub_stat_internal_pointers.insert(sub_stat->getParameter()); - } - } - } - - const bool is_leaf = true; - std::unique_ptr leaf_report_node = createReportNode_( - name, parent_node_ref.getId(), - leftmost_si_index, is_leaf, - obj_mgr); - - createLeafSIMetadata_(stat.second.get(), - *leaf_report_node, - obj_mgr); - - if (sub_stat_internal_pointers.count(ctr) > 0 || - sub_stat_internal_pointers.count(prm) > 0) - { - const void * sub_stat_this_ptr = - ctr ? ((const void*)ctr) : ((const void*)prm); - - sparta_assert(sub_stat_this_ptr != nullptr); - - sdef_sub_stat_ids_[sub_stat_this_ptr].emplace_back( - leaf_report_node->getId()); - } - - unordered_si_ids_[stat.second.get()] = leaf_report_node->getId(); - } - - for (const auto & sr : subreport.getSubreports()) { - const bool is_leaf = false; - std::unique_ptr subreport_node_ref = - createReportNode_(sr.getName(), parent_node_ref.getId(), - leftmost_si_index, is_leaf, - obj_mgr); - - report_node_ids_.emplace_back(&subreport, subreport_node_ref->getId()); - unordered_report_node_ids_[&sr] = subreport_node_ref->getId(); - - recursCreateSubreportNode_(sr, *subreport_node_ref, - leftmost_si_index, - obj_mgr); - } - } - - //! Recursively walk the report/SI hierarchy and serialize mappings - //! that describe sub-statistics hierarchies in this report. This - //! supports ContextCounter sub-hierarchies for SimDB generated - //! report files after simulation. - void recursCreateSubStatisticsNodeHierarchy_( - const Report * r, - const simdb::ObjectManager & obj_mgr) const - { - auto recurse = [&]() { - for (const auto & sr : r->getSubreports()) { - recursCreateSubStatisticsNodeHierarchy_(&sr, obj_mgr); - } - }; - - auto report_node_iter = unordered_report_node_ids_.find(r); - if (report_node_iter == unordered_report_node_ids_.end()) { - recurse(); - return; - } - - const simdb::DatabaseID report_node_id = report_node_iter->second; - for (const auto & stat : r->getStatistics()) { - const auto stat_def = stat.second->getStatisticDef(); - if (!stat_def) { - continue; - } - - const auto & sub_stats = r->getSubStatistics(); - auto sub_stat_iter = sub_stats.find(stat_def); - if (sub_stat_iter == sub_stats.end()) { - continue; - } - - auto sub_stats_hier_tbl = obj_mgr.getTable("SubStatisticsNodeHierarchy"); - for (const auto sub_stat : sub_stat_iter->second) { - auto si_node_iter = unordered_si_ids_.find(sub_stat); - if (si_node_iter == unordered_si_ids_.end()) { - continue; - } - - const simdb::DatabaseID si_node_id = si_node_iter->second; - auto parent_si_node_iter = unordered_si_ids_.find(stat.second.get()); - if (parent_si_node_iter == unordered_si_ids_.end()) { - continue; - } - - const simdb::DatabaseID parent_si_node_id = parent_si_node_iter->second; - - sub_stats_hier_tbl->createObjectWithArgs( - "ReportNodeID", report_node_id, - "SINodeID", si_node_id, - "ParentSINodeID", parent_si_node_id); - } - } - - recurse(); - } - - /*! - * \brief Database-regenerated reports need to exactly match - * simulation-generated reports. For ContextCounter's in JSON - * reports, there is a special code path that takes care of - * writing out "sub-statistics". The outside world - the json, - * json_reduced, and json_detail legacy formatters - are not - * supposed to write sub-statistics (ContextCounter internal - * counters). We have a table called "UnprintableSubStatistics" - * which lets us have careful control over what the legacy - * formatters print, and what they don't. This is put into - * a separate table so we don't impact database size by having - * a null / zeroed metadata column in the ReportNodeHierarchy - * table for *all* nodes, when only a relatively small number - * of nodes in the simulator (ContextCounter internal counters) - * needs this special treatment. - */ - void markSubStatisticNodesAsUnprintable_( - const simdb::ObjectManager & obj_mgr) - { - auto sub_stats_unprintable_tbl = obj_mgr.getTable("UnprintableSubStatistics"); - for (const auto & sub_stat_ids : sdef_sub_stat_ids_) { - for (size_t idx = 0; idx < sub_stat_ids.second.size(); ++idx) { - const simdb::DatabaseID unprintable_si_node_id = sub_stat_ids.second[idx]; - sub_stats_unprintable_tbl->createObjectWithArgs( - "ReportNodeID", unprintable_si_node_id); - } - } - } - - /*! - * \brief Serialize a metadata name-value pair. Unlike - * the serializeReportNodeMetadata() method, this method - * writes metadata that is common / shared with every node - * in this report hierarchy. - */ - void serializeReportGlobalMetadata_( - const simdb::DatabaseID report_id, - const std::string & name, - const std::string & value, - const simdb::ObjectManager & obj_mgr) const - { - auto meta_tbl = obj_mgr.getTable("RootReportNodeMetadata"); - if (auto num_rows_affected = meta_tbl-> - updateRowValues("Value", value). - forRecordsWhere("ReportNodeID", simdb::constraints::equal, report_id, - "Name", simdb::constraints::equal, name)) - { - sparta_assert(num_rows_affected == 1); - return; - } else { - meta_tbl->createObjectWithArgs( - "ReportNodeID", report_id, - "Name", name, - "Value", value); - } - } - - const Report * report_ = nullptr; - std::vector> report_node_ids_; - simdb::DatabaseID root_report_node_db_id_ = 0; - std::unordered_map unordered_report_node_ids_; - std::unordered_map unordered_si_ids_; - std::unordered_map> sdef_sub_stat_ids_; - std::map root_report_metadata_; -}; - -} // namespace statistics -} // namespace sparta diff --git a/sparta/sparta/report/db/ReportTimeseries.hpp b/sparta/sparta/report/db/ReportTimeseries.hpp deleted file mode 100644 index eba8c190ec..0000000000 --- a/sparta/sparta/report/db/ReportTimeseries.hpp +++ /dev/null @@ -1,167 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include - -#include "simdb/schema/Schema.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb_fwd.hpp" -#include "sparta/report/db/Schema.hpp" -#include "simdb/ObjectManager.hpp" -#include "sparta/report/db/ReportHeader.hpp" - -namespace sparta { -namespace db { - -class ReportHeader; - -/*! - * Wrapper around a database record (ObjectRef) which provides - * user-friendly API's to read and write timeseries values and - * report metadata in the database. - */ -class ReportTimeseries -{ -public: - //! Create a report timeseries wrapper around an - //! *existing* database record - explicit ReportTimeseries(std::unique_ptr obj_ref); - - //! Create a new report timeseries - ReportTimeseries(const simdb::ObjectManager & obj_mgr); - - //! Get the unique database ID for this timeseries. - //! This ID is unique in the main ReportTimeseries table, - //! not necessarily globally unique across the entire database. - uint64_t getId() const; - - //! Timeseries reports typically have some header - //! information. Access that object with this method - //! to read or write report metadata. - ReportHeader & getHeader(); - - //! Write SI values at a specific "time point": - //! - current simulated time (picoseconds) - //! - current root clock cycle - //! - //! The database will create an indexed chunk at this "time" value. - void writeStatisticInstValuesAtTimeT( - const uint64_t current_picoseconds, - const uint64_t current_cycle, - const std::vector & si_values, - const db::MajorOrdering major_ordering); - - //! Write *compressed* SI values at a specific "time point": - //! - current simulated time (picoseconds) - //! - current root clock cycle - //! - //! The database will create an indexed chunk at this "time" value. - void writeCompressedStatisticInstValuesAtTimeT( - const uint64_t current_picoseconds, - const uint64_t current_cycle, - const std::vector & compressed_si_values, - const db::MajorOrdering major_ordering, - const uint32_t original_num_si_values); - - //! Write SI values between two "time points": - //! - simulated time (picoseconds) - //! - root clock cycle - //! - //! The database will create an indexed chunk for this "time" range. - void writeStatisticInstValuesInTimeRange( - const uint64_t starting_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t starting_cycle, - const uint64_t ending_cycle, - const std::vector & si_values, - const db::MajorOrdering major_ordering); - - //! Write *compressed* SI values between two "time points": - //! - simulated time (picoseconds) - //! - root clock cycle - //! - //! It is the CALLER'S responsibility to compress this blob. - //! It will be written to the TimeseriesChunk table and will - //! be decompressed for you when you later access those SI values. - //! - //! The database will create an indexed chunk for this "time" range. - void writeCompressedStatisticInstValuesInTimeRange( - const uint64_t beginning_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t beginning_cycle, - const uint64_t ending_cycle, - const std::vector & compressed_si_values, - const db::MajorOrdering major_ordering, - const uint32_t original_num_si_values); - - //! Retrieve all SI data value chunks between "time points" A and B: - //! - between simulated time values A and B (picoseconds) - void getStatisticInstValuesBetweenSimulatedPicoseconds( - const uint64_t start_picoseconds, - const uint64_t end_picoseconds, - std::vector> & si_values); - - //! Retrieve all SI data value chunks between "time points" A and B: - //! - between root clock cycles A and B - void getStatisticInstValuesBetweenRootClockCycles( - const uint64_t start_cycle, - const uint64_t end_cycle, - std::vector> & si_values); - - //! Retrieve SI data values one "time slice" at a time. This can be - //! used to incrementally read SI values into memory for processing - //! in a more performant way than getting all of the values at once - //! with one of the above APIs. - class RangeIterator - { - public: - RangeIterator(ReportTimeseries & db_timeseries); - - //! Prepare to retrieve SI values between two simulated - //! picoseconds. This only sets up the query; call getNext() - //! to read in the first set of values. - void positionRangeAroundSimulatedPicoseconds( - const uint64_t start_picoseconds, - const uint64_t end_picoseconds); - - //! Prepare to retrieve SI values between two root - //! clock cycles. This only sets up the query; call getNext() - //! to read in the first set of values. - void positionRangeAroundRootClockCycles( - const uint64_t start_cycle, - const uint64_t end_cycle); - - //! Advance the iterator to the next set of values. Returns - //! true if successful, false otherwise (occurs when there - //! is no more data in this range). - bool getNext(); - - //! Get a pointer to the current SI range's data values. - const double * getCurrentSliceDataValuesPtr() const; - - //! Get the number of SI data points in the current slice. - //! This is how many points you can read off of the returned - //! pointer from getCurrentSliceDataValuesPtr() - size_t getCurrentSliceNumDataValues() const; - - private: - class Impl; - - std::shared_ptr impl_; - }; - -private: - //! Timeseries database object - std::unique_ptr obj_ref_; - - //! Report header database object - std::unique_ptr header_; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/ReportVerifier.hpp b/sparta/sparta/report/db/ReportVerifier.hpp deleted file mode 100644 index 37b4fc77a3..0000000000 --- a/sparta/sparta/report/db/ReportVerifier.hpp +++ /dev/null @@ -1,163 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "simdb_fwd.hpp" -#include "simdb/ObjectManager.hpp" - -namespace sparta { namespace report { namespace format { - class BaseFormatter; -}}} - -namespace sparta { namespace app { - class ReportDescriptor; -}} - -namespace sparta { - class Scheduler; - -namespace db { - -/*! - * \brief Verification utility which compares a formatted - * report file generated during a SPARTA simulation, against - * a SimDB-generated report file that was made after the - * simulation was already over. - */ -class ReportVerifier -{ -public: - //! Ask for the verification results directory. This is - //! a subdirectory relative to the current directory where - //! verification artifacts live. Failed report files may - //! be copied here for debugging purposes after simulation. - static const std::string & getVerifResultsDir(); - - //! Redirect the verification artifacts to be written - //! to a different directory. This is not required; - //! a default subdirectory will be used unless told - //! otherwise. Call getVerifResultsDir() to see the - //! current artifacts directory. - //! - //! This throws an exception if the post-simulation - //! verification process has already begun. - static void writeVerifResultsTo(const std::string & dir); - - //! SPARTA will add reports one by one for verification - //! at the end of simulation. - void addReportToVerify(const app::ReportDescriptor & rd); - - //! Optionally add one or more BaseFormatter's to this - //! verifier. The doPostProcessingBeforeReportValidation() - //! method will get called in between each report validation. - void addBaseFormatterForPreVerificationReset( - const std::string & filename, - report::format::BaseFormatter * formatter); - - //! Verification summary to be returned to the caller - //! after all SimDB-generated reports have been checked - //! for equivalence against the physical report file - //! left in the pwd during simulation. - class VerificationSummary - { - public: - ~VerificationSummary() = default; - - //! This method returns false only when no report files - //! were added via ReportVerifier::addReportToVerify() - bool hasSummary() const; - - //! Return a list of report files that *passed* verification. - std::set getPassingReportFilenames() const; - - //! Return a list of report files that *failed* verification. - std::set getFailingReportFilenames() const; - - //! For the given report file, was it found in the database - //! as a timeseries or not? - bool reportIsTimeseries(const std::string & filename) const; - - //! For the given report file, return a failure summary - //! that highlights differences between the physical - //! report produced by the simulation, and the report - //! produced from the database after simulation. - //! - //! Returns an empty string if this report passed the - //! verification, or if there was no report by this - //! name found in the VerificationSummary. - std::string getFailureDifferences(const std::string & filename) const; - - //! Write all contents of this report verification summary - //! to the provided database. - void serializeSummary( - const simdb::ObjectManager & sim_db) const; - - //! Get a 1-to-1 mapping of the filenames that were passed - //! into the verifier, and the names of the files that the - //! verifier used in order to run the verification. - //! - //! For instance, say your original yaml file had dest_file's - //! 'foo.csv' and 'bar.json', and the database had file location - //! '/tmp/abcd-1234.db' - //! - //! The SimDB-generated files may end up in a subfolder like this: - //! - //! /abcd-1234 - //! /foo.csv_i34l2kj - //! /bar.json_wqr90w4 - //! - //! The file suffixes may have been added to help ensure - //! that file-to-file comparisons do not fail due to many - //! simulations running in parallel (make regress) and - //! producing files with potentially the same dest_file - //! name. - //! - //! This mapping would then be: - //! - //! { { "bar.json", "abcd-1234/bar.json_wqr90w4" }, - //! { "foo.csv", "abcd-1234/foo.csv_i34l2kj" } } - std::map getFinalDestFiles() const; - - private: - //! Let the ReportVerifier give us report files one - //! by one to verify. Returns true if the verification - //! succeeded, false otherwise. More details can be - //! obtained via the public APIs. - bool verifyReport_(const std::string & filename, - const Scheduler * scheduler); - - //! ReportVerifier is the only one who can create one - //! of these objects. - VerificationSummary(const simdb::ObjectManager & sim_db); - friend class ReportVerifier; - - //! Hide implementation details from sight. - class Impl; - - std::shared_ptr impl_; - }; - - //! Verify each report report file that was added via - //! addReportToVerify(), and return a summary object - //! for inspection and error reporting. - std::unique_ptr verifyAll( - const simdb::ObjectManager & sim_db, - const Scheduler * scheduler); - -private: - std::map to_verify_; - std::map formatters_; - static std::string verif_results_dir_; - static bool verif_results_dir_is_changeable_; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/Schema.hpp b/sparta/sparta/report/db/Schema.hpp deleted file mode 100644 index 94a71f45a2..0000000000 --- a/sparta/sparta/report/db/Schema.hpp +++ /dev/null @@ -1,33 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/schema/Schema.hpp" - -#include - -namespace sparta { -namespace db { - -//! Build a SimDB schema object that can hold all report artifacts -//! and StatisticInstance values for SPARTA simulators. This schema -//! can be given to a simdb::ObjectManager to instantiate a physical -//! database connection. -void buildSimulationDatabaseSchema(simdb::Schema & schema); - -//! Configure the default TableSummaries object for SPARTA simulation -//! databases. This will provide default implementations for common -//! summary calculations like min/max/average, and possibly others. -void configureDatabaseTableSummaries(simdb::TableSummaries & summaries); - -//! \brief Enum specifying whether blobs' data values -//! are stored in row-major or column-major format -enum class MajorOrdering : int32_t -{ - ROW_MAJOR, - COLUMN_MAJOR -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/SimInfoSerializer.hpp b/sparta/sparta/report/db/SimInfoSerializer.hpp deleted file mode 100644 index 0acae8c039..0000000000 --- a/sparta/sparta/report/db/SimInfoSerializer.hpp +++ /dev/null @@ -1,182 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/ObjectManager.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "sparta/app/SimulationInfo.hpp" -#include "sparta/utils/StringUtils.hpp" - -namespace sparta { -namespace db { - -/*! - * \brief This class handles SimDB serialization of all - * SimulationInfo metadata. - */ -class SimInfoSerializer -{ -public: - //! Construct with a reference to a SimulationInfo object - //! you wish to serialize. - explicit SimInfoSerializer(const SimulationInfo & sim_info) : - sim_info_(sim_info) - {} - - //! Noncopyable, immovable - SimInfoSerializer(const SimInfoSerializer &) = delete; - SimInfoSerializer & operator=(const SimInfoSerializer &) = delete; - SimInfoSerializer(SimInfoSerializer &&) = delete; - SimInfoSerializer & operator=(SimInfoSerializer &&) = delete; - - //! Manually set/override SimulationInfo properties. - //! These will be used in place of the equivalent - //! values found in the SimulationInfo object that - //! was given to our constructor. - //! - //! \note The prop_name argument is not case sensitive. - //! - //! \note Properties set more than once will not warn - //! or throw an exception. The most recent prop_value - //! given for the prop_name set more than once will be - //! used to write the SimulationInfo record. - //! - //! \warning The properties "Other" and "ObjMgrID" are - //! not allowed to be set manually. Attempts to do so - //! will result in an exception. - void setPropertyString(const std::string & prop_name, - const std::string & prop_value) - { - const utils::lowercase_string lower_prop_name = prop_name; - if (lower_prop_name == "other" || lower_prop_name == "objmgrid") { - throw SpartaException("SimInfoSerializer::setPropertyString() called ") - << "with prop_name '" << prop_name << "'. This is not allowed."; - } - prop_kvpairs_[prop_name] = prop_value; - } - - //! Write the contents of this SimulationInfo object to - //! the given database. - void serialize(const simdb::ObjectManager & sim_db) - { - sim_db.safeTransaction([&]() { - std::unique_ptr sim_info_tbl = sim_db.getTable("SimInfo"); - if (sim_info_tbl == nullptr) { - //We could hit this code if the ObjectManager was - //connected to a database with a custom schema. The - //SI schema has a SimInfo table, but other user-defined - //schemas propably don't. - std::cout << "SimInfoSerializer could not find the " - << "SimInfo table. If this database is from " - << "a user-defined schema, you can ignore this " - << "warning. Database file is '" - << sim_db.getDatabaseFile() << "'" - << std::endl; - return; - } - - //Helper to extract any property values that were - //manually set via our API, as opposed to being - //taken from the SimulationInfo object itself. - auto get_prop_val = [this](const std::string & metadata_name, - const std::string & metadata_value) - -> std::string - { - const utils::lowercase_string lower_metadata_name = metadata_name; - auto manual_prop_iter = prop_kvpairs_.find(lower_metadata_name); - if (manual_prop_iter != prop_kvpairs_.end()) { - return manual_prop_iter->second; - } - return metadata_value; - }; - - //Loop over the name-value pairs returned by the - //SimulationInfo object, and write them into the - //SimInfo table. - std::unique_ptr sim_info_record; - const auto sim_info_header = sim_info_.getHeaderPairs(); - for (const auto & nv : sim_info_header) { - const std::string & metadata_name = nv.first; - const std::string & metadata_value = nv.second; - - //Some name-value pairs are returned empty from - //the SimulationInfo. We can't serialize those. - if (metadata_name.empty() || metadata_value.empty()) { - continue; - } - - //The "Elapsed" piece of metadata is not actually - //a member variable of the SimulationInfo class. - //It is retrieved on demand from the TimeManager - //for each report as they are written to disk. We - //therefore do not serialize it as part of the - //global / simulator-wide metadata "SimInfo" - //table. - if (metadata_name == "Elapsed") { - continue; - } - - //Since we have an early continue in this loop, - //make sure we only put a record in the SimInfo - //table if we have any non-empty metadata info - //to write. - if (sim_info_record == nullptr) { - sim_info_record = sim_info_tbl->createObject(); - } - - const std::string header_val = get_prop_val(metadata_name, metadata_value); - sim_info_record->setPropertyString(metadata_name, header_val); - } - - if (sim_info_record == nullptr) { - return; - } - - const std::string & working_dir = sim_info_.working_dir; - const std::string working_dir_val = get_prop_val("WorkingDir", working_dir); - if (!working_dir_val.empty()) { - sim_info_record->setPropertyString("WorkingDir", working_dir_val); - } - - const std::string & sparta_version = sim_info_.sparta_version; - const std::string sparta_version_val = get_prop_val("SpartaVersion", sparta_version); - if (!sparta_version_val.empty()) { - sim_info_record->setPropertyString("SpartaVersion", sparta_version_val); - } - - const std::string & repro_info = sim_info_.reproduction_info; - const std::string repro_info_val = get_prop_val("Repro", repro_info); - if (!repro_info_val.empty()) { - sim_info_record->setPropertyString("Repro", repro_info_val); - } - - if (!sim_info_.other.empty()) { - std::ostringstream oss; - if (sim_info_.other.size() == 1) { - oss << sim_info_.other[0]; - } else { - for (size_t idx = 0; idx < sim_info_.other.size() - 1; ++idx) { - oss << sim_info_.other[idx] << ","; - } - oss << sim_info_.other.back(); - } - sim_info_record->setPropertyString("Other", oss.str()); - } - - //We use the ObjectManager's unique ID to link these - //SimInfo records back to other database entities, such - //as report records. Those reports would be serialized - //to other database tables, and have this same ObjMgrID - //as a column in those tables to make the connection. - sim_info_record->setPropertyInt32("ObjMgrID", sim_db.getId()); - }); - } - -private: - const SimulationInfo & sim_info_; - std::map prop_kvpairs_; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/SingleUpdateReport.hpp b/sparta/sparta/report/db/SingleUpdateReport.hpp deleted file mode 100644 index d32d610522..0000000000 --- a/sparta/sparta/report/db/SingleUpdateReport.hpp +++ /dev/null @@ -1,79 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include - -#include "simdb_fwd.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" - -namespace simdb { -class ObjectManager; -} // namespace simdb - -namespace sparta { -namespace db { - -/*! - * \brief Wrapper around SimDB tables that are dedicated - * to report/SI persistence of single-update, non-timeseries - * report formats. - * - * \note All methods in this class are performed synchronously. - * Typically, you will only call the "write*()" method once - * for this object, and synchronous operation yields basically - * the same performance as asynchronous. However, if you have - * many of these reports to write in a SimDB, consider using - * this class together with AsyncNonTimeseriesReport for better - * overall performanace across all of the single-update report - * records. - */ -class SingleUpdateReport -{ -public: - //! Construct with an ObjectRef to an existing record - //! in the SingleUpdateStatInstValues table. - explicit SingleUpdateReport( - std::unique_ptr obj_ref); - - //! Add a new entry in the SingleUpdateStatInstValues - //! table. This record will be added to the given SimDB - //! you pass in, and the database ID corresponding to - //! the root-level Report node (typically retrieved - //! from ReportNodeHierarchy). - SingleUpdateReport( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID root_report_node_id); - - //! Get this report record's database ID. - int getId() const; - - //! Write the given SI values to this database record. - //! No compression will be performed. - void writeStatisticInstValues( - const std::vector & si_values); - - //! Write the *already compressed* SI values into - //! the database record. - void writeCompressedStatisticInstValues( - const std::vector & compressed_si_values, - const uint32_t original_num_si_values); - - //! Retrieve the SI values for this report. As this - //! is for single-update reports only, there are no - //! start/end time points like you see with the read - //! API's for the ReportTimeseries class. - void getStatisticInstValues( - std::vector & si_values); - -private: - std::unique_ptr obj_ref_; - simdb::DatabaseID root_report_node_id_ = 0; -}; - -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/db/StatInstRowIterator.hpp b/sparta/sparta/report/db/StatInstRowIterator.hpp deleted file mode 100644 index a30ea8de8a..0000000000 --- a/sparta/sparta/report/db/StatInstRowIterator.hpp +++ /dev/null @@ -1,202 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/ObjectManager.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/schema/Schema.hpp" -#include "sparta/utils/ValidValue.hpp" - -namespace sparta { - -/*! - * \brief This class wraps an ObjectQuery which is - * positioned to loop over one or more rows of SI - * blob data. - */ -class StatInstRowIterator -{ -public: - //! This class can be used standalone as an ObjectQuery - //! wrapper, using the getNext() method to iterate over - //! an SI dataset. - //! - //! However, this class is also used together with - //! StatInstValueLookup objects, which are given to - //! all SI's in a SimDB report. We do not want to - //! expose public API's like getNext() to every SI - //! in a report, since a call to that method has an - //! irreversible effect on all StatInstValueLookup's - //! that are pointing to this row iterator. - //! - //! This nested class overcomes this, and only provides - //! SI's a getter method returning a const reference to - //! the underlying SI double vector. - class RowAccessor - { - public: - //! Construct with an SI values vector. This accessor - //! holds a reference to that vector. - explicit RowAccessor(const std::vector & row_ref) : - row_(row_ref) - {} - - //! Return the row of SI values this accessor is tied to. - const std::vector & getCurrentRow() const { - return row_; - } - - private: - const std::vector & row_; - }; - - using RowAccessorPtr = std::shared_ptr; - - //! Construct a row iterator for a root-level report node - //! in the given database. You can find available root-level - //! nodes in the ReportNodeHierarchy table where the column - //! ParentNodeID equals 0. - StatInstRowIterator(const simdb::DatabaseID report_root_node_id, - const simdb::ObjectManager & obj_mgr) : - row_accessor_(new RowAccessor(raw_si_values_)) - { - simdb::ObjectQuery query(obj_mgr, "SingleUpdateStatInstValues"); - - query.addConstraints( - "RootReportNodeID", - simdb::constraints::equal, - report_root_node_id); - - query.writeResultIterationsTo( - "RawBytes", &raw_si_bytes_, - "NumPts", &raw_si_num_pts_, - "WasCompressed", &raw_si_was_compressed_); - - result_iter_ = query.executeQuery(); - if (result_iter_ == nullptr) { - throw SpartaException("Unable to use StatInstRowIterator. The ") - << "database query failed."; - } - } - - //! Get a "row accessor" object which can give you a const - //! reference to this row iterator's current SI row values. - virtual const RowAccessorPtr & getRowAccessor() const { - return row_accessor_; - } - - //! Advance this iterator to the next row of SI values. - //! Returns true on success, false otherwise. This will - //! typically return false only when there are no more - //! SI values in this data set. - //! - //! If this method returns false for any reason, consider - //! the accompanying RowAccessor to be invalidated. Once - //! this iterator is out of data to loop over, calls to - //! RowAccessor::getCurrentRow() are undefined. - virtual bool getNext() { - //Advance the ObjectQuery - if (!result_iter_->getNext()) { - return false; - } - - return getCurrentRowDoubles_(); - } - - virtual ~StatInstRowIterator() = default; - - //! This class can be "partially constructed" if you - //! only have a *non* root-level report node ID on - //! hand, and cannot retrieve its root-level node ID - //! right away for some reason. In that case, create a - //! placeholders::StatInstRowIterator object with the - //! non-root-level report node ID, and later call its - //! realizePlaceholder() method when you are ready. - //! The placeholder subclass will find the root-level - //! report node ID for you at that time. - virtual sparta::StatInstRowIterator * realizePlaceholder() { - return this; - } - -protected: - StatInstRowIterator() = default; - -private: - //! Turn the vector of char's we are holding onto - //! into a vector double's. This takes into account - //! compression if it was performed on the current - //! SI row. Returns true on success. - bool getCurrentRowDoubles_(); - - int raw_si_num_pts_; - int raw_si_was_compressed_; - RowAccessorPtr row_accessor_; - std::vector raw_si_values_; - std::vector raw_si_bytes_; - std::unique_ptr result_iter_; -}; - -} // namespace sparta - -namespace placeholders { - -//! This placeholder is used when you want to ultimately -//! create a sparta::StatInstRowIterator object, but you -//! only have a non-root-level report node ID. It can -//! be somewhat expensive to get the root-level node ID -//! for the report node that you have on hand, and -//! you may wish to delay that expensive database query -//! until later for some reason. -//! -//! These "placeholder" objects are later turned into -//! "realized" objects when you call the realizePlaceholder() -//! method. -//! -//! IMPORTANT: All public API's in the base class are -//! off limits until you first call realizePlaceholder(). -//! The *returned* finalized, realized row iterator object -//! is the only one who can touch those API's. Attempts to -//! call base class API's on an unrealized placeholder -//! will throw an exception. -class StatInstRowIterator : public sparta::StatInstRowIterator -{ -public: - //! Construct one of these placeholders with a non-root-level - //! report node ID. These would be any Id's for records in the - //! ReportNodeHierarchy table whose ParentNodeID field is not - //! zero. - //! - //! These placeholders can also be given a root-level report - //! node ID, though you may incur unnecessary overhead if you - //! use a placeholder when you don't need to. You'll typically - //! give the base class your root-level ID directly, and skip - //! the step of having a placeholder at all. - StatInstRowIterator(const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager * obj_mgr) : - sparta::StatInstRowIterator(), - report_hier_node_id_(report_hier_node_id), - obj_mgr_(obj_mgr) - {} - - virtual sparta::StatInstRowIterator * realizePlaceholder() override; - -private: - //! Override base class public API's so we can throw if - //! anyone tries to use a placeholder like it was a finalized - //! sparta::StatInstRowIterator - using RowAccessorPtr = sparta::StatInstRowIterator::RowAccessorPtr; - virtual const RowAccessorPtr & getRowAccessor() const override final { - throw sparta::SpartaException("StatInstRowIterator::getRowAccessor() called ") - << "on a placeholder object that has not yet been realized!"; - } - virtual bool getNext() override final { - throw sparta::SpartaException("StatInstRowIterator::getNext() called ") - << "on a placeholder object that has not yet been realized!"; - } - - const simdb::DatabaseID report_hier_node_id_; - const simdb::ObjectManager *const obj_mgr_; -}; - -} // namespace placeholders - diff --git a/sparta/sparta/report/db/StatInstValueLookup.hpp b/sparta/sparta/report/db/StatInstValueLookup.hpp deleted file mode 100644 index 3f8d20dcf8..0000000000 --- a/sparta/sparta/report/db/StatInstValueLookup.hpp +++ /dev/null @@ -1,160 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "sparta/report/db/StatInstRowIterator.hpp" - -namespace sparta { - -/* - * \brief This class holds a shared StatInstRowIterator - * object which owns a vector of SI values. This class - * also has an index value which tells it which element - * in the row iterator's vector belongs to this direct- - * lookup object. - * - * More specifically, it owns a StatInstRowIterator's - * "RowAccessor" object, which has just one read-only - * API to ask for the current SI row's double vector - * by const reference. - */ -class StatInstValueLookup { -public: - using RowAccessorPtr = - std::shared_ptr; - - //! Construct with a shared RowAccessor object (obtained - //! from a StatInstRowIterator), and an SI index which - //! tells us where our SI value can be found in the - //! row accessor's underlying double vector. - StatInstValueLookup(const RowAccessorPtr & row_accessor, - const size_t si_index) : - StatInstValueLookup(row_accessor, si_index, true) - { - //Constructor delegation. Pass in true for third argument - //so we know to perform argument validation. - } - - //! Verify that this lookup object's SI index is within - //! the range of the underlying RowAccessor's SI values - //! vector. You may want to call this just once after - //! calling StatInstRowIterator::getNext() for the first - //! time, or call it once after every call to getNext(). - //! Or never call it at all for best performance, if you - //! are willing to give up safety checks. This would be - //! the same decision as calling std::vector::operator[] - //! versus std::vector::at() - virtual bool isIndexValidForCurrentRow() const { - return si_index_ < row_accessor_->getCurrentRow().size(); - } - - //! Get this lookup object's SI value for the SI row - //! its RowAccessor is currently pointing to. - virtual double getCurrentValue() const { - return row_accessor_->getCurrentRow()[si_index_]; - } - - //! This class can be "partially constructed" if you - //! only have the SI index value on hand, but not the - //! RowAccessor object that goes with it. Create a - //! placeholders::StatInstValueLookup object with the - //! SI index value, and call its realizePlaceholder() - //! method when you later get the RowAccessor. - virtual sparta::StatInstValueLookup * realizePlaceholder( - const RowAccessorPtr & row_accessor) - { - (void) row_accessor; - return this; - } - - virtual ~StatInstValueLookup() = default; - -protected: - StatInstValueLookup() : - StatInstValueLookup(nullptr, 0, false) - { - //Constructor delegation. Pass in false for third argument - //so we do not perform argument validation. - } - -private: - //! The public constructor meant for non-placeholder or realized - //! placeholder objects calls this private constructor. And the - //! protected constructor meant for for non-realized placeholders - //! calls here too. The only difference is if we validate the - //! constructor arguments. - StatInstValueLookup(const RowAccessorPtr & row_accessor, - const size_t si_index, - const bool validate) : - row_accessor_(row_accessor), - si_index_(si_index) - { - if (validate and !row_accessor_) { - throw SpartaException("Null StatInstRowIterator::RowAccessor ") - << "given to a StatInstValueLookup constructor"; - } - //We can't validate the SI index right now. The row - //accessor could still be holding a reference to an - //empty vector. This would be the case if the owning - //StatInstRowIterator::getNext() method hasn't been - //called yet. - } - - RowAccessorPtr row_accessor_; - const size_t si_index_; -}; - -} // namespace sparta - -namespace placeholders { - -//! This placeholder is used when you want to ultimately -//! create a sparta::StatInstValueLookup object, but you -//! only have the leaf SI index value on hand. -//! -//! These "placeholder" objects are later turned into -//! "realized" objects when you call the realizePlaceholder() -//! method. -//! -//! IMPORTANT: All public API's in the base class are -//! off limits until you first call realizePlaceholder(). -//! The *returned* finalized, realized value lookup object -//! is the only one who can touch those API's. Attemps to -//! call base class API's on an unrealized placeholder -//! will throw an exception. -class StatInstValueLookup : public sparta::StatInstValueLookup -{ -public: - //! Construct with the leaf SI index that you want to - //! later give to a sparta::StatInstValueLookup. - explicit StatInstValueLookup(const size_t si_index) : - sparta::StatInstValueLookup(), - si_index_(si_index) - {} - - //! Combine the leaf SI index you gave our constructor - //! with a RowAccessor object to "realize" this placeholder - //! into a finalized, usable, StatInstValueLookup object. - virtual sparta::StatInstValueLookup * realizePlaceholder( - const sparta::StatInstValueLookup::RowAccessorPtr & row_accessor) override; - -private: - //! Override base class public API's so we can throw if - //! anyone tries to use a placeholder like it was a finalized - //! sparta::StatInstValueLookup - virtual bool isIndexValidForCurrentRow() const override final { - throw sparta::SpartaException("StatInstValueLookup::isIndexValidForCurrentRow() ") - << "called on a placeholder object that has not yet " - << "been realized!"; - } - virtual double getCurrentValue() const override final { - throw sparta::SpartaException("StatInstValueLookup::getCurrentValue() ") - << "called on a placeholder object that has not yet " - << "been realized!"; - } - - const size_t si_index_; -}; - -} // namespace placeholders - diff --git a/sparta/sparta/report/db/format/toCSV.hpp b/sparta/sparta/report/db/format/toCSV.hpp deleted file mode 100644 index fd0f317d0f..0000000000 --- a/sparta/sparta/report/db/format/toCSV.hpp +++ /dev/null @@ -1,144 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/report/Report.hpp" -#include "sparta/utils/SpartaException.hpp" - -namespace sparta { -namespace db { -namespace format { - -/*! - * \brief Utility which writes out an entire timeseries - * object found in a database file to the provided filename. - */ -inline void toCSV(ReportTimeseries * ts, const std::string & filename) -{ - std::ostream * out_ptr = nullptr; - std::ofstream fout; - - if (filename == "1") { - //SPARTA uses the "filename" of "1" to indicate - //writing to stdout. - out_ptr = &std::cout; - } else { - fout.open(filename); - if (!fout) { - throw SpartaException("Unable to open file for write: '") - << filename << "'"; - } - out_ptr = &fout; - } - std::ostream & out = *out_ptr; - - const auto & header = ts->getHeader(); - - // Write all header comments. These are written in this order (as - // opposed to alphabetical, or any other order) so that - // database-regenerated CSV report files match exactly what you - // would get using legacy CSV formatters during simulation - // (BaseFormatter). - // - // We're on the first line of metadata: - // - // # report="stats.yaml on top.core0",start=4436,end=SIMULATION_END,report_format=csv - // # enabled=none,period=5,type=nanoseconds,counter=NS,terminate=none,warmup=1202 - // - const std::string raw_header = header.getStringMetadata("RawHeader"); - if (!raw_header.empty() && raw_header != "unset") { - out << raw_header; - } else { - out << "# report=\"" << header.getReportName() << "\","; - const auto sim_start = header.getReportStartTime(); - out << "start=" << sim_start << ","; - out << "end="; - const auto sim_end = header.getReportEndTime(); - if (sim_end == std::numeric_limits::max()) { - out << "SIMULATION_END"; - } else { - out << sim_end; - } - - //Add any additional metadata to the CSV header - std::map string_metadata = - header.getAllStringMetadata(); - - // There is one piece of metadata that we *always* know is - // going to be in the database, because SimDB is writing it - // regardless. Is is the "report_format" string. We write this - // at the end of the first line of metadata because that's - // where the BaseFormatter always puts it. - auto format_iter = string_metadata.find("report_format"); - sparta_assert(format_iter != string_metadata.end()); - out << ",report_format=" << format_iter->second; - string_metadata.erase(format_iter); - out << "\n"; - - //The "Elapsed" metadata is only used in certain other - //non-timeseries report formats. But legacy CSV does not - //use it, so we'll discard it here. - format_iter = string_metadata.find("Elapsed"); - if (format_iter != string_metadata.end()) { - string_metadata.erase(format_iter); - } - - //All other metadata name-value pairs get written on - //their own line near the top of the CSV file. They - //all go in alphabetical order. We're on the second - //line of metadata: - // - // # report="stats.yaml on top.core0",start=4436,end=SIMULATION_END,report_format=csv - // # enabled=none,period=5,type=nanoseconds,counter=NS,terminate=none,warmup=1202 - // - const size_t num_string_metadata = string_metadata.size(); - if (num_string_metadata > 0) { - out << "# "; - if (num_string_metadata == 1) { - out << string_metadata.begin()->first << "=" - << string_metadata.begin()->second << "\n"; - } else { - auto md_iter = string_metadata.begin(); - for (size_t md_idx = 0; md_idx < num_string_metadata-1; ++md_idx) { - out << md_iter->first << "=" << md_iter->second << ","; - ++md_iter; - } - out << md_iter->first << "=" << md_iter->second << "\n"; - } - } - } - - //Write the SI locations ("scheduler.ticks,scheduler.seconds,...") - out << header.getCommaSeparatedSILocations() << "\n"; - - //Read the SI data from the database blob by blob. The RangeIterator - //class will handle the internals for maximum performance, whether - //the SI values were compressed or not, saved in row-major or column- - //major format, etc. - ReportTimeseries::RangeIterator iterator(*ts); - static const auto start = std::numeric_limits::min(); - static const auto end = std::numeric_limits::max(); - iterator.positionRangeAroundSimulatedPicoseconds(start, end); - - while (iterator.getNext()) { - const double * si_values_ptr = iterator.getCurrentSliceDataValuesPtr(); - const size_t num_si_values = iterator.getCurrentSliceNumDataValues(); - - sparta_assert(num_si_values > 0 && si_values_ptr != nullptr); - if (num_si_values == 1) { - out << Report::formatNumber(*si_values_ptr) << "\n"; - } else { - for (size_t si_idx = 0; si_idx < num_si_values-1; ++si_idx) { - out << Report::formatNumber(*(si_values_ptr+si_idx)) << ","; - } - out << Report::formatNumber(*(si_values_ptr+num_si_values-1)) << "\n"; - } - } -} - -} // namespace format -} // namespace db -} // namespace sparta - diff --git a/sparta/sparta/report/format/JSON_detail.hpp b/sparta/sparta/report/format/JSON_detail.hpp index ecb7e49d89..cd33642a6a 100644 --- a/sparta/sparta/report/format/JSON_detail.hpp +++ b/sparta/sparta/report/format/JSON_detail.hpp @@ -15,7 +15,6 @@ #include #include "sparta/report/format/BaseOstreamFormatter.hpp" -#include "sparta/report/db/DatabaseContextCounter.hpp" #include "sparta/utils/SpartaException.hpp" #include "sparta/utils/SpartaAssert.hpp" @@ -212,82 +211,7 @@ class JSON_detail : public BaseOstreamFormatter local_name = p_name + "." + flattenReportName(r->getName()); } - auto extract_stat = [&local_name](const statistics::stat_pair_t & si) { - std::string full_name = local_name + "." + si.first; - std::string desc = si.second->getDesc(false); - boost::replace_all(desc, "\"", "\\\""); - struct info_data tmp; - tmp.name = full_name; - tmp.desc = desc; - tmp.vis = si.second->getVisibility(); - tmp.n_class = si.second->getClass(); - const StatisticDef * stat_defn = si.second->getStatisticDef(); - if (stat_defn != nullptr) { - tmp.metadata = stat_defn->getMetadata(); - } else { - tmp.metadata = si.second->getMetadata(); - } - return tmp; - }; - - const Report::SubStaticticInstances & sub_stats = r->getSubStatistics(); - const Report::DBSubStatisticInstances & db_sub_stats = r->getDBSubStatistics(); - - std::set dont_print_these; - std::set db_dont_print_these; - for (const statistics::stat_pair_t& si : r->getStatistics()) { - if(si.first != ""){ - const StatisticInstance * stat_inst = si.second.get(); - const StatisticDef * stat_defn = si.second->getStatisticDef(); - const CounterBase * ctr = si.second->getCounter(); - const ParameterBase * prm = si.second->getParameter(); - sparta_assert(static_cast(this) != static_cast(ctr)); - sparta_assert(static_cast(this) != static_cast(prm)); - - auto sub_stat_iter = sub_stats.find(stat_defn); - const bool valid_stat_def = (stat_defn != nullptr); - const bool has_valid_sub_stats = - (valid_stat_def && sub_stat_iter != sub_stats.end()); - - auto db_sub_stat_iter = db_sub_stats.find(stat_inst); - const bool has_valid_db_sub_stats = (db_sub_stat_iter != db_sub_stats.end()); - - if (has_valid_sub_stats && stat_defn->groupedPrintingDetail(sub_stat_iter->second, - dont_print_these, - nullptr, - nullptr)) { - detail_json_map[si.first].push_back(extract_stat(si)); - continue; - } - if (dont_print_these.count(ctr) > 0 || dont_print_these.count(prm) > 0) { - continue; - } - dont_print_these.clear(); - - if (has_valid_db_sub_stats) { - const std::shared_ptr & db_ctx_ctr = - db_sub_stat_iter->second.first; - - const std::vector & db_sub_sis = - db_sub_stat_iter->second.second; - - if (db_ctx_ctr->groupedPrintingDetail(db_sub_sis, - db_dont_print_these, - nullptr, - nullptr)) - { - detail_json_map[si.first].push_back(extract_stat(si)); - continue; - } - } - if (db_dont_print_these.count(stat_inst) > 0) { - continue; - } - db_dont_print_these.clear(); - - detail_json_map[si.first].push_back(extract_stat(si)); - } - } + // TODO cnyce // Go through all subreports for (const Report& sr : r->getSubreports()) { diff --git a/sparta/sparta/resources/AgedArrayCollector.hpp b/sparta/sparta/resources/AgedArrayCollector.hpp deleted file mode 100644 index f0c76e2b84..0000000000 --- a/sparta/sparta/resources/AgedArrayCollector.hpp +++ /dev/null @@ -1,71 +0,0 @@ -// -*- C++ -*- - -/** - * \file ArrayCollector.hpp - * \brief Class used by the sparta::Array class - */ -#pragma once - -#include - -#include "sparta/resources/VectorResourceCollectable.hpp" - -namespace sparta -{ -namespace collection -{ - - /** - * \class AgedArrayCollector - * \brief An AgedArrayCollector is created by the Array class in - * sparta when Pipeline collection is required. - * - * This collector will always present the data of an Aged array - * starting at index 0. - */ - template - class AgedArrayCollector : public VectorResourceCollectable - { - using VectorResourceCollectable::collected_resource_; - using VectorResourceCollectable::collectors_; - using VectorResourceCollectable::closeRecord; - - public: - /** - * \brief Construct an AgedArrayCollector - * \param parent A pointer to the parent treenode for wich - * collectable objects will be created under. - * \param array The array to be collected - */ - AgedArrayCollector(sparta::TreeNode* parent, const ArrayType* array) : - VectorResourceCollectable(parent, array, - array->getName() + "_age_ordered", - array->getName() + " Age-Ordered") - {} - - /** - * \brief set up the Collector with the current state of the Array, - * and begin collection going forward. - */ - void collect() override final - { - // We need to step through the array and validate all valid positions. - const typename ArrayType::AgedList & aged_list = - collected_resource_->getInternalAgedList_(); - uint32_t collector_idx = aged_list.size() - 1; - for(const auto & obj : aged_list) - { - collectors_[collector_idx]->collect(collected_resource_->read(obj)); - --collector_idx; - } - - for(uint32_t i = aged_list.size(); i < collected_resource_->capacity(); ++i) { - collectors_[i]->closeRecord(); - } - - } - }; -}//namespace collection -}//namespace sparta - - diff --git a/sparta/sparta/resources/Array.hpp b/sparta/sparta/resources/Array.hpp index 15e0432351..6d1219edce 100644 --- a/sparta/sparta/resources/Array.hpp +++ b/sparta/sparta/resources/Array.hpp @@ -14,8 +14,8 @@ #include "sparta/utils/SpartaAssert.hpp" #include "sparta/statistics/CycleHistogram.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/utils/IteratorTraits.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -847,17 +847,10 @@ namespace sparta */ void enableCollection(TreeNode* parent) { - // Create the collector instance of the appropriate type. - sparta_assert(parent != nullptr); - collector_. - reset(new collection::IterableCollector - (parent, name_, this, capacity())); + collector_ = std::make_unique>(parent, name_, this, capacity()); if constexpr (ArrayT == ArrayType::AGED) { - age_collector_.reset(new collection::IterableCollector - (parent, name_ + "_age_ordered", - &aged_array_col_, capacity())); + age_collector_ = std::make_unique>(parent, name_ + "_age_ordered", &aged_array_col_, capacity()); } } @@ -869,6 +862,9 @@ namespace sparta //! Typedef for size_type typedef uint32_t size_type; + //! Typedef for value_type + typedef Array value_type; + AgedArrayCollectorProxy(FullArrayType * array) : array_(array) { } typedef FullArrayType::iterator iterator; @@ -1010,10 +1006,8 @@ namespace sparta //////////////////////////////////////////////////////////// // Collectors - std::unique_ptr> collector_; - std::unique_ptr > age_collector_; + std::unique_ptr> collector_; + std::unique_ptr> age_collector_; }; template diff --git a/sparta/sparta/resources/Buffer.hpp b/sparta/sparta/resources/Buffer.hpp index 9f23785ea5..dbb8e80085 100644 --- a/sparta/sparta/resources/Buffer.hpp +++ b/sparta/sparta/resources/Buffer.hpp @@ -19,9 +19,9 @@ #include "sparta/statistics/CycleHistogram.hpp" #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/StatisticDef.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/Counter.hpp" #include "sparta/utils/IteratorTraits.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -731,9 +731,7 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_. - reset(new collection::IterableCollector >(parent, getName(), - this, capacity())); + collector_ = std::make_unique(parent, name_, this, capacity()); } /** @@ -1020,7 +1018,8 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - std::unique_ptr > > collector_; + using IterableCollectorType = sparta::collection::IterableCollector>; + std::unique_ptr collector_; //! Flag which tells various methods if infinite_mode is turned on or not. // The behaviour of these methods change accordingly. @@ -1084,7 +1083,6 @@ namespace sparta num_valid_(rval.num_valid_), validator_(new DataPointerValidator(*this)), utilization_(std::move(rval.utilization_)), - collector_(std::move(rval.collector_)), is_infinite_mode_(rval.is_infinite_mode_), resize_delta_(std::move(rval.resize_delta_)), address_map_(std::move(rval.address_map_)){ @@ -1095,10 +1093,6 @@ namespace sparta rval.first_position_ = nullptr; rval.num_valid_ = 0; rval.utilization_ = nullptr; - rval.collector_ = nullptr; validator_->validator_ = std::move(rval.validator_->validator_); - if(collector_) { - collector_->reattach(this); - } } } diff --git a/sparta/sparta/resources/CircularBuffer.hpp b/sparta/sparta/resources/CircularBuffer.hpp index 3630fee22f..4bb359bcb1 100644 --- a/sparta/sparta/resources/CircularBuffer.hpp +++ b/sparta/sparta/resources/CircularBuffer.hpp @@ -19,9 +19,9 @@ #include "sparta/statistics/CycleCounter.hpp" #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/StatisticDef.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/Counter.hpp" #include "sparta/utils/IteratorTraits.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -442,9 +442,7 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_. - reset(new collection::IterableCollector >(parent, getName(), - this, capacity())); + collector_ = std::make_unique(parent, name_, this, capacity()); } //! Get this CircularBuffer's name @@ -788,7 +786,8 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - std::unique_ptr > > collector_; + using IterableCollectorType = sparta::collection::IterableCollector>; + std::unique_ptr collector_; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/sparta/sparta/resources/Pipe.hpp b/sparta/sparta/resources/Pipe.hpp index f041552a76..d8fce5155c 100644 --- a/sparta/sparta/resources/Pipe.hpp +++ b/sparta/sparta/resources/Pipe.hpp @@ -17,9 +17,11 @@ #include "sparta/ports/Port.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/MathUtils.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/utils/ValidValue.hpp" #include "sparta/utils/IteratorTraits.hpp" +#include "sparta/events/UniqueEvent.hpp" +#include "sparta/events/EventSet.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -221,7 +223,6 @@ class Pipe * finialization nor after enabling pipeline collection. */ void resize(uint32_t new_size) { - sparta_assert(collector_ == nullptr); //sparta_assert(!getClock()->isFinalized()); initPipe_(new_size); } @@ -513,15 +514,17 @@ class Pipe */ template void enableCollection(TreeNode * parent) { - collector_.reset (new collection::IterableCollector, phase, true> - (parent, name_, this, capacity())); + static_assert(phase == SchedulingPhase::Collection, + "TODO cnyce: Only Collection phase is supported for Pipe"); + + collector_ = std::make_unique(parent, name_, this, capacity()); } /** * \brief Check if pipe is collecting */ bool isCollected() const { - return collector_ && collector_->isCollected(); + return false; } private: @@ -593,8 +596,8 @@ class Pipe ////////////////////////////////////////////////////////////////////// // Collectors - std::unique_ptr collector_; - + using CollectorType = collection::IterableCollector, true>; + std::unique_ptr collector_; }; } diff --git a/sparta/sparta/resources/Pipeline.hpp b/sparta/sparta/resources/Pipeline.hpp index ba74effccc..628133a2bf 100644 --- a/sparta/sparta/resources/Pipeline.hpp +++ b/sparta/sparta/resources/Pipeline.hpp @@ -18,7 +18,6 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/MathUtils.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/resources/Pipe.hpp" #include "sparta/events/Scheduleable.hpp" diff --git a/sparta/sparta/resources/Queue.hpp b/sparta/sparta/resources/Queue.hpp index a88ef50bc2..eecd4db66b 100644 --- a/sparta/sparta/resources/Queue.hpp +++ b/sparta/sparta/resources/Queue.hpp @@ -17,9 +17,9 @@ #include "sparta/ports/Port.hpp" #include "sparta/statistics/CycleHistogram.hpp" #include "sparta/statistics/StatisticSet.hpp" -#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/InstrumentationNode.hpp" #include "sparta/utils/IteratorTraits.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -447,8 +447,7 @@ namespace sparta InstrumentationNode::visibility_t stat_vis_avg = InstrumentationNode::AUTO_VISIBILITY) : num_entries_(num_entries), vector_size_(nextPowerOfTwo_(num_entries*2)), - name_(name), - collector_(nullptr) + name_(name) { if((num_entries > 0) && statset) @@ -601,8 +600,7 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_.reset(new collection::IterableCollector > - (parent, name_, this, capacity())); + collector_ = std::make_unique(parent, name_, this, capacity()); } /** @@ -826,7 +824,8 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - std::unique_ptr > > collector_; + using IterableCollectorType = sparta::collection::IterableCollector>; + std::unique_ptr collector_; // Notice that our list for storing data is a dynamic array. // This is used instead of a stl vector to promote debug diff --git a/sparta/sparta/simulation/Clock.hpp b/sparta/sparta/simulation/Clock.hpp index c4ff0e65e3..0ba4db0577 100644 --- a/sparta/sparta/simulation/Clock.hpp +++ b/sparta/sparta/simulation/Clock.hpp @@ -24,18 +24,11 @@ #include "sparta/statistics/ReadOnlyCounter.hpp" #include "sparta/utils/Rational.hpp" #include "sparta/utils/MathUtils.hpp" -#include "simdb_fwd.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" #include "sparta/statistics/CounterBase.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/SpartaException.hpp" #include "sparta/statistics/StatisticSet.hpp" -namespace simdb { - class TableRef; - class ObjectManager; -} // namespace simdb - namespace sparta { /** @@ -284,12 +277,6 @@ namespace sparta os << std::endl; } - //! Persist clock hierarchy in the provided database, - //! treating 'this' sparta::Clock as the hierarchy root. - //! Returns the database ID of the clock node that was - //! put into this database. - simdb::DatabaseID serializeTo(const simdb::ObjectManager & sim_db) const; - // Overload of TreeNode::stringize virtual std::string stringize(bool pretty=false) const override { (void) pretty; @@ -356,17 +343,6 @@ namespace sparta return clk_.currentCycle(); } } cycles_roctr_ = {*this, &sset_}; - - // Persist clock hierarchy in the provided database, - // recursing down through child nodes as necessary. - // Output argument 'db_ids' tracks database record - // ID's for each sparta::Clock that was written to - // the clock hierarchy table. The map keys are - // sparta::Clock 'this' pointers. - void recursSerializeToTable_( - simdb::TableRef & clock_tbl, - const simdb::DatabaseID parent_clk_id, - std::map & db_ids) const; }; inline std::ostream& operator<<(std::ostream& os, const sparta::Clock& clk) diff --git a/sparta/sparta/simulation/Resource.hpp b/sparta/sparta/simulation/Resource.hpp index e7d808fe6f..48a7bde730 100644 --- a/sparta/sparta/simulation/Resource.hpp +++ b/sparta/sparta/simulation/Resource.hpp @@ -19,7 +19,6 @@ #include "sparta/utils/SpartaAssert.hpp" #include "sparta/simulation/TreeNode.hpp" #include "sparta/utils/Utils.hpp" -#include "simdb/Errors.hpp" namespace sparta { diff --git a/sparta/sparta/statistics/StatisticInstance.hpp b/sparta/sparta/statistics/StatisticInstance.hpp index aa2bd72ac5..dc0c88ca9e 100644 --- a/sparta/sparta/statistics/StatisticInstance.hpp +++ b/sparta/sparta/statistics/StatisticInstance.hpp @@ -26,9 +26,6 @@ namespace sparta { - //Forward declarations related to SimDB - class StatInstValueLookup; - class StatInstRowIterator; using statistics::expression::Expression; /*! @@ -272,38 +269,6 @@ namespace sparta } } - /*! - * \brief During SimDB->report generation, there is - * a notion of "placeholder" objects which get set - * on StatisticInstance/Report objects temporarily. - * These placeholders can be cloned into "realized" - * versions of themselves later on. - * - * This method lets SimDB-recreated Report objects - * set placeholders this SI will soon use to get - * SI data values directly from a SimDB blob (not - * from an actual simulation). - */ - void setSIValueDirectLookupPlaceholder( - const std::shared_ptr & direct_lookup); - - /*! - * \brief Our StatInstValueLookup *placeholder* object - * needs to bind itself to a StatInstRowIterator object, - * since these two classes go hand in hand. Now that we're - * being given the row iterator, we can use it to "realize" - * our "SI direct value lookup" object now. - */ - void realizeSIValueDirectLookup( - const StatInstRowIterator & si_row_iterator); - - /*! - * \brief If this SI is using a StatInstValueLookup object - * to get its SI values, ask if this direct-lookup object - * can be used to get the current SI value. - */ - bool isSIValueDirectLookupValid() const; - /*! * \brief Returns the value computed for this statistic instance at the * current time @@ -437,15 +402,6 @@ namespace sparta */ InstrumentationNode::class_t getClass() const; - /*! - * \brief Give the reporting infrastructure access to all metadata - * that has been set. The database report writers need this metadata, - * and others may need it as well. - */ - const std::vector> & getMetadata() const { - return provided_metadata_; - } - /*! * \brief Returns the StatisticDef used to compute this statistic */ @@ -527,19 +483,6 @@ namespace sparta */ double computeValue_() const; - /*! - * \brief Ask the StatInstValueLookup object for our current - * SI value. Throws an exception if the direct-value object - * is not being used. - * - * This does not apply to normal, in-simulation SI's. This - * supports post-simulation SimDB workflows only. This - * method is not implemented inline in this header so that - * SimDB headers aren't included in downstream builds that - * don't care about it. - */ - double getCurrentValueFromDirectLookup_() const; - /*! * \brief Append one pending substatistic for future creation (and addition to the * appropriate report) @@ -677,41 +620,11 @@ namespace sparta */ mutable std::vector sub_statistics_; - /*! + /*! * \brief User-provided callback which generates the stat value */ std::shared_ptr user_calculated_si_value_; - /*! - * \brief SimDB-recreated StatisticInstance's do not get their - * SI values from CounterBase/ParameterBase/StatisticDef objects - * like live-simulation SI's do. Those SI's which are created - * from SimDB records are bound to their SI value blobs using - * StatInstValueLookup objects. These are lightweight wrappers - * around a shared vector, who know their individual - * element index/offsets into that vector. - */ - std::shared_ptr direct_lookup_si_value_; - - /*! - * \brief Typically, SI's will defer to their underlying counter/ - * parameter/StatDef for properties like location and description. - * But some SI's may not have these internal pieces (counters and - * such) because they are being created outside of a simulation, - * and outside of a device tree. - * - * These member variables are prefixed with "provided_" to mean - * that they were *provided* these values directly during SI - * construction. - */ - utils::ValidValue provided_location_; - utils::ValidValue provided_description_; - utils::ValidValue provided_expr_string_; - utils::ValidValue provided_value_semantic_; - utils::ValidValue provided_visibility_; - utils::ValidValue provided_class_; - std::vector> provided_metadata_; - }; // class StatisticInstance diff --git a/sparta/sparta/statistics/db/SINodeHierarchy.hpp b/sparta/sparta/statistics/db/SINodeHierarchy.hpp deleted file mode 100644 index e1aa53ec57..0000000000 --- a/sparta/sparta/statistics/db/SINodeHierarchy.hpp +++ /dev/null @@ -1,62 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include - -#include "simdb_fwd.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" - -namespace simdb { -class ObjectManager; -} // namespace simdb - -namespace sparta { - -class Report; - -namespace db { - class ReportTimeseries; -} - -namespace statistics { - -/*! - * \brief This class serializes a sparta::Report's entire - * SI tree (node names, parent nodes / child nodes / etc.) - * into the SINodeHierarchy table in the database object - * you provide. - * - * TODO: This is specifically for timeseries reports. - * All other report formats go through ReportNodeHierarchy - * to get their report/SI trees written to a different - * table. It would be easier to work with a schema that - * can put timeseries and non-timeseries report hierarchies - * and metadata into the same set of tables, but for now - * they are separate. - */ -class SINodeHierarchy -{ -public: - //! Construct with the sparta::Report we are serializing, - //! and the database ReportTimeseries object we are writing - //! all report information into. - SINodeHierarchy( - db::ReportTimeseries & db_timeseries, - const Report & report); - - //! Write out all report/subreport/SI hierarchy metadata - //! for this report into the provided database. Returns - //! the database ID corresponding to the root-level Report - //! node in this hierarchy. - simdb::DatabaseID serializeHierarchy(simdb::ObjectManager & obj_mgr); - -private: - class Impl; - - std::shared_ptr impl_; -}; - -} // namespace statistics -} // namespace sparta - diff --git a/sparta/sparta/statistics/db/SIValuesBuffer.hpp b/sparta/sparta/statistics/db/SIValuesBuffer.hpp deleted file mode 100644 index c68e4fe764..0000000000 --- a/sparta/sparta/statistics/db/SIValuesBuffer.hpp +++ /dev/null @@ -1,348 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "sparta/statistics/StatisticInstance.hpp" -#include "simdb/schema/Schema.hpp" -#include "sparta/report/db/Schema.hpp" - -namespace sparta { -namespace statistics { - -//! Utility to set all elements of a vector to NaN -inline void refillWithNaNs(std::vector & vec) -{ - vec = std::vector(vec.size(), NAN); -} - -/*! - * \brief This class helps organize contiguous blocks of - * SI values. These values are buffered at each report - * update, and they are organized in the buffer so that - * individual SI's have their values right next to each - * other. To illustrate, say we had the following CSV - * file: - * - * si_foo si_bar si_biz si_baz - * 1.2 450 1000 12 - * 1.4 453 1001 12 - * 1.4 460 1005 14 - * - * An SIValuesBuffer could be used so that these 12 values - * appear in this order in a single vector: - * - * [1.2, 1.4, 1.4, 450, 453, 460, 1000, 1001, 1005, 12, 12, 14] - * - * This is useful when SI values are compressed since data - * streams with lower entropy tend to compress better than - * those with higher entropy (depends on compression scheme - * used). Adjacent SI's will usually display smaller changes - * from one update to the next, which is why this class buffers - * them together in column-major format. The equivalent buffer - * in row-major format would be much more random and would likely - * show more modest benefits from compression: - * - * [1.2, 450, 1000, 12, 1.4, 453, 1001, 12, 1.4, 460, 1005, 14] - */ -class SIValuesBuffer -{ -public: - //! Construct an empty buffer for a given set of SI's, and - //! the scheduler & root clock the simulation is tied to. - SIValuesBuffer(const std::vector & stats, - const Clock & root_clk) : - stats_(stats), - scheduler_(*root_clk.getScheduler()), - root_clk_(root_clk) - { - //We are going to be asking the SI's for their values - //ourselves. Don't take the performance hit of having - //them writing their values into SnapshotLogger's that - //nobody is listening to. - for (const auto si : stats_) { - si->disableSnapshotLogging(); - } - } - - //! Switch this buffer to start using row-major ordering - //! as it fills its internal SI buffers. This is the default - //! behavior. - //! \note Must be called when buffersAreEmpty() - void useRowMajorOrdering() { - if (!buffersAreEmpty()) { - throw SpartaException( - "Cannot change row/column-major ordering when " - "SIValuesBuffer contains buffered data"); - } - is_row_major_ = true; - } - - //! Switch this buffer to start using column-major ordering - //! as it fills its internal SI buffers. - //! \note Must be called when buffersAreEmpty() - void useColumnMajorOrdering() { - if (!buffersAreEmpty()) { - throw SpartaException( - "Cannot change row/column-major ordering when " - "SIValuesBuffer contains buffered data"); - } - is_row_major_ = false; - } - - //! Ask this buffer if it is using row-major or column- - //! major SI ordering - db::MajorOrdering getMajorOrdering() const { - return is_row_major_ ? - db::MajorOrdering::ROW_MAJOR : - db::MajorOrdering::COLUMN_MAJOR; - } - - //! Initialize the number of SI buffers this container - //! should be able to hold. In the comment above this - //! class, that string of SI values had *three* buffers - //! for *four* SI's. - //! - //! The number of SI buffers you choose will dictate - //! how many report updates can hit before this container - //! is full and needs to be consumed (written to disk). - void initializeNumSIBuffers(const size_t num_si_buffers) { - si_values_buffer_.resize(num_si_buffers * stats_.size()); - max_num_si_buffers_ = num_si_buffers; - } - - //! Tell this SIValuesBuffer to update how many contiguous - //! blocks of SI's it can hold. This will not take effect - //! until right after this container is reset/cleared. - //! - //! SIValuesBuffer buf(stats); - //! buf.initializeNumSIBuffers(3); - //! ... - //! buf.bufferCurrentSIValues(); <-- report update - //! buf.bufferCurrentSIValues(); <-- report update - //! buf.updateNumSIBuffers(2); <-- no effect yet - //! buf.bufferCurrentSIValues(); <-- report update - //! - //! if (buf.buffersAreFilled()) { - //! //call getBufferedSIValues() and flush the data - //! buf.resetSIBuffers(); <-- resized to 2 SI blocks - //! } - //! - //! buf.bufferCurrentSIValues(); <-- report update - //! buf.bufferCurrentSIValues(); <-- report update - //! buf.bufferCurrentSIValues(); <-- ASSERT! We don't - //! have space for a - //! third SI block! - void updateNumSIBuffers(const size_t num_si_buffers) { - sparta_assert(num_si_buffers > 0, "You cannot have an " - "SIValuesBuffer with zero SI capacity"); - updated_num_si_buffers_ = num_si_buffers; - } - - //! Ask if this buffer has any room for another SI block. - //! You should call this before bufferCurrentSIValues() - //! is called during each report update. If you try to - //! call bufferCurrentSIValues() and the buffer is full, - //! it will assert. - bool buffersAreFilled() const { - return (current_buffer_write_idx_ == max_num_si_buffers_); - } - - //! Ask if this buffer is completely empty - bool buffersAreEmpty() const { - return (current_buffer_write_idx_ == 0); - } - - //! Ask this container how many blocks of SI values it - //! currently has buffered. - size_t getNumBufferedSIBlocks() const { - return current_buffer_write_idx_; - } - - //! Typically, you will only call this method right after - //! you get all the buffered SI data out of this container - //! and consume it first. - //! - //! This applies any pending updated # of SI blocks that - //! you set if you previously called updateNumSIBuffers() - void resetSIBuffers(const bool fill_with_nans = true) { - current_buffer_write_idx_ = 0; - if (updated_num_si_buffers_.isValid()) { - initializeNumSIBuffers(updated_num_si_buffers_); - updated_num_si_buffers_.clearValid(); - } - - if (fill_with_nans) { - refillWithNaNs(si_values_buffer_); - } - - si_buffer_beginning_picoseconds_.clearValid(); - si_buffer_ending_picoseconds_.clearValid(); - si_buffer_beginning_clock_cycles_.clearValid(); - si_buffer_ending_clock_cycles_.clearValid(); - } - - //! Loop over this container's SI's and put their current - //! values into the buffer. Each SI value will go just to - //! the right of its previous value. For example: - //! - //! Say the container has 4 SI's, can hold a maximum of - //! three report update's worth of SI data, and currently - //! 2 of those 3 report updates have already hit. - //! - //! [1.2, 1.4, ---, 450, 453, ---, 1000, 1001, ---, 12, 12, ---] - //! *** *** *** *** **** **** ** ** - //! | | | | | | | | - //! ----------------------------------------------- | - //! | | | | | - //! | | | Update #1 | - //! | | | | - //! ---------------------------------------------- - //! | - //! Update #2 - //! - //! Then we call 'bufferCurrentSIValues()', and our SI's - //! have values 1.4, 460, 1005, and 14 at this moment. - //! - //! We would then have the following SI values at the - //! end of the third report update: - //! - //! [1.2, 1.4, 1.4, 450, 453, 460, 1000, 1001, 1005, 12, 12, 14] - //! *** *** **** ** - //! | | | | - //! ---------------------------------------------- - //! | - //! Update #3 - void bufferCurrentSIValues() { - //Ensure that we have the space in our buffer to append the - //current SI values - sparta_assert(current_buffer_write_idx_ < max_num_si_buffers_); - - //Capture the current simulated picoseconds & root clock cycle - //if this is the first write into a fresh buffer - if (buffersAreEmpty()) { - si_buffer_beginning_picoseconds_ = scheduler_.getSimulatedPicoSeconds(); - si_buffer_beginning_clock_cycles_ = root_clk_.currentCycle(); - } - - //For row-major ordering, we start the write index at the Nth SI - //position, where N equals the current buffer index. For column- - //major ordering, we start the write index at the 0th position - //of the Mth buffer, where M equals the current buffer index. - size_t buffer_idx = - is_row_major_ ? - current_buffer_write_idx_ * stats_.size() : - current_buffer_write_idx_; - for (const auto si : stats_) { - si_values_buffer_[buffer_idx] = si->getValue(); - //Row-major ordering jumps the write index ahead by 1. - //Column-major ordering jumps this index to the next - //available buffer. - buffer_idx += is_row_major_ ? 1 : max_num_si_buffers_; - } - ++current_buffer_write_idx_; - - //Capture the ending simulated picoseconds & root clock cycle - //in this buffer - si_buffer_ending_picoseconds_ = scheduler_.getSimulatedPicoSeconds(); - si_buffer_ending_clock_cycles_ = root_clk_.currentCycle(); - } - - //! Ask this container for all of its buffered SI values. If this - //! container is empty, it will return a vector of NaN's. If it - //! is *partially* filled, it will squeeze the SI values like so: - //! - //! Say we have 4 SI's, a maximum of 3 report updates before - //! this container is filled, and 2 of those updates have hit. - //! - //! [1.2, 1.4, ---, 450, 453, ---, 1000, 1001, ---, 12, 12, ---] - //! - //! If you called this method at this time, it would return a - //! vector of size 8 (2 report updates * 4 SI's) - //! - //! [1.2, 1.4, 450, 453, 1000, 1001, 12, 12] - //! - //! This is more expensive than asking for the buffered data when - //! the container is full, and we only do this at the end of the - //! simulation when we need to get any leftover report updates' - //! SI values out of the buffer and written to disk. - const std::vector & getBufferedSIValues() { - if (buffersAreFilled()) { - return si_values_buffer_; - } - - if (buffersAreEmpty()) { - refillWithNaNs(si_values_buffer_); - return si_values_buffer_; - } - - const size_t num_filled_buffers = current_buffer_write_idx_; - squeezed_si_values_.resize(num_filled_buffers * stats_.size()); - auto read_iter = si_values_buffer_.begin(); - - //When using row-major ordering, simply copy the SI values - //buffer as-is into the squeezed SI values vector. - if (is_row_major_) { - memcpy(&squeezed_si_values_[0], - &si_values_buffer_[0], - squeezed_si_values_.size() * sizeof(double)); - } else { - //Column-major ordering is a little bit different. - //For each "block" of buffers, whether filled or unfilled... - for (size_t si_idx = 0; si_idx < stats_.size(); ++si_idx) { - //...copy over the SI values from the filled buffer slots... - std::copy(read_iter, - read_iter + num_filled_buffers, - squeezed_si_values_.begin() + (si_idx * num_filled_buffers)); - - //...and jump over the unfilled buffer slots to the start - //of the next filled buffer. - std::advance(read_iter, max_num_si_buffers_); - } - } - - return squeezed_si_values_; - } - - //! Get the starting and ending simulated picoseconds and root clock cycle - //! for the SI's in this buffer. - //! - //! This will *assert* if the buffer is empty. Call either buffersAreFilled(), - //! buffersAreEmpty(), or getNumBufferedSIBlocks() first to see if calling - //! this method here is even valid. - void getBeginningAndEndingTimestampsForBufferedSIs( - uint64_t & starting_picoseconds, - uint64_t & ending_picoseconds, - uint64_t & starting_cycles, - uint64_t & ending_cycles) const - { - starting_picoseconds = si_buffer_beginning_picoseconds_; - ending_picoseconds = si_buffer_ending_picoseconds_; - starting_cycles = si_buffer_beginning_clock_cycles_; - ending_cycles = si_buffer_ending_clock_cycles_; - } - -private: - const std::vector stats_; - std::vector si_values_buffer_; - std::vector squeezed_si_values_; - size_t current_buffer_write_idx_ = 0; - size_t max_num_si_buffers_ = 0; - utils::ValidValue updated_num_si_buffers_; - bool is_row_major_ = true; - - utils::ValidValue si_buffer_beginning_picoseconds_; - utils::ValidValue si_buffer_ending_picoseconds_; - utils::ValidValue si_buffer_beginning_clock_cycles_; - utils::ValidValue si_buffer_ending_clock_cycles_; - - //! Simulation's scheduler and root clock. Used in order to get - //! the current "time values" when we are asked to write the SI - //! blobs into the database. - const Scheduler & scheduler_; - const Clock & root_clk_; - -}; - -} // namespace statistics -} // namespace sparta - diff --git a/sparta/sparta/utils/MathUtils.hpp b/sparta/sparta/utils/MathUtils.hpp index 0e7b9c24ba..33cde0eac6 100644 --- a/sparta/sparta/utils/MathUtils.hpp +++ b/sparta/sparta/utils/MathUtils.hpp @@ -8,9 +8,11 @@ #include #include #include +#include #include "sparta/utils/SpartaException.hpp" #include "sparta/utils/SpartaAssert.hpp" + namespace sparta { namespace utils { @@ -351,5 +353,55 @@ namespace sparta { return result; } + //! \brief Comparison of two floating-point values with + //! a supplied tolerance. The tolerance value defaults + //! to machine epsilon. + template + inline + typename std::enable_if< + std::is_floating_point::value, + bool>::type + approximatelyEqual(const T a, const T b, + const T epsilon = std::numeric_limits::epsilon()) + { + const T fabs_a = std::fabs(a); + const T fabs_b = std::fabs(b); + const T fabs_diff = std::fabs(a - b); + + return fabs_diff <= ((fabs_a < fabs_b ? fabs_b : fabs_a) * epsilon); + } + + //! Static/global random number generator + struct RandNumGen { + static std::mt19937 & get() { + static std::mt19937 rng(time(nullptr)); + return rng; + } + }; + + //! \brief Pick a random integral number + template + inline + typename std::enable_if< + std::is_integral::value, + T>::type + chooseRand() + { + std::uniform_int_distribution dist; + return dist(RandNumGen::get()); + } + + //! \brief Pick a random floating-point number + template + inline + typename std::enable_if< + std::is_floating_point::value, + T>::type + chooseRand() + { + std::normal_distribution dist(0, 1000); + return dist(RandNumGen::get()); + } + } // utils } // sparta diff --git a/sparta/sparta/utils/MetaStructs.hpp b/sparta/sparta/utils/MetaStructs.hpp index b461db4b55..50b13ac0eb 100644 --- a/sparta/sparta/utils/MetaStructs.hpp +++ b/sparta/sparta/utils/MetaStructs.hpp @@ -118,6 +118,14 @@ namespace MetaStruct { all_are_integral::value}; }; + /** + * \brief Check for POD types. + */ + template + struct is_pod { + static constexpr bool value = std::is_pod::value; + }; + /** \brief Alias Template for std::enable_if. */ template @@ -396,6 +404,11 @@ namespace MetaStruct { is_stl_container::type>::value; }; + template + struct is_pod { + static constexpr bool value = std::is_trivial::value && std::is_standard_layout::value; + }; + /** * \brief This Variadic templated struct contains a nested value * which stores the length of any parameter pack it gets templatized on. diff --git a/sparta/sparta/utils/SpartaSharedPointer.hpp b/sparta/sparta/utils/SpartaSharedPointer.hpp index fcdc79690e..074a4e4545 100644 --- a/sparta/sparta/utils/SpartaSharedPointer.hpp +++ b/sparta/sparta/utils/SpartaSharedPointer.hpp @@ -14,6 +14,7 @@ #include "sparta/utils/Utils.hpp" #include "sparta/utils/MetaStructs.hpp" +#include "simdb/utils/MetaStructs.hpp" #include "sparta/utils/SpartaSharedPointerBaseAllocator.hpp" namespace sparta @@ -584,3 +585,35 @@ namespace MetaStruct { template struct remove_any_pointer const &> { using type = T; }; } + +// Helper methods to determine pointer type and/or remove it +namespace simdb { +namespace meta_utils { + + // Helper structs + template + struct is_any_pointer> : public std::true_type {}; + + template + struct is_any_pointer const> : public std::true_type {}; + + template + struct is_any_pointer &> : public std::true_type {}; + + template + struct is_any_pointer const &> : public std::true_type {}; + + template + struct remove_any_pointer> { using type = T; }; + + template + struct remove_any_pointer const> { using type = T; }; + + template + struct remove_any_pointer &> { using type = T; }; + + template + struct remove_any_pointer const &> { using type = T; }; + +} // namespace meta_utils +} // namespace simdb diff --git a/sparta/sparta/utils/SpartaTester.hpp b/sparta/sparta/utils/SpartaTester.hpp index 72e8503178..d819c35f39 100644 --- a/sparta/sparta/utils/SpartaTester.hpp +++ b/sparta/sparta/utils/SpartaTester.hpp @@ -18,7 +18,7 @@ #include #include #include "sparta/utils/Colors.hpp" -#include "simdb/utils/MathUtils.hpp" +#include "sparta/utils/MathUtils.hpp" namespace sparta { @@ -223,7 +223,7 @@ namespace sparta ++num_errors_; ret = false; } else { - ret = simdb::utils::approximatelyEqual(v1, v2, tol); + ret = utils::approximatelyEqual(v1, v2, tol); if (!ret) { cerr_ << SPARTA_CURRENT_COLOR_BRIGHT_RED << "Test '" << test_type << "' FAILED on line " diff --git a/sparta/src/AsyncNonTimeseriesReport.cpp b/sparta/src/AsyncNonTimeseriesReport.cpp deleted file mode 100644 index bea472332b..0000000000 --- a/sparta/src/AsyncNonTimeseriesReport.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// -*- C++ -*- - -#include "sparta/async/AsyncNonTimeseriesReport.hpp" - -#include - -//SQLite-specific headers -#include - -namespace sparta { -namespace async { - -//! Compress SI values and write them to the database. -//! This is called on a background thread. -void AsyncNonTimeseriesReport::StatInstValuesWriter::completeTask() -{ - if (using_compression_) { - z_stream defstream; - defstream.zalloc = Z_NULL; - defstream.zfree = Z_NULL; - defstream.opaque = Z_NULL; - - //Setup the source stream - auto num_bytes_before = (uint32_t)(si_values_.size() * sizeof(double)); - defstream.avail_in = (uInt)(num_bytes_before); - defstream.next_in = (Bytef*)(&si_values_[0]); - - //Setup the destination stream - std::vector compressed_si_values(num_bytes_before); - defstream.avail_out = (uInt)(compressed_si_values.size()); - defstream.next_out = (Bytef*)(&compressed_si_values[0]); - - //Compress it! - deflateInit(&defstream, Z_DEFAULT_COMPRESSION); - deflate(&defstream, Z_FINISH); - deflateEnd(&defstream); - - //Now send this compressed SI blob to the database. - compressed_si_values.resize(defstream.total_out); - const uint32_t original_num_si_values = si_values_.size(); - - si_values_writer_->writeCompressedStatisticInstValues( - compressed_si_values, - original_num_si_values); - } - - else { - si_values_writer_->writeStatisticInstValues(si_values_); - } -} - -} // namespace async -} // namespace sparta diff --git a/sparta/src/AsyncTimeseriesReport.cpp b/sparta/src/AsyncTimeseriesReport.cpp deleted file mode 100644 index 33577c6f5a..0000000000 --- a/sparta/src/AsyncTimeseriesReport.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// -*- C++ -*- - -#include "sparta/async/AsyncTimeseriesReport.hpp" - -//SQLite-specific headers -#include - -//! Compress buffered SI values and write them to the database. -//! This is called on a background thread. -void sparta::async::AsyncTimeseriesReport::CompressedStatInstValuesWriter::completeTask() -{ - z_stream defstream; - defstream.zalloc = Z_NULL; - defstream.zfree = Z_NULL; - defstream.opaque = Z_NULL; - - //Setup the source stream - auto num_bytes_before = (uint32_t)(si_values_.size() * sizeof(double)); - defstream.avail_in = (uInt)(num_bytes_before); - defstream.next_in = (Bytef*)(&si_values_[0]); - - //Setup the destination stream - std::vector compressed_si_values(num_bytes_before); - defstream.avail_out = (uInt)(compressed_si_values.size()); - defstream.next_out = (Bytef*)(&compressed_si_values[0]); - - //Compress it! - deflateInit(&defstream, Z_DEFAULT_COMPRESSION); - deflate(&defstream, Z_FINISH); - deflateEnd(&defstream); - - //How did we do? - auto num_bytes_after = (uint32_t)(defstream.total_out); - - //Before writing the data to the database, let's tell - //the async timeseries object how much compression we - //just saw. This gives it a chance to make some tweaks - //to its buffers to try for more compression going - //forward if possible. - if (post_compression_callback_.isValid()) { - post_compression_callback_.getValue()(num_bytes_before, num_bytes_after); - } - - compressed_si_values.resize(num_bytes_after); - - //Now send this compressed SI blob to the database. - const uint32_t original_num_si_values = si_values_.size(); - - db_timeseries_->writeCompressedStatisticInstValuesInTimeRange( - starting_picoseconds_, - ending_picoseconds_, - starting_cycles_, - ending_cycles_, - compressed_si_values, - major_ordering_, - original_num_si_values); -} diff --git a/sparta/src/Clock.cpp b/sparta/src/Clock.cpp index 1ad094b3cf..470e5b3e09 100644 --- a/sparta/src/Clock.cpp +++ b/sparta/src/Clock.cpp @@ -6,10 +6,6 @@ #include -#include "simdb/ObjectManager.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/ObjectRef.hpp" - namespace sparta { Clock::Clock(const std::string& name, Scheduler *scheduler) : TreeNode(name, "Clock"), @@ -79,45 +75,4 @@ namespace sparta { normalized_ = false; } - //! Persist clock hierarchy in the provided database, - //! treating 'this' sparta::Clock as the hierarchy root. - simdb::DatabaseID Clock::serializeTo(const simdb::ObjectManager & sim_db) const - { - auto clock_tbl = sim_db.getTable("ClockHierarchy"); - if (!clock_tbl) { - return 0; - } - - std::map db_ids; - - sim_db.safeTransaction([&]() { - recursSerializeToTable_(*clock_tbl, 0, db_ids); - }); - - auto iter = db_ids.find(this); - sparta_assert(iter != db_ids.end()); - return iter->second; - } - - //! Persist clock hierarchy in the provided database, - //! recursing down through child nodes as necessary. - void Clock::recursSerializeToTable_( - simdb::TableRef & clock_tbl, - const simdb::DatabaseID parent_clk_id, - std::map & db_ids) const - { - auto row = clock_tbl.createObjectWithArgs( - "ParentClockID", parent_clk_id, - "Name", getName(), - "Period", getPeriod(), - "FreqMHz", frequency_mhz_, - "RatioToParent", static_cast(getRatio())); - - for (const auto child : children_) { - child->recursSerializeToTable_(clock_tbl, row->getId(), db_ids); - } - - db_ids[this] = row->getId(); - } - } diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index f9eee50bb2..cc8c3e496e 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -34,15 +34,12 @@ #include "sparta/utils/StringUtils.hpp" #include "sparta/utils/ValidValue.hpp" #include "sparta/report/format/BaseFormatter.hpp" -#include "sparta/report/db/ReportVerifier.hpp" #include "sparta/parsers/ConfigEmitterYAML.hpp" -#include "sparta/report/DatabaseInterface.hpp" // // For filtered printouts #include "sparta/statistics/Counter.hpp" #include "sparta/ports/Port.hpp" #include "sparta/utils/File.hpp" #include "sparta/kernel/SleeperThread.hpp" -#include "simdb/ObjectManager.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/utils/Colors.hpp" #include "sparta/simulation/GlobalTreeNode.hpp" @@ -145,8 +142,6 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, pipeout_opts_("Pipeline-Collection Options", OPTIONS_DOC_WIDTH), log_opts_("Logging Options", OPTIONS_DOC_WIDTH), report_opts_("Report Options", OPTIONS_DOC_WIDTH), - simdb_opts_("SimDB Options", OPTIONS_DOC_WIDTH), - simdb_internal_opts_("SimDB Options (internal / developer use)", OPTIONS_DOC_WIDTH), app_opts_("Application-Specific Options", OPTIONS_DOC_WIDTH), feature_opts_("Feature Evaluation Options", OPTIONS_DOC_WIDTH), advanced_opts_("Advanced Options", OPTIONS_DOC_WIDTH) @@ -585,10 +580,6 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, "Disable pretty print / verbose print for all JSON statistics reports") ("omit-zero-value-stats-from-json_reduced", "Omit all statistics that have value 0 from json_reduced statistics reports") - ("report-verif-output-dir", - named_value>("DIR_NAME", 1, 1), - "When SimDB report verification is enabled, this option will send all verification " - "artifacts to the specified directory, relative to the current working directory.") ("report-warmup-icount", named_value(""), "DEPRECATED") @@ -615,29 +606,6 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, "Writes all reports even when run exits with error.") ; - // SimDB Options - simdb_opts_.add_options() - ("simdb-dir", - named_value>("DIR", 1, 1), - "Specify the location where the simulation database will be written") - ("simdb-enabled-components", - named_value>>("", 1, INT_MAX)->multitoken(), - "Specify which simulator components should be enabled for SimDB access.\n" - "Example: \"--simdb-enabled-components dbaccess.yaml\"") - ; - - // SimDB Options (internal / developer use) - simdb_internal_opts_.add_options() - ("collect-legacy-reports", - named_value>("DIR", 1, INT_MAX)->multitoken(), - "Specify the root directory where all legacy report files will be written. " - "This directory will be created if needed. Optionally supply one or more " - "specific report format types that you *only* want to be collected, otherwise " - "all report formats will be collected by default.\n" - "Example: \"--collect-legacy-reports test/report/dir\"\n" - "Example: \"--collect-legacy-reports test/report/dir json_reduced csv_cumulative\"") - ; - // Feature Options feature_opts_.add_options() ("feature", @@ -716,8 +684,6 @@ bool CommandLineSimulator::parse(int argc, .add(log_opts_.getVerboseOptions()) .add(pipeout_opts_.getVerboseOptions()) .add(report_opts_.getVerboseOptions()) - .add(simdb_opts_.getVerboseOptions()) - .add(simdb_internal_opts_.getVerboseOptions()) .add(app_opts_.getVerboseOptions()) .add(feature_opts_.getVerboseOptions()) .add(advanced_opts_.getVerboseOptions()); @@ -1115,9 +1081,6 @@ bool CommandLineSimulator::parse(int argc, } sim_config_.setMemoryUsageDefFile(def_file); opts.options.erase(opts.options.begin() + i); - }else if (o.string_key == "report-verif-output-dir") { - db::ReportVerifier::writeVerifResultsTo(o.value[0]); - opts.options.erase(opts.options.begin() + i); }else if (o.string_key == "report-warmup-icount") { throw_report_deprecated = true; ++i; @@ -1430,49 +1393,6 @@ bool CommandLineSimulator::parse(int argc, std::cout << " set infinite loop protection timeout to " << seconds << " seconds" << std::endl; SleeperThread::getInstance()->setInfLoopSleepInterval(std::chrono::seconds(seconds)); ++i; - } else if(o.string_key == "simdb-dir") { - const std::string & db_dir = o.value[0]; - auto p = sfs::path(db_dir); - if (!sfs::exists(p)) { - sfs::create_directories(p); - } else if (!sfs::is_directory(p)) { - throw SpartaException("Invalid 'simdb-dir' argument. Path ") - << "exists but is not a directory."; - } - sim_config_.setSimulationDatabaseLocation(db_dir); - ++i; - } else if(o.string_key == "simdb-enabled-components") { - std::vector yaml_opts_files; - auto is_yaml_file = [](const std::string & opt) { - auto p = sfs::path(opt); - return (sfs::exists(p) && !sfs::is_directory(p)); - }; - - for (size_t idx = 0; idx < o.value.size(); ++idx) { - sparta_assert(is_yaml_file(o.value[idx]), - "File not found: " << o.value[idx]); - yaml_opts_files.emplace_back(o.value[idx]); - } - - for (const auto & opt_file : yaml_opts_files) { - sim_config_.addSimulationDatabaseAccessOptsYaml(opt_file); - } - ++i; - } else if(o.string_key == "collect-legacy-reports") { - const std::string & reports_root_dir = o.value[0]; - auto p = sfs::path(reports_root_dir); - if (!sfs::exists(p)) { - sfs::create_directories(p); - } else if (!sfs::is_directory(p)) { - throw SpartaException("Invalid 'collect-legacy-reports' argument. Path ") - << "exists but is not a directory."; - } - std::set collected_formats; - for (size_t idx = 1; idx < o.value.size(); ++idx) { - collected_formats.insert(o.value[idx]); - } - sim_config_.setLegacyReportsCopyDir(reports_root_dir, collected_formats); - ++i; } else if(o.string_key == "feature") { const std::string & name = o.value[0]; const int value = boost::lexical_cast(o.value[1]); @@ -1847,6 +1767,18 @@ bool CommandLineSimulator::parse(int argc, sim_config_.report_on_error = vm_.count("report-on-error") > 0; sim_config_.reports = reports; + if (sim_config_.pipeline_collection_file_prefix != NoPipelineCollectionStr) { + auto& simdb_filename = sim_config_.pipeline_collection_file_prefix; + auto fs_path = std::filesystem::path(simdb_filename); + if (!fs_path.has_extension() || fs_path.extension() != ".db") { + simdb_filename += ".db"; + } + + if (pipeline_heartbeat_ == DefaultHeartbeat) { + pipeline_heartbeat_ = "10"; + } + } + //pevents run_pevents_ = (vm_.count("pevents-at") > 0) || (vm_.count("pevents") > 0) || (vm_.count("verbose-pevents") > 0); @@ -1910,7 +1842,7 @@ bool CommandLineSimulator::parse(int argc, //print out some stuff about the pipeline collections run status. std::cout << " pipeline-collection: " << std::boolalpha << collecting << std::endl; if(collecting){ - std::cout << " output dir: " << sim_config_.pipeline_collection_file_prefix << std::endl; + std::cout << " simdb file: " << sim_config_.pipeline_collection_file_prefix << std::endl; std::cout << " pipeline heartbeat: " << pipeline_heartbeat_ << std::endl; } } @@ -1948,20 +1880,14 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) } // Convert heartbeat command line string to int - uint32_t heartbeat; try{ size_t end_pos; - heartbeat = utils::smartLexicalCast(pipeline_heartbeat_, end_pos); + utils::smartLexicalCast(pipeline_heartbeat_, end_pos); }catch (SpartaException const&){ - throw SpartaException("HEARTBEAT for pipeline collection must be an integer value and a multiple of 100 > 0, not \"") + throw SpartaException("HEARTBEAT for pipeline collection must be an integer value > 0, not \"") << pipeline_heartbeat_ << "\""; } - if(heartbeat != 0 && heartbeat % 100 != 0){ - throw SpartaException("HEARTBEAT for pipeline collection must be a multiple of 100 > 0, not \"") - << heartbeat << "\""; - } - // Pevent if(run_pevents_) { pevent_trigger_.reset(new sparta::trigger::PeventTrigger(sim->getRoot())); @@ -2002,22 +1928,6 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) } sim_config_.copyTreeNodeExtensionsFromArchAndConfigPTrees(); - - //The simdb feature is enabled by default unless it was explicitly - //disabled at the command line. The reports will go to the pwd and - //to the database at the same time, and at the end of simulation - //the SimDB reports are exported to the filesystem, and the two - //formatted report files are compared. - // - //This slows down simulation overall since we're capturing twice - //the amount of report data / metadata, but this is to ensure the - //new database is working for all scenarios while the database - //backend gets some bake time. But we'll leave it here for a little - //while so downstream SPARTA clients can revert to legacy reporting - //infrastructure if they really need to. - if (!feature_config_.isFeatureValueSet("simdb")) { - feature_config_.setFeatureValue("simdb", 0); - } sim->setFeatureConfig(&feature_config_); // Configure the simulator itself (not its content) @@ -2123,21 +2033,12 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) if(sim_config_.pipeline_collection_file_prefix != NoPipelineCollectionStr) { - const bool multiple_triggers = sim_config_.trigger_on_type == SimulationConfiguration::TriggerSource::TRIGGER_ON_ROI; - pipeline_collection_triggerable_.reset(new PipelineTrigger(sim_config_.pipeline_collection_file_prefix, - pipeline_enabled_node_names_, - heartbeat, - multiple_triggers, - sim->getRootClock(), - sim->getRoot())); - - // If pipeline collection is turned on begin writing an info file - // about the simulation. - info_out_.reset(new sparta::InformationWriter(sim_config_.pipeline_collection_file_prefix+"simulation.info")); - info_out_->write("Pipeline Collection files generated from simulator "); - info_out_->write(sim->getSimName()); - info_out_->write("\n\nSimulation started at: "); - info_out_->writeLine(sparta::TimeManager::getTimeManager().getLocalTime()); + pipeline_collection_triggerable_.reset(new PipelineTrigger( + sim_config_.pipeline_collection_file_prefix, + pipeline_enabled_node_names_, + std::atoi(pipeline_heartbeat_.c_str()), + sim->getRoot(), + sim->findSemanticCounter(Simulation::CSEM_INSTRUCTIONS))); } // Finalize the pevent controller now that the tree is built. @@ -2392,8 +2293,6 @@ void CommandLineSimulator::runSimulator_(Simulation* sim, uint64_t ticks) if(pipeline_collection_triggerable_->isTriggered()) { pipeline_collection_triggerable_->stop(); } - info_out_->write("Simulation aborted at: "); - info_out_->writeLine(sparta::TimeManager::getTimeManager().getLocalTime()); } // In interactive simulation, we would try and enter a "debug mode" and @@ -2407,12 +2306,6 @@ void CommandLineSimulator::runSimulator_(Simulation* sim, uint64_t ticks) if(pipeline_collection_triggerable_->isTriggered()) { pipeline_collection_triggerable_->stop(); } - - // Write the end time of the simulation. - info_out_->write("Simulation ended at: "); - info_out_->writeLine(sparta::TimeManager::getTimeManager().getLocalTime()); - sparta::InformationWriter& outputter = *(info_out_.get()); - outputter << "Heartbeat interval: " << pipeline_heartbeat_ << " ticks" << "\n"; } if(show_tree_){ @@ -2432,33 +2325,8 @@ void CommandLineSimulator::postProcess(Simulation* sim) } } -void CommandLineSimulator::postProcess_(Simulation* sim) +void CommandLineSimulator::postProcess_(Simulation*) { - auto simdb = GET_DB_FOR_COMPONENT(Stats, sim); - - if (simdb) { - auto feature_cfg = sim->getFeatureConfiguration(); - if (IsFeatureValueEnabled(feature_cfg, "simdb-verify")) { - std::string simdb_fname = simdb->getDatabaseFile(); - const std::string simdb_src_fname = simdb_fname; - simdb_fname = sfs::path(simdb_fname).filename().string(); - - sfs::path cwd = sfs::current_path(); - const std::string simdb_dest_dir = - cwd.string() + "/" + db::ReportVerifier::getVerifResultsDir(); - - const std::string simdb_dest_fname = simdb_dest_dir + "/" + simdb_fname; - std::error_code err; - sfs::copy_file(simdb_src_fname, simdb_dest_fname, err); - if (err) { - std::cout << " [simdb] Warning: The 'simdb-verify' post processing step " - << "encountered and trapped a std::filesystem error: \"" - << err.message() << "\"" << std::endl; - } - } - } - - sim->postProcessingLastCall(); } void CommandLineSimulator::printUsageHelp_() const @@ -2475,7 +2343,6 @@ void CommandLineSimulator::printOptionsHelp_(uint32_t level) const << pipeout_opts_.getOptionsLevelUpTo(level) << std::endl << debug_opts_.getOptionsLevelUpTo(level) << std::endl << report_opts_.getOptionsLevelUpTo(level) << std::endl - << simdb_opts_.getOptionsLevelUpTo(level) << std::endl << app_opts_.getOptionsLevelUpTo(level) << std::endl; if(0 == level){ diff --git a/sparta/src/DatabaseContextCounter.cpp b/sparta/src/DatabaseContextCounter.cpp deleted file mode 100644 index 351da2ff9b..0000000000 --- a/sparta/src/DatabaseContextCounter.cpp +++ /dev/null @@ -1,318 +0,0 @@ -// -*- C++ -*- - - -#include "sparta/report/db/DatabaseContextCounter.hpp" - -#include -#include -#include -#include - -#include "sparta/report/Report.hpp" -#include "rapidjson/document.h" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/statistics/StatisticInstance.hpp" - -namespace sparta { -namespace db { - -//! Pick off the last part of an SI's location, for example -//! if the location is "top.core0.rob.stats.ipc", the name -//! returned from this function would be "ipc". -std::string getNameFromSILocation(const StatisticInstance * si) -{ - const std::string name = si->getLocation(); - auto dot = static_cast(name.find_last_of(".")); - if (dot < static_cast(name.size()) - 1) { - return name.substr(static_cast(dot) + 1ul); - } - return name; -} - -//! Construct with a root SI, and a list of SI's which are -//! "unprintable" to the outside world (the report formatters). -//! The root SI corresponds to the original simulation's -//! ContextCounter (StatisticDef), and the unprintable -//! SI's correspond to the original simulation's Context -//! Counter sub-statistics (internal_counters_). -DatabaseContextCounter::DatabaseContextCounter( - const StatisticInstance * cc_node, - std::shared_ptr & unprintable_sis) -{ - cc_node_ = cc_node; - cc_desc_ = cc_node_->getDesc(false); - cc_name_ = getNameFromSILocation(cc_node); - unprintable_sis_ = unprintable_sis; -} - -//! Analogous to TreeNode::getName() -const std::string & DatabaseContextCounter::getName() const -{ - return cc_name_; -} - -//! Analogous to InstrumentationNode::groupedPrinting() -bool DatabaseContextCounter::groupedPrinting( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const -{ - extractCtxInfo_(sub_stats); - - const bool printing_taken_care_of = groupedPrinting_( - dont_print_these, grouped_json, doc, - ctx_info_, cc_desc_, cc_node_->getVisibility()); - - if (printing_taken_care_of) { - appendUnprintablesToSet_(dont_print_these); - } - return printing_taken_care_of; -} - -//! Analogous to InstrumentationNode::groupedPrintingReduced() -bool DatabaseContextCounter::groupedPrintingReduced( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const -{ - extractCtxInfo_(sub_stats); - - const bool printing_taken_care_of = groupedPrintingReduced_( - dont_print_these, grouped_json, doc, ctx_info_); - - if (printing_taken_care_of) { - appendUnprintablesToSet_(dont_print_these); - } - return printing_taken_care_of; -} - -//! Analogous to InstrumentationNode::groupedPrintingDetail() -bool DatabaseContextCounter::groupedPrintingDetail( - const std::vector & sub_stats, - std::set & dont_print_these, - void * grouped_json, - void * doc) const -{ - extractCtxInfo_(sub_stats); - - const bool printing_taken_care_of = groupedPrintingDetail_( - dont_print_these, grouped_json, doc, ctx_info_); - - if (printing_taken_care_of) { - appendUnprintablesToSet_(dont_print_these); - } - return printing_taken_care_of; -} - -//! Mimics ContextCounter::extractCtxInfo_ -void DatabaseContextCounter::extractCtxInfo_( - const std::vector & sub_stats) const -{ - if (!ctx_info_.empty()) { - sparta_assert(sub_stats.size() == ctx_info_.size()); - for (size_t idx = 0; idx < sub_stats.size(); ++idx) { - const auto stat_si = sub_stats[idx]; - auto & ctx_info = ctx_info_[idx]; - ctx_info.val_ = stat_si->getValue(); - } - return; - } - - ctx_info_.reserve(sub_stats.size()); - for (size_t idx = 0; idx < sub_stats.size(); ++idx) { - const auto stat_si = sub_stats[idx]; - - ContextCounterInfo info; - info.name_ = getNameFromSILocation(stat_si); - info.desc_ = stat_si->getDesc(false); - info.vis_ = stat_si->getVisibility(); - info.val_ = stat_si->getValue(); - info.ctx_addr_ = stat_si; - ctx_info_.push_back(info); - } -} - -//! Mimics __groupedPrinting() free function in sparta/ContextCounter.h -bool DatabaseContextCounter::groupedPrinting_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info, - const std::string & aggregate_desc, - const InstrumentationNode::visibility_t aggregate_vis) const -{ - rapidjson::Value & grouped_json = *reinterpret_cast(grouped_json_); - rapidjson::Document & doc = *reinterpret_cast(doc_); - - if (ctx_info.empty()) { - return false; - } - - rapidjson::Value ordered_keys; - ordered_keys.SetArray(); - grouped_json.SetObject(); - - double aggregate = 0; - for (const auto & counter : ctx_info) { - rapidjson::Value counter_info; - counter_info.SetObject(); - - counter_info.AddMember("desc", rapidjson::StringRef(counter.desc_.c_str()), - doc.GetAllocator()); - counter_info.AddMember("vis", rapidjson::Value(counter.vis_), - doc.GetAllocator()); - - const std::string & counter_name = counter.name_; - const double dbl_value = counter.val_; - - if (std::isnan(dbl_value)) { - counter_info.AddMember("val", rapidjson::Value("nan"), doc.GetAllocator()); - } else if(std::isinf(dbl_value)) { - counter_info.AddMember("val", rapidjson::Value("inf"), doc.GetAllocator()); - } else { - double dbl_formatted = 0; - std::stringstream ss; - ss << Report::formatNumber(dbl_value); - ss >> dbl_formatted; - - double int_part = 0; - const double remainder = std::modf(dbl_formatted, &int_part); - if (remainder == 0) { - //This double has no remainder, so pretty print it as an integer - counter_info.AddMember("val", rapidjson::Value(static_cast(dbl_formatted)), - doc.GetAllocator()); - } else { - //This double has some remainder, so pretty print it as-is - counter_info.AddMember("val", rapidjson::Value(dbl_formatted), doc.GetAllocator()); - } - } - - aggregate += counter.val_; - dont_print_these.insert(counter.ctx_addr_); - grouped_json.AddMember(rapidjson::StringRef(counter_name.c_str()), counter_info, - doc.GetAllocator()); - ordered_keys.PushBack(rapidjson::StringRef(counter_name.c_str()), - doc.GetAllocator()); - } - - rapidjson::Value aggregate_info; - aggregate_info.SetObject(); - aggregate_info.AddMember("desc", rapidjson::StringRef(aggregate_desc.c_str()), - doc.GetAllocator()); - aggregate_info.AddMember("vis", rapidjson::Value(aggregate_vis), - doc.GetAllocator()); - - const double dbl_aggregate = static_cast(aggregate); - if (std::isnan(dbl_aggregate)) { - aggregate_info.AddMember("val", rapidjson::Value("nan"), doc.GetAllocator()); - } else if (std::isinf(dbl_aggregate)) { - aggregate_info.AddMember("val", rapidjson::Value("inf"), doc.GetAllocator()); - } else { - double dbl_formatted = 0; - std::stringstream ss; - ss << Report::formatNumber(dbl_aggregate); - ss >> dbl_formatted; - - double int_part = 0; - const double remainder = std::modf(dbl_formatted, &int_part); - if (remainder == 0) { - //This double has no remainder, so pretty print it as an integer - aggregate_info.AddMember("val", rapidjson::Value(static_cast(dbl_formatted)), - doc.GetAllocator()); - } else { - //This double has some remainder, so pretty print it as-is - aggregate_info.AddMember("val", rapidjson::Value(dbl_formatted), doc.GetAllocator()); - } - } - - ordered_keys.PushBack("agg", doc.GetAllocator()); - grouped_json.AddMember("agg", aggregate_info, doc.GetAllocator()); - grouped_json.AddMember("ordered_keys", ordered_keys, doc.GetAllocator()); - - return true; -} - -//! Mimics __groupedPrintingReduced() free function in sparta/ContextCounter.h -bool DatabaseContextCounter::groupedPrintingReduced_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info) const -{ - rapidjson::Value & grouped_json = *reinterpret_cast(grouped_json_); - rapidjson::Document & doc = *reinterpret_cast(doc_); - - if (ctx_info.empty()) { - return false; - } - - grouped_json.SetObject(); - - double aggregate = 0; - for (const auto & counter : ctx_info) { - const std::string & counter_name = counter.name_; - const double dbl_value = counter.val_; - - if (std::isnan(dbl_value)) { - grouped_json.AddMember(rapidjson::StringRef(counter_name.c_str()), - rapidjson::Value("nan"), - doc.GetAllocator()); - } else if(std::isinf(dbl_value)) { - grouped_json.AddMember(rapidjson::StringRef(counter_name.c_str()), - rapidjson::Value("inf"), - doc.GetAllocator()); - } else { - double dbl_formatted = 0; - std::stringstream ss; - ss << Report::formatNumber(dbl_value); - ss >> dbl_formatted; - - double int_part = 0; - const double remainder = std::modf(dbl_formatted, &int_part); - if (remainder == 0) { - //This double has no remainder, so pretty print it as an integer - grouped_json.AddMember(rapidjson::StringRef(counter_name.c_str()), - rapidjson::Value(static_cast(dbl_formatted)), - doc.GetAllocator()); - } else { - //This double has some remainder, so pretty print it as-is - grouped_json.AddMember(rapidjson::StringRef(counter_name.c_str()), - rapidjson::Value(dbl_formatted), - doc.GetAllocator()); - } - } - - aggregate += counter.val_; - dont_print_these.insert(counter.ctx_addr_); - } - grouped_json.AddMember("agg", rapidjson::Value(aggregate), doc.GetAllocator()); - - return true; -} - -//! Mimics __groupedPrintingDetail() free function in sparta/ContextCounter.h -bool DatabaseContextCounter::groupedPrintingDetail_( - std::set & dont_print_these, - void * grouped_json_, void * doc_, - const std::vector & ctx_info) const -{ - (void) grouped_json_; - (void) doc_; - for (const auto & info : ctx_info) { - dont_print_these.insert(info.ctx_addr_); - } - return true; -} - -//! At the end of the various "grouped printing" methods, tack on -//! any "unprintable SI(s)" into the "dont_print_these" sets. -void DatabaseContextCounter::appendUnprintablesToSet_( - std::set & dont_print_these) const -{ - for (const auto unprintable : *unprintable_sis_) { - dont_print_these.insert(unprintable); - } -} - -} // namespace db -} // namespace sparta diff --git a/sparta/src/DatabaseSchema.cpp b/sparta/src/DatabaseSchema.cpp deleted file mode 100644 index 355a5206ca..0000000000 --- a/sparta/src/DatabaseSchema.cpp +++ /dev/null @@ -1,642 +0,0 @@ -// -*- C++ -*- - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "simdb/schema/Schema.hpp" -#include "simdb/schema/TableSummaries.hpp" -#include "sparta/report/db/Schema.hpp" -#include "sparta/statistics/StatisticDef.hpp" -#include "sparta/statistics/InstrumentationNode.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "sparta/trigger/ExpressionTrigger.hpp" -#include "sparta/report/SubContainer.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" -#include "sparta/simulation/RootTreeNode.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/kernel/SpartaHandler.hpp" - -//! Static initializers for SimDB-related classes -namespace sparta { - std::unordered_set DatabaseAccessor::all_simulation_accessors_; - bool DatabaseAccessor::static_simdb_accessor_invoked_ = false; -} - -namespace sparta { -namespace db { - -//! Build a SimDB schema object that can hold all report artifacts -//! and StatisticInstance values for SPARTA simulators. This schema -//! can be given to a simdb::ObjectManager to instantiate a physical -//! database connection. -void buildSimulationDatabaseSchema(simdb::Schema & schema) -{ - using dt = simdb::ColumnDataType; - - //Statistics databases are comprised of run/simulation - //metadata, and SI values stored as blobs. - - { - //Let's start by creating the metadata table. - schema.addTable("ReportHeader") - .addColumn("TimeseriesID", dt::fkey_t) - ->index() - ->setDefaultValue(0) - .addColumn("ReportName", dt::string_t) - .addColumn("StartTime", dt::uint64_t) - ->setDefaultValue(0) - .addColumn("EndTime", dt::uint64_t) - ->setDefaultValue(-1) - .addColumn("WarmupInsts", dt::uint64_t) - .addColumn("DestFile", dt::string_t) - .addColumn("SILocations", dt::string_t) - .addColumn("NumStatInsts", dt::int32_t) - .addColumn("SIRootNodeID", dt::fkey_t); - - //Records from two or more tables may need to be linked - //together in some way. We have this general-purpose table - //which holds nothing but ObjectManager UUID's to help - //accomplish this. The ObjectManager class has a getId() - //method, and records from different tables can be linked - //via this UUID. As an example: - // - // |=======================================================| - // | SimInfo | - // | ------------------------------------------------------| - // | Id | Name | SimulatorVersion | ObjMgrID | - // | ------------------------------------------------------| - // | 1 | "MySim" | "2.0" | 14 | - // | 2 | "DVM" | "2.3" | 16 | - // | ------------------------------------------------------| - // - // |=====================================| - // | ReportNodeHierarchy | - // | ------------------------------------| - // | Id | Name | ParentNodeID | - // | ------------------------------------| - // | 8 | "top" | 0 | - // | 9 | "core0" | 8 | - // | 10 | "rob" | 9 | - // | ------------------------------------| - // - //If we wanted to write an API which takes a report node - //database ID (say, 10) and create a report from the SI - //data stored in the database, we will need a quick way - //to get from any report node ID to a row in the SimInfo - //table. Perhaps we write our code like this: - // - // 1) Take the report node ID of 10 and keep running - // queries on the ReportNodeHierarchy table until - // we are at the top report node in this specific - // SI hierarchy. In this case, 10->9->8, and the - // root report node has an ID of 8. - // 2) Say we had another table which mapped root- - // level report node ID's to their corresponding - // ObjMgrID, which looks something like this: - // - // |===========================================| - // | RootReportObjMgrIDs | - // | ------------------------------------------| - // | Id | RootReportNodeID | ObjMgrID | - // | ------------------------------------------| - // | 1 | 8 | 16 | - // | ------------------------------------------| - // - //Now our API could be implemented like this: - // - // void makeReportFromDatabaseNode(int node_db_id) - // { - // //Say node_db_id = 10 - // auto root_db_id = getRootDbIdFrom(node_db_id); - // - // //Now root_db_id = 8 - // obj_mgr_id = getObjMgrIdForRootReportNode(root_db_id); - // - // //Now obj_mgr_id = 16. The final pseudo-code: - // auto sim_info = eval_sql( - // "SELECT * FROM SimInfo WHERE ObjMgrID = 16"); - // - // std::cout << sim_info.Name << " ran with simulator version " - // << sim_info.SimulatorVersion << std::endl; - // - // // " DVM ran with simulator version 2.3 " // - // } - schema.addTable("ObjectManagersInDatabase") - .addColumn("ObjMgrID", dt::fkey_t); - - //Table for SimulationInfo. These records are linked - //to root-level nodes in the ReportNodeHierarchy table - //via the ObjectManager UUID they both share. - schema.addTable("SimInfo") - .addColumn("Name", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Cmdline", dt::string_t) - ->setDefaultValue("unset") - .addColumn("WorkingDir", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Exe", dt::string_t) - ->setDefaultValue("unset") - .addColumn("SimulatorVersion", dt::string_t) - ->setDefaultValue("unset") - .addColumn("SpartaVersion", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Repro", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Start", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Other", dt::string_t) - ->setDefaultValue("unset") - .addColumn("ObjMgrID", dt::fkey_t) - ->setDefaultValue(0); - - //Table which describes report/subreport node hierarchy - schema.addTable("ReportNodeHierarchy") - .addColumn("ParentNodeID", dt::fkey_t) - ->index() - .addColumn("Name", dt::string_t) - .addColumn("IsLeafSI", dt::int32_t) - ->noSummary() - ->setDefaultValue(-1) - .addColumn("LeftmostSIIndex", dt::int32_t) - ->noSummary() - ->setDefaultValue(-1); - - //Table which describes sub-statistics node hierarchies. - //Used to serialize the basic layout of ContextCounter's - //in a given report. - schema.addTable("SubStatisticsNodeHierarchy") - .addColumn("ReportNodeID", dt::fkey_t) - ->index() - .addColumn("SINodeID", dt::fkey_t) - .addColumn("ParentSINodeID", dt::fkey_t); - - //This table is used to tell the SimDB->report code which - //sub-statistics are "unprintable". This supports Context - //Counter's which have special treatment in the JSON formatter - //code. We won't have any sparta::ContextCounter objects - //available when we regenerate reports just from records - //in a database, so this table helps mimic what the original - //simulator's legacy json* formatters would have done during - //simulation. - schema.addTable("UnprintableSubStatistics") - .addColumn("ReportNodeID", dt::fkey_t) - ->index(); - - //Table which stores all metadata that is common to all - //report/subreport nodes - schema.addTable("ReportNodeMetadata") - .addColumn("Author", dt::string_t) - ->setDefaultValue("unset") - .addColumn("InfoString", dt::string_t) - ->setDefaultValue("unset") - .addColumn("StartTick", dt::uint64_t) - ->setDefaultValue(0) - .addColumn("EndTick", dt::uint64_t) - ->setDefaultValue(-1) - .addColumn("ReportNodeID", dt::fkey_t) - ->setDefaultValue(-1) - ->index(); - - //Unlike the ReportNodeMetadata table, this table stores - //metadata that is common to all report nodes in a given - //report/subreport hierarchy. - schema.addTable("RootReportNodeMetadata") - .addColumn("ReportNodeID", dt::fkey_t) - ->indexAgainst("Name") - .addColumn("Name", dt::string_t) - .addColumn("Value", dt::string_t); - - //Table which stores all style metadata for - //a given report/subreport - schema.addTable("ReportStyle") - .addColumn("StyleName", dt::string_t) - .addColumn("StyleValue", dt::string_t) - .addColumn("ReportNodeID", dt::fkey_t) - ->index(); - - //SI metadata used in report generation (all formats) - constexpr int vs_default = static_cast( - sparta::StatisticDef::ValueSemantic::VS_INVALID); - constexpr int vis_default = static_cast( - sparta::InstrumentationNode::VIS_NORMAL); - constexpr int cls_default = static_cast( - sparta::InstrumentationNode::DEFAULT_CLASS); - - schema.addTable("SIMetadata") - .addColumn("Location", dt::string_t) - ->setDefaultValue("unset") - .addColumn("Desc", dt::string_t) - ->setDefaultValue("unset") - .addColumn("ExprString", dt::string_t) - ->setDefaultValue("unset") - .addColumn("ValueSemantic", dt::int32_t) - ->noSummary() - ->setDefaultValue(vs_default) - .addColumn("Visibility", dt::int32_t) - ->noSummary() - ->setDefaultValue(vis_default) - .addColumn("Class", dt::int32_t) - ->noSummary() - ->setDefaultValue(cls_default) - .addColumn("ReportNodeID", dt::fkey_t) - ->setDefaultValue(-1) - ->index(); - - //Make a 1-to-1 link from all root-level report nodes - //to the ID of the ObjectManager they came from - schema.addTable("RootReportObjMgrIDs") - .addColumn("RootReportNodeID", dt::fkey_t) - .addColumn("ObjMgrID", dt::fkey_t); - - //The above report metadata columns are for the most - //common pieces of metadata found in statistics reports. - //Let's use a catch-all string metadata table that any - //generic name/value pair can go into. We don't need - //a dedicated wrapper API around every possible metadata - //we can think of. - schema.addTable("StringMetadata") - .addColumn("ReportHeaderID", dt::fkey_t) - ->indexAgainst("MetadataName") - .addColumn("MetadataName", dt::string_t) - .addColumn("MetadataValue", dt::string_t); - - //Create an SI hierarchy table. Say there was an SI - //tree that looked like this (assume just 1 timeseries): - // - // top (id 1) - // ----------------------------- - // | | - // foo (id 2) bar (id 3) - // -------------------- -------------------- - // | | | | | - // leafA leafB leafC leafD leafE - // (id 4) (id 5) (id 6) (id 7) (id 8) - // - //This SINodeHierarchy table would look like this: - // - // Id ParentNodeID RelativeSIIndex NodeName - // ---- -------------- ----------------- ---------- - // 1 0 0 top - // 2 1 0 foo - // 3 1 3 bar - // 4 2 0 leafA - // 5 2 1 leafB - // 6 2 2 leafC - // 7 3 3 leafD - // 8 3 4 leafE - // - //The "RelativeSIIndex" column answers the question: - //"If I traveled from this SI node to the first leaf - //SI node I encountered in a depth-first traversal, - //what would be that leaf SI's index?" Where leaf SI - //indexes go from 0 to N-1, N being the number of SI's - //in this entire report/SI hierarchy (0 is leftmost - //SI index, N-1 is rightmost SI index). - schema.addTable("SINodeHierarchy") - .addColumn("TimeseriesID", dt::fkey_t) - .addColumn("ParentNodeID", dt::fkey_t) - ->indexAgainst("TimeseriesID") - .addColumn("NodeName", dt::string_t) - .addColumn("RelativeSIIndex", dt::int32_t) - ->noSummary(); - - //Clock hierarchies. Simulations will serialize the - //hieararchy from the root clock down through any - //children it has. - schema.addTable("ClockHierarchy") - .addColumn("ParentClockID", dt::fkey_t) - .addColumn("Name", dt::string_t) - .addColumn("Period", dt::uint32_t) - .addColumn("FreqMHz", dt::double_t) - .addColumn("RatioToParent", dt::double_t); - } - - { - //Create the Timeseries table - schema.addTable("Timeseries") - .addColumn("ReportHeaderID", dt::fkey_t); - - //Create the TimeseriesChunk table - schema.addTable("TimeseriesChunk") - .addColumn("TimeseriesID", dt::fkey_t) - ->indexAgainst({ - "StartPS", - "EndPS", - "StartCycle", - "EndCycle" - }) - .addColumn("StartPS", dt::uint64_t) - ->noSummary() - .addColumn("EndPS", dt::uint64_t) - ->noSummary() - .addColumn("StartCycle", dt::uint64_t) - ->noSummary() - .addColumn("EndCycle", dt::uint64_t) - ->noSummary(); - - //Create the StatInstValues table - schema.addTable("StatInstValues") - .addColumn("TimeseriesChunkID", dt::fkey_t) - ->index() - .addColumn("RawBytes", dt::blob_t) - .addColumn("NumPts", dt::int32_t) - ->noSummary() - .addColumn("WasCompressed", dt::int32_t) - ->noSummary() - .addColumn("MajorOrdering", dt::int32_t) - ->noSummary() - ->setDefaultValue( - static_cast(db::MajorOrdering::ROW_MAJOR)); - - //Hold SI value blobs for single-update, non- - //timeseries report formats in a separate table. - //Reports like json_reduced and html are stored - //in this table. - schema.addTable("SingleUpdateStatInstValues") - .addColumn("RootReportNodeID", dt::fkey_t) - ->index() - ->setDefaultValue(-1) - .addColumn("RawBytes", dt::blob_t) - .addColumn("NumPts", dt::int32_t) - ->noSummary() - .addColumn("WasCompressed", dt::int32_t) - ->noSummary(); - } - - { - //All of the tables in this section are here to support - //post-simulation report verification against legacy report - //files. They are only here for smoke testing, debugging - //report-related bugs, etc. and may be removed at any point - //in the future. - - //Maintain a mapping from report database ID to the original - //descriptor's dest_file and format strings. - schema.addTable("ReportVerificationMetadata") - .addColumn("RootReportNodeID", dt::fkey_t) - .addColumn("DestFile", dt::string_t) - ->index() - .addColumn("Format", dt::string_t); - - //High-level pass/fail results for each report in this - //database. Also includes a key to get each reports' - //accompanying SimInfo record. Useful information for - //debugging failed verifications can be found in the - //SimInfo table, such as repro commands. - schema.addTable("ReportVerificationResults") - .addColumn("DestFile", dt::string_t) - .addColumn("SimInfoID", dt::fkey_t) - ->indexAgainst("Passed") - .addColumn("Passed", dt::int32_t) - .addColumn("IsTimeseries", dt::int32_t) - ->noSummary(); - - //We use the SpartaTester utility class to find any differences - //between database-produced report files and their baselines. - //SpartaTester gives us a quick summary of file diff(s) just like - //you would see printed to stdout while running regression tests. - //We store those summaries in this table. - schema.addTable("ReportVerificationFailureSummaries") - .addColumn("ReportVerificationResultID", dt::fkey_t) - ->index() - .addColumn("FailureSummary", dt::string_t) - ->setDefaultValue("unset"); - - //When report verification is enabled, we may store deep copies - //of the diff'd files when failures occur so we don't have to - //rely on repro steps found in the SimInfo table. This is costly - //for regression tests that result in many failed verifications, - //but these tables are more for developer use / debugging than - //production simulators. - schema.addTable("ReportVerificationDeepCopyFiles") - .addColumn("DestFile", dt::string_t) - ->index() - .addColumn("Expected", dt::string_t) - .addColumn("Actual", dt::string_t); - } -} - -//! Configure the default TableSummaries object for SPARTA simulation -//! databases. This will provide default implementations for common -//! summary calculations like min/max/average, and possibly others. -void configureDatabaseTableSummaries(simdb::TableSummaries & config) -{ - config.define("min", - [](const std::vector & vals) -> double { - if (vals.empty()) { - return NAN; - } - return *std::min_element(vals.begin(), vals.end()); - }) - .define("max", - [](const std::vector & vals) -> double { - if (vals.empty()) { - return NAN; - } - return *std::max_element(vals.begin(), vals.end()); - }) - .define("avg", - [](const std::vector & vals) -> double { - if (vals.empty()) { - return NAN; - } - size_t n = 0; - double mean = 0; - for (auto val : vals) { - const double delta = (val - mean); - mean += delta / ++n; - } - return mean; - }); -} - -} // namespace db - -DatabaseAccessor::AccessTrigger::AccessTrigger( - DatabaseAccessor * db_accessor, - const std::string & db_namespace, - const std::string & start_expr, - const std::string & stop_expr, - RootTreeNode * rtn, - std::shared_ptr & sub_container) -{ - if (!start_expr.empty() || !stop_expr.empty()) { - sparta_assert(!db_namespace.empty()); - sparta_assert(rtn != nullptr); - } - - db_accessor_ = db_accessor; - db_namespace_ = db_namespace; - - if (!start_expr.empty()) { - const std::string trig_name = "GrantAccess_" + db_namespace; - auto handler = CREATE_SPARTA_HANDLER( - DatabaseAccessor::AccessTrigger, grantAccess_); - - start_ = std::make_shared( - trig_name, - handler, - start_expr, - rtn->getSearchScope(), - sub_container); - } - - if (!stop_expr.empty()) { - const std::string trig_name = "RevokeAccess_" + db_namespace; - auto handler = CREATE_SPARTA_HANDLER( - DatabaseAccessor::AccessTrigger, revokeAccess_); - - stop_ = std::make_shared( - trig_name, - handler, - stop_expr, - rtn->getSearchScope(), - sub_container); - } -} - -void DatabaseAccessor::setAccessOptsFromFile_( - const std::string & opt_file) -{ - if (!sub_container_) { - sub_container_ = std::make_shared(); - } - - std::ifstream fin(opt_file); - if (!fin) { - return; - } - - struct NamespaceAccess { - void beginNewNamespace(const std::string & line) { - components_.clear(); - start_trig_expr_.clear(); - stop_trig_expr_.clear(); - is_parsing_components_ = false; - - auto sep = line.find(":"); - ns_name_ = line.substr(0, sep); - boost::trim(ns_name_); - std::transform(ns_name_.begin(), ns_name_.end(), - ns_name_.begin(), ::tolower); - } - - const std::string & getNamespaceName() const { - return ns_name_; - } - - void beginComponents() { - is_parsing_components_ = true; - } - - bool isParsingComponents() const { - return is_parsing_components_; - } - - void addComponent(const std::string & component) { - components_.emplace_back(component); - } - - const std::vector & getComponents() const { - return components_; - } - - void setStartTrigExpr(const std::string & expr) { - start_trig_expr_ = expr; - } - - void setStopTrigExpr(const std::string & expr) { - stop_trig_expr_ = expr; - } - - const std::string & getStartTrigExpr() const { - return start_trig_expr_; - } - - const std::string & getStopTrigExpr() const { - return stop_trig_expr_; - } - - private: - std::string ns_name_; - std::vector components_; - std::string start_trig_expr_; - std::string stop_trig_expr_; - bool is_parsing_components_ = false; - }; - - auto is_namespace = [](const std::string & line) { - if (line.find("components:") != std::string::npos) { - return false; - } - if (line.find("start:") != std::string::npos) { - return false; - } - if (line.find("stop:") != std::string::npos) { - return false; - } - return line.find(":") != std::string::npos; - }; - - auto is_component = [](const std::string & line) { - std::string line_(line); - boost::trim(line_); - return line_ == "components:"; - }; - - auto is_start_trig = [](const std::string & line) { - std::string line_(line); - boost::trim(line_); - return line_.find("start:") == 0; - }; - - auto is_stop_trig = [](const std::string & line) { - std::string line_(line); - boost::trim(line_); - return line_.find("stop:") == 0; - }; - - NamespaceAccess accessor; - std::vector finalized_accessors; - for (std::string line; std::getline(fin, line); ) { - if (is_namespace(line)) { - finalized_accessors.emplace_back(accessor); - accessor.beginNewNamespace(line); - } else if (is_component(line)) { - accessor.beginComponents(); - } else if (is_start_trig(line)) { - accessor.setStartTrigExpr(line.substr(line.find(":") + 1)); - } else if (is_stop_trig(line)) { - accessor.setStopTrigExpr(line.substr(line.find(":") + 1)); - } else if (accessor.isParsingComponents()) { - accessor.addComponent(line); - } - } - finalized_accessors.emplace_back(accessor); - - for (const auto & access : finalized_accessors) { - if (!access.getNamespaceName().empty()) { - access_triggers_.emplace_back(new AccessTrigger( - this, - access.getNamespaceName(), - access.getStartTrigExpr(), - access.getStopTrigExpr(), - root_, - sub_container_)); - } - - for (const auto & comp : access.getComponents()) { - enableComponentAtLocation_(access.getNamespaceName(), comp); - } - } -} - -} // namespace sparta diff --git a/sparta/src/ExpressionGrammar.cpp b/sparta/src/ExpressionGrammar.cpp index 4c50b52158..4013327408 100644 --- a/sparta/src/ExpressionGrammar.cpp +++ b/sparta/src/ExpressionGrammar.cpp @@ -16,7 +16,6 @@ #include "sparta/utils/SpartaException.hpp" #include "sparta/simulation/TreeNodePrivateAttorney.hpp" #include "sparta/trigger/ContextCounterTrigger.hpp" -#include "sparta/statistics/StatInstCalculator.hpp" #include namespace phoenix = boost::phoenix; diff --git a/sparta/src/ExpressionTrigger.cpp b/sparta/src/ExpressionTrigger.cpp index 7082b41f40..875df41236 100644 --- a/sparta/src/ExpressionTrigger.cpp +++ b/sparta/src/ExpressionTrigger.cpp @@ -10,7 +10,6 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/simulation/GlobalTreeNode.hpp" #include "sparta/simulation/RootTreeNode.hpp" -#include "sparta/statistics/StatInstCalculator.hpp" namespace sparta { namespace trigger { @@ -448,56 +447,36 @@ bool ExpressionCounterTrigger::tryParseContextCounterTrigger_( //done using the REGISTER_CONTEXT_COUNTER_AGGREGATE_FCN //macro. sparta_assert(trigger_context != nullptr); - std::shared_ptr calc = - ContextCounterTrigger::findRegisteredContextCounterAggregateFcn( - trigger_context, counter_path); - - //If found, invoke the user's C++ code to get the - //current aggregated value of their ContextCounter. - //We will use that as the trigger point's offset. - if (calc) { - const double absolute_offset_dbl = calc->getCurrentValue(); - const uint64_t truncated_absolute_offset = (uint64_t)absolute_offset_dbl; - if (absolute_offset_dbl != truncated_absolute_offset) { - std::cout << " * * * Warning: " - "A ContextCounter trigger point is being truncated from " - << absolute_offset_dbl << " to " << truncated_absolute_offset - << std::endl; - } - absolute_offset = (uint64_t)absolute_offset_dbl; - } - else { - //If there is no user-supplied aggregation function, - //we can still try to get the default aggregate value - //just by adding up the internal counters' current - //values. - utils::ValidValue raw_sum; - for (const auto & sub_stat : stat_def->getSubStatistics()) { - //Get the sub-statistic as a CounterBase. If any of - //the sub-statistics are *not* CounterBase objects, - //then this is not a ContextCounter. - auto tn = dynamic_cast(sub_stat.getNode()); - if (tn == nullptr) { - raw_sum.clearValid(); - break; - } - - //We got another CounterBase* sub-statistic. Add their - //value to the raw sum. - if (!raw_sum.isValid()) { - raw_sum = tn->get(); - } else { - raw_sum += tn->get(); - } + //If there is no user-supplied aggregation function, + //we can still try to get the default aggregate value + //just by adding up the internal counters' current + //values. + utils::ValidValue raw_sum; + for (const auto & sub_stat : stat_def->getSubStatistics()) { + //Get the sub-statistic as a CounterBase. If any of + //the sub-statistics are *not* CounterBase objects, + //then this is not a ContextCounter. + auto tn = dynamic_cast(sub_stat.getNode()); + if (tn == nullptr) { + raw_sum.clearValid(); + break; } - //If all sub-statistics were CounterBase*, we would have a valid - //raw sum value at this point. - if (raw_sum.isValid()) { - absolute_offset = raw_sum.getValue(); + //We got another CounterBase* sub-statistic. Add their + //value to the raw sum. + if (!raw_sum.isValid()) { + raw_sum = tn->get(); + } else { + raw_sum += tn->get(); } } + + //If all sub-statistics were CounterBase*, we would have a valid + //raw sum value at this point. + if (raw_sum.isValid()) { + absolute_offset = raw_sum.getValue(); + } } //If we were asked to apply an offset to the trigger point, but diff --git a/sparta/src/JsonFormatter.cpp b/sparta/src/JsonFormatter.cpp index 95807f77ce..faf56c4773 100644 --- a/sparta/src/JsonFormatter.cpp +++ b/sparta/src/JsonFormatter.cpp @@ -18,7 +18,6 @@ #include "sparta/report/format/JSON.hpp" #include "sparta/report/format/JSON_reduced.hpp" -#include "sparta/report/db/DatabaseContextCounter.hpp" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/prettywriter.h" @@ -110,64 +109,10 @@ void extractStatisticsJsonFull(rapidjson::Document & doc, std::set dont_print_these; std::set db_dont_print_these; - const Report::SubStaticticInstances & sub_stats = r->getSubStatistics(); - const Report::DBSubStatisticInstances & db_sub_stats = r->getDBSubStatistics(); - for (const statistics::stat_pair_t & si : r->getStatistics()) { const std::string stat_name = !si.first.empty() ? si.first : si.second->getLocation(); if (!stat_name.empty()) { - const StatisticInstance * stat_inst = si.second.get(); - const StatisticDef * def = stat_inst->getStatisticDef(); - const CounterBase * ctr = stat_inst->getCounter(); - const ParameterBase * prm = stat_inst->getParameter(); - - auto sub_stat_iter = sub_stats.find(def); - const bool valid_stat_def = (def != nullptr); - const bool has_valid_sub_stats = - (valid_stat_def && sub_stat_iter != sub_stats.end()); - - auto db_sub_stat_iter = db_sub_stats.find(stat_inst); - const bool has_valid_db_sub_stats = (db_sub_stat_iter != db_sub_stats.end()); - - rapidjson::Value grouped_json; - if (has_valid_sub_stats && def->groupedPrinting(sub_stat_iter->second, - dont_print_these, - &grouped_json, &doc)) { - const std::string & name = def->getName(); - contents.AddMember(rapidjson::StringRef(name.c_str()), - grouped_json, - doc.GetAllocator()); - continue; - } - if (dont_print_these.count(ctr) > 0 || dont_print_these.count(prm) > 0) { - continue; - } - dont_print_these.clear(); - - if (has_valid_db_sub_stats) { - const std::shared_ptr & db_ctx_ctr = - db_sub_stat_iter->second.first; - - const std::vector & db_sub_sis = - db_sub_stat_iter->second.second; - - if (db_ctx_ctr->groupedPrinting(db_sub_sis, - db_dont_print_these, - &grouped_json, &doc)) - { - const std::string & name = db_ctx_ctr->getName(); - contents.AddMember(rapidjson::StringRef(name.c_str()), - grouped_json, - doc.GetAllocator()); - continue; - } - } - if (db_dont_print_these.count(stat_inst) > 0) { - continue; - } - db_dont_print_these.clear(); - rapidjson::Value stats_json; stats_json.SetObject(); @@ -389,9 +334,6 @@ void extractStatisticsJsonReduced(rapidjson::Document & doc, // XXX: This might cause name collisions if max_report_depth != -1 std::string local_name = flattenReportName(r->getName()); - const Report::SubStaticticInstances & sub_stats = r->getSubStatistics(); - const Report::DBSubStatisticInstances & db_sub_stats = r->getDBSubStatistics(); - // Keep track of the order in which stats and subunits are written std::set dont_print_these; std::set db_dont_print_these; @@ -399,57 +341,6 @@ void extractStatisticsJsonReduced(rapidjson::Document & doc, for (const statistics::stat_pair_t & si : r->getStatistics()) { const std::string stat_name = !si.first.empty() ? si.first : si.second->getLocation(); if (!stat_name.empty()) { - const StatisticInstance * stat_inst = si.second.get(); - const StatisticDef * def = stat_inst->getStatisticDef(); - const CounterBase * ctr = stat_inst->getCounter(); - const ParameterBase * prm = stat_inst->getParameter(); - - auto sub_stat_iter = sub_stats.find(def); - const bool valid_stat_def = (def != nullptr); - const bool has_valid_sub_stats = - (valid_stat_def && sub_stat_iter != sub_stats.end()); - - auto db_sub_stat_iter = db_sub_stats.find(stat_inst); - const bool has_valid_db_sub_stats = (db_sub_stat_iter != db_sub_stats.end()); - - rapidjson::Value grouped_json; - if (has_valid_sub_stats && def->groupedPrintingReduced(sub_stat_iter->second, - dont_print_these, - &grouped_json, &doc)) { - const std::string & name = def->getName(); - contents.AddMember(rapidjson::StringRef(name.c_str()), - grouped_json, - doc.GetAllocator()); - continue; - } - if (dont_print_these.count(ctr) > 0 || dont_print_these.count(prm) > 0) { - continue; - } - dont_print_these.clear(); - - if (has_valid_db_sub_stats) { - const std::shared_ptr & db_ctx_ctr = - db_sub_stat_iter->second.first; - - const std::vector & db_sub_sis = - db_sub_stat_iter->second.second; - - if (db_ctx_ctr->groupedPrintingReduced(db_sub_sis, - db_dont_print_these, - &grouped_json, &doc)) - { - const std::string & name = db_ctx_ctr->getName(); - contents.AddMember(rapidjson::StringRef(name.c_str()), - grouped_json, - doc.GetAllocator()); - continue; - } - } - if (db_dont_print_these.count(stat_inst) > 0) { - continue; - } - db_dont_print_these.clear(); - const double val = si.second->getValue(); if (omit_zero_values && val == 0) { continue; diff --git a/sparta/src/Report.cpp b/sparta/src/Report.cpp index 5ade518c91..252ce41be4 100644 --- a/sparta/src/Report.cpp +++ b/sparta/src/Report.cpp @@ -37,10 +37,6 @@ #include "sparta/utils/SmartLexicalCast.hpp" #include "sparta/statistics/HistogramFunctionManager.hpp" #include "sparta/utils/MetaStructs.hpp" -#include "simdb/ObjectManager.hpp" -#include "sparta/report/db/StatInstRowIterator.hpp" -#include "sparta/report/db/StatInstValueLookup.hpp" -#include "sparta/report/db/DatabaseContextCounter.hpp" namespace sparta { @@ -1789,945 +1785,4 @@ bool Report::hasHeader() const return (header_ != nullptr); } -/*! - * \brief Reconstruct a StatisticInstance from a record found - * in the given database. - */ -std::unique_ptr createSIFromSimDB( - const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr) -{ - std::vector> metadata; - std::string location, description, expr_string; - uint32_t value_semantic, visibility; - int cls; - - obj_mgr.safeTransaction([&]() { - //Start out by using the incoming node ID to query - //the SIMetadata table for basic SI properties like - //location, description, visibility, etc. - std::unique_ptr si_meta_query( - new simdb::ObjectQuery(obj_mgr, "SIMetadata")); - - si_meta_query->addConstraints( - "ReportNodeID", simdb::constraints::equal, report_hier_node_id); - - si_meta_query->writeResultIterationsTo( - "Location", &location, - "Desc", &description, - "ExprString", &expr_string, - "ValueSemantic", &value_semantic, - "Visibility", &visibility, - "Class", &cls); - - auto result_iter = si_meta_query->executeQuery(); - if (!result_iter->getNext()) { - throw SpartaException("Unable to locate SIMetadata record ") - << "with ReportNodeID " << report_hier_node_id; - } - - //Empty string metadata values are stored with default - //value "unset" in the database. Inherit these values - //now, or throw if they were never set. - if (location.empty() || location == "unset") { - throw SpartaException("SIMetadata record with ReportNodeID ") - << report_hier_node_id << " did not have its Location " - << "column set"; - } - - if (description.empty() || description == "unset") { - throw SpartaException("SIMetadata record with ReportNodeID ") - << report_hier_node_id << " did not have its Desc column set"; - } - - if (expr_string.empty() || expr_string == "unset") { - throw SpartaException("SIMetadata record with ReportNodeID ") - << report_hier_node_id << " did not have its ExprString " - << "column set"; - } - - //Tack on any additional metadata we can find - si_meta_query.reset(new simdb::ObjectQuery( - obj_mgr, "RootReportNodeMetadata")); - - si_meta_query->addConstraints( - "ReportNodeID", simdb::constraints::equal, report_hier_node_id); - - std::string meta_name, meta_value; - si_meta_query->writeResultIterationsTo( - "Name", &meta_name, "Value", &meta_value); - - result_iter = si_meta_query->executeQuery(); - while (result_iter->getNext()) { - if (!meta_name.empty() && !meta_value.empty()) { - metadata.emplace_back(meta_name, meta_value); - } - } - }); - - std::unique_ptr si( - new StatisticInstance(location, description, expr_string, - (StatisticDef::ValueSemantic)value_semantic, - (InstrumentationNode::visibility_t)visibility, - (InstrumentationNode::class_t)cls, - metadata)); - return si; -} - -/*! - * \brief Starting with the given report node database ID, - * find its root report node ID in the provided database. - */ -simdb::DatabaseID getRootReportNodeIDFrom( - const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr) -{ - simdb::ObjectQuery query(obj_mgr, "ReportNodeHierarchy"); - - int parent_report_hier_node_id = 0; - - query.addConstraints( - "Id", simdb::constraints::equal, report_hier_node_id); - - query.writeResultIterationsTo( - "ParentNodeID", &parent_report_hier_node_id); - - auto result_iter = query.executeQuery(); - if (result_iter == nullptr) { - return report_hier_node_id; - } - - sparta_assert(result_iter->getNext()); - if (parent_report_hier_node_id == 0) { - return report_hier_node_id; - } - - return getRootReportNodeIDFrom( - parent_report_hier_node_id, obj_mgr); -} - -/*! - * \brief Go through all reports/subreports and their SI's, and - * ask all of these objects to verify their StatInstValueLookup - * objects are ready to being reading SimDB data values. - */ -void recursValidateSIDirectLookups(const Report & report) -{ - for (const auto & si : report.getStatistics()) { - if (!si.second->isSIValueDirectLookupValid()) { - throw SpartaException("Encountered an SI whose StatInstValueLookup ") - << "could not be read. This indicates there is a bug in the SimDB " - << "tables/records related to report hierarchies."; - } - } - - for (const auto & sr : report.getSubreports()) { - recursValidateSIDirectLookups(sr); - } -} - -/*! - * \brief SimDB->report generation workflows invoke this constructor, - * which recreates a Report object (and its subreports and all of - * their SI's) from a database connection and a report node database - * ID. - */ -Report::Report(const simdb::DatabaseID report_hier_node_id, - const simdb::ObjectManager & obj_mgr, - const Scheduler * scheduler) : - Report("nameless", scheduler) -{ - obj_mgr.safeTransaction([&]() { - //To recreate a report from a database entry, start with - //the ReportNodeHierarchy table, looking for the one whose - //primary key equals the given database ID. - simdb::ObjectQuery hier_query(obj_mgr, "ReportNodeHierarchy"); - - hier_query.addConstraints("Id", simdb::constraints::equal, - report_hier_node_id); - - //Request the report's name when found. - hier_query.writeResultIterationsTo("Name", &name_); - - auto result_iter = hier_query.executeQuery(); - if (!result_iter->getNext()) { - throw SpartaException("Unable to locate ReportNodeHierarchy ") - << "record with ReportNodeID " << report_hier_node_id; - } - - //The hierarchy query we just ran was filtered against - //a unique database ID, so we should never have more than - //one match. - if (result_iter->getNext()) { - throw SpartaException("Found more than one record in the ") - << "ReportNodeHierarchy table with ReportNodeID " - << report_hier_node_id << ". This is a bug."; - } - - //The next step is to get this report's metadata. Look - //for the record in the ReportNodeMetadata table whose - //ReportNodeID field (foreign key) matches the given - //database ID (ReportNodeHierarchy primary key). - simdb::ObjectQuery meta_query(obj_mgr, "ReportNodeMetadata"); - meta_query.addConstraints("ReportNodeID", - simdb::constraints::equal, - report_hier_node_id); - - //Request common metadata properties such as Start/EndTick, etc. - meta_query.writeResultIterationsTo("Author", &author_, - "InfoString", &info_string_, - "StartTick", &start_tick_, - "EndTick", &end_tick_); - - //The only way this query would fail is if the database ID - //that was passed into this constructor was not a valid ID - //in the ReportNodeMetadata table. - result_iter = meta_query.executeQuery(); - if (!result_iter->getNext()) { - throw SpartaException("Unable to locate ReportNodeMetadata ") - << "record with ReportNodeID " << report_hier_node_id; - } - - //The metadata query we just ran was also filtered against - //a unique database ID. Verify there is only one such match - //in this database. - if (result_iter->getNext()) { - throw SpartaException("Found more than one record in the ") - << "ReportNodeMetadata table with ReportNodeID " - << report_hier_node_id << ". This is a bug."; - } - - //Empty metadata fields have the default value "unset". - //Clear our member variables where this is the case. - auto clear_unset_string = [](std::string & str) { - if (str == "unset") { - str.clear(); - } - }; - clear_unset_string(name_); - clear_unset_string(author_); - clear_unset_string(info_string_); - - //Look for any records in the ReportStyle table whose - //ReportNodeID field (foreign key) matches the given - //database ID (ReportNodeHierarchy primary key). - simdb::ObjectQuery style_query(obj_mgr, "ReportStyle"); - style_query.addConstraints("ReportNodeID", - simdb::constraints::equal, - report_hier_node_id); - - //Styles are given in simple name-value pairs. - std::string style_name, style_value; - style_query.writeResultIterationsTo("StyleName", &style_name, - "StyleValue", &style_value); - - result_iter = style_query.executeQuery(); - - //Unlike ReportNodeHierarchy, ReportStyle is not required. - //The original report/subreport that was written to the - //database may not have had any style info. - if (result_iter) { - while (result_iter->getNext()) { - //But if there was such a style record, its name/value - //columns should have been set. - if (style_name.empty() || style_value.empty()) { - throw SpartaException("StyleName and/or StyleValue columns ") - << "were not set in the ReportStyle table for the record " - << "with ReportNodeID " << report_hier_node_id; - } - - style_[style_name] = style_value; - } - } - - //Get the next generation of nodes under this report. These - //could be subreport nodes or SI nodes, which we'll figure - //out shortly. - simdb::ObjectQuery nextgen_query(obj_mgr, "ReportNodeHierarchy"); - nextgen_query.addConstraints("ParentNodeID", - simdb::constraints::equal, - report_hier_node_id); - - //Request these next generation's Id field (primary key), - //and their Name, IsLeafSI, and LeftmostSIIndex field - //values. - simdb::DatabaseID nextgen_node_id = 0; - std::string nextgen_node_name; - int is_leaf_si, leaf_si_index; - - nextgen_query.writeResultIterationsTo( - "Id", &nextgen_node_id, - "Name", &nextgen_node_name, - "IsLeafSI", &is_leaf_si, - "LeftmostSIIndex", &leaf_si_index); - - result_iter = nextgen_query.executeQuery(); - - //Leaf SI info includes the SI name, its database ID (primary - //key in ReportNodeHierarchy table), and its leaf SI index. - // - //The SI index will let us know the offset into the contiguous - //SI double values vector, which we'll use to set up a link - //between each leaf SI and its associated StatInstValueLookup. - struct LeafSIInfo { - std::string name; - utils::ValidValue db_id; - utils::ValidValue linear_idx; - }; - - std::vector nextgen_si_leaves; - std::vector nextgen_subreport_db_ids; - while (result_iter->getNext()) { - sparta_assert(nextgen_node_id > 0); - sparta_assert(is_leaf_si == 0 || is_leaf_si == 1); - - if (is_leaf_si) { - sparta_assert(!nextgen_node_name.empty()); - sparta_assert(nextgen_node_name != "unset"); - - LeafSIInfo info; - info.name = nextgen_node_name; - info.db_id = nextgen_node_id; - info.linear_idx = leaf_si_index; - - nextgen_si_leaves.emplace_back(info); - } else { - nextgen_subreport_db_ids.emplace_back(nextgen_node_id); - } - } - - report_node_id_ = report_hier_node_id; - - //Nothing left to do if there are no next-generation nodes. - if (nextgen_si_leaves.empty() && nextgen_subreport_db_ids.empty()) { - return; - } - - //If we have any SI leaves beneath this report node, we need - //to create a "StatInstRowIterator" right now. More specifically, - //we need to create a "placeholder" for one of these iterator - //objects. - // - //It works like this: - // - // 1) Recreate a sparta::Report from a database ID - // - This recursively creates subreports from other - // database ID's that it finds along the way - // - // 2) Out of all of the report/subreport nodes in this tree, - // we could use any of them to create a StatInstRowIterator - // which they could all share. Say we have the following tree: - // - // top - // (id 1) - // ------------------------- - // core0 core1 - // (id 2) (id 3) - // ---------------- | - // | | baz - // foo bar (id 6) - // (id 4) (id 5) - // - //A StatInstRowIterator is a wrapper around an ObjectQuery. - //It runs a query for all SI blobs using the report *root ID* - //in its query constraints. All nodes in this example tree - //share the same report root ID of 1 ("top"). - // - //The sparta::StatInstRowIterator constructor runs a query - //against the SimDB, so it is not something we want to do - //more times than we have to. However, the object we make - //here is a placeholders::StatInstRowIterator (note the - //namespace is different), which gives us a place to hold - //all of our individual report node ID's until this entire - //sparta::Report's recursive constructor calls are done. Then, - //we will ask one of these placeholders to turn itself into - //a "real" sparta::StatInstRowIterator object only once, and - //share that object with all nodes in this report structure. - // - //This extra step of having placeholder objects like this - //was preferred over passing in our 'this' pointer into - //a sparta::Report constructor, denoting the parent: - // - // //Somewhere in this constructor: - // subreps_.emplace_back(..., this); - // - // //Which calls a constructor that looks like this: - // Report(..., Report * parent) : - // si_row_iterator_(parent->si_row_iterator_), - // parent_(parent) - // { - // //run other DB queries to get our metadata, - // //and recursively create more subreports if - // //needed - // } - // - //It seems preferrable to reuse the "addSubreport()" method - //out of the box, to help guarantee the recreated hierarchy - //is identical to the original one in the simulator. As - //opposed to creating another private constructor to do the - //same thing. Plus, this code just seems odd, and potentially - //has a subtle bug in it somewhere: - // - // subreps_.emplace_back(..., this); - // ******** ********* - // this->subreps_ Report::ctor(..., parent) : - // this->parent_(parent) - // {} - // - if (!nextgen_si_leaves.empty()) { - si_row_iterator_.reset(new placeholders::StatInstRowIterator( - report_hier_node_id, &obj_mgr)); - } - - //Add StatisticInstance's to this report, if any. This - //calls an SI constructor which takes a database ID which - //it knows how to use to recreate the same SI from the - //original simulation. - for (const auto & leaf_si_db_info : nextgen_si_leaves) { - auto simdb_si = createSIFromSimDB(leaf_si_db_info.db_id, obj_mgr); - sparta_assert(simdb_si != nullptr); - stats_.emplace_back(leaf_si_db_info.name, simdb_si.release()); - si_node_ids_.emplace_back(leaf_si_db_info.db_id); - - //Similar to the StatInstRowIterator for report nodes, we - //give each SI a placeholders::StatInstValueLookup object. - //The only info this placeholder needs up front is the leaf - //SI index. We'll resolve these placeholders at the same - //time we resolve the StatInstRowIterator placeholder(s). - //This is done shortly after this constructor call in the - //SimDB->report generation code path. - std::shared_ptr direct_lookup( - new placeholders::StatInstValueLookup( - leaf_si_db_info.linear_idx.getValue())); - - stats_.back().second->setSIValueDirectLookupPlaceholder( - direct_lookup); - } - - //Add subreports to this report, if any. Calls this same - //Report constructor with the appropriate database ID. - for (const auto subreport_db_id : nextgen_subreport_db_ids) { - Report subrep(subreport_db_id, obj_mgr, scheduler); - addSubreport(subrep); - } - }); -} - -/*! - * \brief During SimDB->report creation workflows, this method - * is called which sets up database/table cursors to get ready - * to iterate over SI records/blobs. - */ -bool Report::prepareForSIDatabaseIteration_( - const simdb::ObjectManager & obj_mgr) -{ - //First clear out some report style metadata that was only - //stashed in the database in order to make the appropriate - //BaseFormatter API calls. - style_.erase("OmitZero"); - style_.erase("PrettyPrintDisabled"); - - //All report/subreport nodes in this tree can share the - //same StatInstRowIterator object. Go to the root node, - //fetch the first row iterator object we find (closest - //to the root, i.e. short-circuit the tree walk once - //we find one of these iterator objects). - auto root = getRoot(); - - //Walk the report/SI tree and gather all database ID's - //for the reports, subreports, and all their SI's. This - //will be used for ContextCounter hierarchy recreation. - std::unordered_map report_nodes_by_id; - std::unordered_map si_nodes_by_id; - root->recursGetReportAndSINodeDatabaseIDs_( - report_nodes_by_id, si_nodes_by_id); - - simdb::DatabaseID unprintable_si_id; - simdb::ObjectQuery unprintable_sis_query(obj_mgr, "UnprintableSubStatistics"); - unprintable_sis_query.writeResultIterationsTo("ReportNodeID", &unprintable_si_id); - - auto unprintable_sis = std::make_shared(); - auto result_iter = unprintable_sis_query.executeQuery(); - while (result_iter->getNext()) { - auto unprintable_iter = si_nodes_by_id.find(unprintable_si_id); - if (unprintable_iter != si_nodes_by_id.end()) { - unprintable_sis->insert(unprintable_iter->second); - } - } - - //To get the same report formatting for ContextCounter's, we - //have a "DatabaseContextCounter" class which mimics the - //sparta::ContextCounter class. But unlike the "real" Context - //Counter class, DatabaseContextCounter is only responsible - //for post-simulation formatting of ContextCounter data in - //report files. - // - //To build up DatabaseContextCounter objects, we need to - //look for some sub-statistics information in the database - //that live in 'this' report. - std::vector report_node_ids; - report_node_ids.reserve(report_nodes_by_id.size()); - for (const auto & node : report_nodes_by_id) { - report_node_ids.emplace_back(node.first); - } - - simdb::ObjectQuery substats_query(obj_mgr, "SubStatisticsNodeHierarchy"); - - substats_query.addConstraints( - "ReportNodeID", simdb::constraints::in_set, report_node_ids); - - simdb::DatabaseID report_node_id, si_node_id, parent_si_node_id; - substats_query.writeResultIterationsTo( - "ReportNodeID", &report_node_id, - "SINodeID", &si_node_id, - "ParentSINodeID", &parent_si_node_id); - - //The DBSubStatisticInstances data structure used below is defined - //as a mapping from SI's to their sub-SI's. - // - //To illustrate what this data structure represents, say that we had - //the following hierarchy in the original SPARTA simulation: - // Report - // SI (wraps a ContextCounter) - // internal_counters_[0] (wraps a CounterBase) - // internal_counters_[1] (wraps a CounterBase) - // - //The equivalent hierarchy when recreating the same report *after* - //simulation looks like this: - // - // Report - // SI (is the root node of a DatabaseContextCounter) - // SI (is the first sub-statistic under it) - // SI (is the second sub-statistic under it) - // - result_iter = substats_query.executeQuery(); - while (result_iter->getNext()) { - DBSubStatisticInstances & substats = - report_nodes_by_id[report_node_id]->db_sub_statistics_; - - const StatisticInstance * cc_root = si_nodes_by_id[parent_si_node_id]; - auto substats_iter = substats.find(cc_root); - if (substats_iter == substats.end()) { - std::shared_ptr db_ctx_counter = - std::make_shared(cc_root, unprintable_sis); - - substats[cc_root].first = db_ctx_counter; - substats_iter = substats.find(cc_root); - } - - substats_iter->second.second.emplace_back(si_nodes_by_id[si_node_id]); - } - - //Now let's setup every Report's row iterator objects. - //Start by getting a shared row iterator from the first - //one we find starting at the root Report node. - auto row_iterator_placeholder = - root->recursFindTopmostSIRowIteratorPlaceholder_(); - - //Null out the report nodes' row iterator objects. - std::shared_ptr null_iter; - root->recursSetSIRowIterator_(null_iter); - - //Now, all of the placeholder row iterator objects are - //destroyed *except* for the one we have right here, - //taken from the topmost node we could find which had - //one. Let's tell this placeholder to clone itself into - //a "realized" StatInstRowIterator. - std::shared_ptr realized_iterator( - row_iterator_placeholder->realizePlaceholder()); - - //Walk back down through the report tree, and give - //all report nodes shared ownership of this row - //iterator. - root->recursSetSIRowIterator_(realized_iterator); - - //Return true on success. - return si_row_iterator_ != nullptr; -} - -/*! - * \brief Starting at 'this' report node, recursively get - * all mappings from Report/SI database node ID to the - * Report or SI that lives at each node. - */ -void Report::recursGetReportAndSINodeDatabaseIDs_( - std::unordered_map & report_nodes_by_id, - std::unordered_map & si_nodes_by_id) -{ - sparta_assert(report_node_id_ > 0); - report_nodes_by_id[report_node_id_] = this; - - sparta_assert(stats_.size() == si_node_ids_.size()); - for (size_t idx = 0; idx < stats_.size(); ++idx) { - si_nodes_by_id[si_node_ids_[idx]] = stats_[idx].second.get(); - } - - for (auto & sr : subreps_) { - sr.recursGetReportAndSINodeDatabaseIDs_(report_nodes_by_id, si_nodes_by_id); - } -} - -/*! - * \brief Starting at 'this' report node, find the first - * StatInstRowIterator member variable we encounter while - * traversing in a depth-first fashion. - */ -std::shared_ptr - Report::recursFindTopmostSIRowIteratorPlaceholder_() -{ - if (si_row_iterator_ != nullptr) { - return si_row_iterator_; - } - for (auto & sr : subreps_) { - auto placeholder = sr.recursFindTopmostSIRowIteratorPlaceholder_(); - if (placeholder) { - return placeholder; - } - } - return nullptr; -} - -/*! - * \brief Set/reset/unset the StatInstRowIterator that is - * given to us. Passing in a null row iterator is the same - * thing as resetting this report's row iterator; it will - * not reject a null iterator object. - */ -void Report::recursSetSIRowIterator_( - std::shared_ptr & si_row_iterator) -{ - si_row_iterator_ = si_row_iterator; - for (auto & sr : subreps_) { - sr.recursSetSIRowIterator_(si_row_iterator); - } - - //Nothing left to do if this call just reset - //our iterator to null. - if (si_row_iterator_ == nullptr) { - return; - } - - //Now that we have our row iterator object set, tell - //all of our SI's to realize their StatInstValueLookup - //placeholders. It works like this: - // - // 1. Create sparta::Report/StatisticInstance objects - // from SimDB records. This results in: - // - Report nodes have a StatInstRowIterator *placeholder* - // - SI nodes have a StatInstValueLookup *placeholder* - // - // 2. Get the topmost StatInstRowIterator placeholder - // - // 3. Clone that placeholder into a "real" row iterator - // - // 4. Share this "real" row iterator with all report nodes - // - // 5. Let the individual SI's use this "real" row iterator - // to clone their StatInstValueLookup *placeholders* - // into "real" SI value lookups. - // - //Step 5 is what we're doing here: - for (auto & si : stats_) { - si.second->realizeSIValueDirectLookup(*si_row_iterator_); - } -} - -/*! - * \brief This utility can be used to generate a formatted - * report from a root-level ReportNodeHierarchy record in - * the provided database (ObjectManager). - * - * The given report_hier_node_id must have ParentNodeID=0 - * in the ReportNodeHierarchy table, or this method will - * throw an exception. - */ -bool Report::createFormattedReportFromDatabase( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID report_hier_node_id, - const std::string & filename, - const std::string & format, - const Scheduler * scheduler) -{ - std::string _format(format); - std::transform(_format.begin(), _format.end(), - _format.begin(), ::tolower); - - if (_format == "csv" || _format == "csv_cumulative") { - std::cout << "Timeseries report generation is currently " - << "unsupported for this API" << std::endl; - return false; - } - - utils::ValidValue success; - - obj_mgr.safeTransaction([&]() { - //We currently only support SimDB->report creation - //starting with root-level report nodes (i.e. "_global" - //as opposed to "top.core0.rob" which lives under - //"_global"). - std::unique_ptr query( - new simdb::ObjectQuery(obj_mgr, "ReportNodeHierarchy")); - - query->addConstraints("Id", simdb::constraints::equal, - report_hier_node_id); - - simdb::DatabaseID parent_id = -1; - query->writeResultIterationsTo("ParentNodeID", &parent_id); - - auto result_iter = query->executeQuery(); - sparta_assert(result_iter->getNext()); - - if (parent_id > 0) { - //TODO: Add support for generating any SPARTA report from - //any node in the original SI hierarchy. There is no reason - //the entire, potentially large, SI data set needs to be - //written to the file. We should be able to pick off a - //subset of the data by node path, for example in Python: - // - // >>> dbconn.sim.top.core0.rob.toJsonReduced('rob_reduced.json') - // - throw SpartaException( - "Report::createFormattedReportFromDatabase() " - "API cannot be called for a ReportNodeHierarchy " - "record which has a non-zero/non-null ParentNodeID. " - "Only root-level report nodes can be used to create " - "formatted report files from the SimDB."); - } - - Report report(report_hier_node_id, obj_mgr, scheduler); - - if (_format.find("auto") == 0) { - //We use ReportDescriptor directly to generate formatted - //reports from the database for most formats. But auto-summary - //formats have some "hard-coded" formatting options that other - //formats don't have. This is the same way the app::Simulation - //class prints the --auto-summary to stdout: - report::format::Text summary_fmt(&report); - summary_fmt.setValueColumn(summary_fmt.getRightmostNameColumn()); - summary_fmt.setReportPrefix(""); - summary_fmt.setQuoteReportNames(false); - summary_fmt.setWriteContentlessReports(false); - summary_fmt.setShowSimInfo(false); - - if (_format == "auto_verbose") { - summary_fmt.setShowDescriptions(true); - } - - if (filename == utils::COUT_FILENAME) { - std::cout << summary_fmt << std::endl; - } else { - std::ofstream auto_fout(filename); - if (!auto_fout) { - throw SpartaException("Unable to open file for write: '") - << filename << "'"; - } - auto_fout << summary_fmt << std::endl; - } - - success = true; - return; - } - - //Now that we have a Report (and its SI's) recreated - //completely, use a ReportDescriptor object directly - //so we get all of the legacy formatting code for - //free. The format we give the descriptor will turn - //into the right BaseFormatter under the hood for us. - app::ReportDescriptor rd("", "", filename, _format); - report::format::BaseFormatter * formatter = - rd.addInstantiation(&report, nullptr, nullptr); - - //Tack on any report metadata we find in the database. - //Give this metadata to the BaseFormatter. The formatter - //subclasses will write this metadata out to the report - //files. - simdb::ObjectQuery root_metadata_query(obj_mgr, "RootReportNodeMetadata"); - root_metadata_query.addConstraints( - "ReportNodeID", simdb::constraints::equal, report_hier_node_id); - - std::string root_meta_name, root_meta_value; - root_metadata_query.writeResultIterationsTo( - "Name", &root_meta_name, - "Value", &root_meta_value); - - result_iter = root_metadata_query.executeQuery(); - while (result_iter->getNext()) { - if (root_meta_name == "Elapsed") { - continue; - } - formatter->setMetadataByNameAndStringValue( - root_meta_name, root_meta_value); - } - - //The "OmitZero" and "PrettyPrintDisable" report styles are - //put in the database to signal when the original simulator - //used the "--omit-zero-value-stats-from-json_reduced" or - //the "--no-json-pretty-print" command line options. - if (report.getStyle("OmitZero") == "true") { - formatter->omitStatsWithValueZero(); - } - if (report.getStyle("PrettyPrintDisabled") == "true") { - formatter->disablePrettyPrint(); - } - - report.prepareForSIDatabaseIteration_(obj_mgr); - if (report.si_row_iterator_ == nullptr) { - success = false; - return; - } - - //Advance the ObjectQuery to the first row / SI blob - if (!report.si_row_iterator_->getNext()) { - success = false; - return; - } - - //Recurse down through the report nodes and ask all - //SI's if their StatInstValueLookup objects are ready - //to go. - recursValidateSIDirectLookups(report); - - //Manually recreate a SimulationInfo object from - //the contents of this database. - std::unique_ptr sim_info; - - /* Start by taking this report ID we were given, and - * getting its root report ID. For example, we could - * be recreating a report for the "top.core0" node, - * but the root is "top". - * - * The connection between reports and sim info records - * looks like this: - * - * RootReportObjMgrIDs SimInfo - * ---------------------- ---------- - * RootReportNodeID |--> ObjMgrID - * ObjMgrID <--| Name --\ - * Cmdline \ (SimInfo - * ... / metadata) - * ... --/ - */ - query.reset(new simdb::ObjectQuery(obj_mgr, "RootReportObjMgrIDs")); - - const auto root_id = getRootReportNodeIDFrom( - report_hier_node_id, obj_mgr); - - query->addConstraints( - "RootReportNodeID", simdb::constraints::equal, root_id); - - int32_t obj_mgr_id = 0; - query->writeResultIterationsTo("ObjMgrID", &obj_mgr_id); - - result_iter = query->executeQuery(); - if (result_iter->getNext()) { - //Let the SimulationInfo's constructor take this database - //ID and turn it into a live object. While this object is - //in scope (during this post-simulation report generation), - //calls to SimulationInfo::getInstance() will actually be - //talking to this recreated object directly, not the normal - //singleton that is active during simulations. - sim_info.reset(new SimulationInfo( - obj_mgr, obj_mgr_id, report_hier_node_id)); - } - - rd.writeOutput(); - - //Give the BaseFormatter subclasses a chance to clear - //any internal state before continuing. In the case of - //exporting multiple reports from SimDB to a formatted - //report file one at a time, any leftover state from - //one export to the next can cause problems. - formatter->doPostProcessingBeforeReportValidation(); - - success = true; - }); - - return success; -} - -/*! - * \brief Turn the vector of char's we are holding onto into - * a vector double's. This takes into account compression if - * it was performed on the current SI row. - */ -bool StatInstRowIterator::getCurrentRowDoubles_() -{ - if (raw_si_was_compressed_) { - //Re-inflate the compressed SI blob - z_stream infstream; - infstream.zalloc = Z_NULL; - infstream.zfree = Z_NULL; - infstream.opaque = Z_NULL; - - //Setup the source stream - infstream.avail_in = (uInt)(raw_si_bytes_.size()); - infstream.next_in = (Bytef*)(&raw_si_bytes_[0]); - - //Setup the destination stream - raw_si_values_.resize(raw_si_num_pts_); - infstream.avail_out = (uInt)(raw_si_values_.size() * sizeof(double)); - infstream.next_out = (Bytef*)(&raw_si_values_[0]); - - //Inflate it! - inflateInit(&infstream); - inflate(&infstream, Z_FINISH); - inflateEnd(&infstream); - } - - else { - raw_si_values_.resize(raw_si_num_pts_); - - memcpy(&raw_si_values_[0], - &raw_si_bytes_[0], - raw_si_num_pts_ * sizeof(double)); - } - - return !raw_si_values_.empty(); -} - } // namespace sparta - -//! Classes and methods in the placeholders namespace are -//! used in order to "partially create" an object when you -//! have some of, but not all of, its constructor arguments -//! on hand. When you eventually get the rest of its constructor -//! arguments later on, those arguments are forwarded to the -//! placeholder object, who returns a clone of the final, -//! *realized* object. -namespace placeholders { - -//! Given a placeholders::StatInstRowIterator object, ask it -//! to clone itself into a sparta::StatInstRowIterator object. -sparta::StatInstRowIterator * -placeholders::StatInstRowIterator::realizePlaceholder() -{ - //The row iterator class expects a root-level report - //hierarchy node. Let's take the report node ID that - //we were originally given, get the ID of the node - //at the top of its report tree, and realize the row - //iterator using that root-level node ID. - const simdb::DatabaseID root_node_id = - sparta::getRootReportNodeIDFrom(report_hier_node_id_, *obj_mgr_); - - sparta_assert(root_node_id > 0, "Invalid report database ID"); - return new sparta::StatInstRowIterator(root_node_id, *obj_mgr_); -} - -//! Given a placeholders::StatInstValueLookup object, ask it -//! to clone itself into a sparta::StatInstValueLookup object. -sparta::StatInstValueLookup * -placeholders::StatInstValueLookup::realizePlaceholder( - const sparta::StatInstRowIterator::RowAccessorPtr & row_accessor) -{ - //The only thing this placeholders object had to begin - //with was the SI's individual leaf index. In this method, - //we are being given a indirect reference to the underlying - //vector of SI values that all SI's in this report tree - //belong to. The direct reference to this vector is through - //the RowAccessor's getCurrentRow() method. That vector reference, - //combined with our unique leaf SI index, is enough to realize - //a finalized, usable, sparta::StatInstValueLookup. - return new sparta::StatInstValueLookup(row_accessor, si_index_); -} - -} // namespace placeholders diff --git a/sparta/src/ReportDescriptor.cpp b/sparta/src/ReportDescriptor.cpp index 019402e2fd..dff786cca5 100644 --- a/sparta/src/ReportDescriptor.cpp +++ b/sparta/src/ReportDescriptor.cpp @@ -29,16 +29,7 @@ #include "sparta/statistics/dispatch/archives/ReportStatisticsArchive.hpp" #include "sparta/statistics/dispatch/streams/StreamNode.hpp" #include "sparta/statistics/dispatch/ReportStatisticsHierTree.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "sparta/async/AsyncNonTimeseriesReport.hpp" -#include "sparta/async/AsyncTimeseriesReport.hpp" #include "sparta/app/FeatureConfiguration.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" #include "sparta/report/Report.hpp" #include "sparta/simulation/RootTreeNode.hpp" #include "sparta/kernel/Scheduler.hpp" @@ -47,7 +38,6 @@ #include "sparta/simulation/TreeNode.hpp" #include "sparta/utils/Utils.hpp" #include "sparta/app/SimulationConfiguration.hpp" -#include "sparta/report/db/ReportHeader.hpp" #include "sparta/report/format/BaseFormatter.hpp" #include "sparta/trigger/ExpressionTrigger.hpp" @@ -59,80 +49,11 @@ class EventHandler; #include "python/sparta_support/module_sparta.hpp" #endif - -namespace sparta { -namespace db { - -//! Asynchronous report metadata writer which persists -//! some information about SimDB report records that -//! this ReportDescriptor is holding onto. -class ReportVerifMetaWriter : public simdb::WorkerTask -{ -public: - ReportVerifMetaWriter(simdb::ObjectManager * sim_db, - const simdb::DatabaseID root_report_node_id, - const std::string & dest_file, - const std::string & format) : - sim_db_(sim_db), - root_report_node_id_(root_report_node_id), - dest_file_(dest_file), - format_(format) - {} - -private: - //! WorkerTask implementation - virtual void completeTask() override final { - auto verif_meta_tbl = sim_db_->getTable("ReportVerificationMetadata"); - - verif_meta_tbl->createObjectWithArgs( - "RootReportNodeID", root_report_node_id_, - "DestFile", dest_file_, - "Format", format_); - } - - simdb::ObjectManager * sim_db_ = nullptr; - simdb::DatabaseID root_report_node_id_ = 0; - std::string dest_file_; - std::string format_; -}; - -} // namespace db -} // namespace sparta - namespace sparta { namespace app { const char* ReportDescriptor::GLOBAL_KEYWORD = "_global"; -//! Single out descriptors who only have one .csv destination -//! file/report. This is used to test out timeseries database -//! designs / ideas. -bool ReportDescriptor::isSingleTimeseriesReport() const -{ - if (getAllInstantiations().size() != 1) { - return false; - } - - utils::lowercase_string fmt(format); - if (fmt.getString() == "csv" || fmt.getString() == "csv_cumulative") { - return true; - } else if (!fmt.getString().empty()) { - return false; - } - - std::string extension = std::filesystem::path(dest_file).extension(); - utils::lowercase_string ext(extension); - return ext.getString() == ".csv"; -} - -bool ReportDescriptor::isSingleNonTimeseriesReport() const -{ - if (getAllInstantiations().size() != 1) { - return false; - } - return !isSingleTimeseriesReport(); -} - bool ReportDescriptor::isValidFormatName(const std::string& format) { return sparta::report::format::BaseFormatter::isValidFormatName(format); @@ -261,197 +182,6 @@ std::shared_ptr ReportDescriptor::createRootStatisticsSt return nullptr; } -void ReportDescriptor::configureAsyncTimeseriesReport( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db, - const Clock & root_clk) -{ - auto reports = getAllInstantiations(); - sparta_assert(reports.size() == 1 && reports[0] != nullptr); - const Report * report = reports[0]; - - db_timeseries_.reset(new async::AsyncTimeseriesReport( - task_queue, sim_db, root_clk, *report, - simdb_feature_opts_)); - - //Put the comma-separated SI locations string into the - //database. This will be added as a single row to the - //CSV file above the SI values when users ask the SI - //database to generate their .csv file: - // - // scheduler.ticks,scheduler.seconds,... - // 1000,1.00E-09,... - const std::vector & si_locations = - db_timeseries_->getStatInstLocations(); - - std::ostringstream loc_oss; - if (si_locations.size() == 1) { - loc_oss << si_locations[0]; - } else if (si_locations.size() > 1) { - for (size_t idx = 0; idx < si_locations.size() - 1; ++idx) { - loc_oss << si_locations[idx] << ","; - } - loc_oss << si_locations.back(); - } - - const std::string comma_separated_si_locs = loc_oss.str(); - db::ReportHeader & db_header = db_timeseries_->getTimeseriesHeader(); - auto header_tbl = sim_db->getTable("ReportHeader"); - - db_header.getObjectRef().getObjectManager().safeTransaction([&]() { - header_tbl->updateRowValues("ReportName", report->getName(), - "StartTime", report->getStart(), - "EndTime", report->getEnd(), - "DestFile", dest_file, - "SILocations", comma_separated_si_locs, - "NumStatInsts", (int)si_locations.size()). - forRecordsWhere("Id", simdb::constraints::equal, db_header.getId()); - }); -} - -db::ReportHeader * ReportDescriptor::getTimeseriesDatabaseHeader() -{ - if (db_timeseries_ == nullptr) { - return nullptr; - } - return &db_timeseries_->getTimeseriesHeader(); -} - -void ReportDescriptor::configureAsyncNonTimeseriesReport( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db) -{ - auto reports = getAllInstantiations(); - sparta_assert(reports.size() == 1 && reports[0] != nullptr); - const Report * report = reports[0]; - - db_non_timeseries_.reset(new async::AsyncNonTimeseriesReport( - task_queue, sim_db, *report, simdb_feature_opts_)); -} - -void ReportDescriptor::doPostProcessing( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db) -{ - if (instantiations_.size() == 1) { - auto formatter = instantiations_.back().second; - sparta_assert(formatters_.find(dest_file) != formatters_.end()); - sparta_assert(formatters_[dest_file].get() == formatter); - - const auto & metadata = formatter->getMetadataKVPairs(); - const auto & elapsed = SimulationInfo::getInstance().getLastCapturedElapsedTime(); - - auto report_style_tbl = sim_db ? sim_db->getTable("ReportStyle") : nullptr; - - auto create_omit_zero_obj = [&report_style_tbl]( - const simdb::DatabaseID report_id) - { - if (!report_style_tbl) { - return; - } - static const std::string OmitZeroName = "OmitZero"; - static const std::string OmitZeroValue = "true"; - report_style_tbl->createObjectWithArgs( - "StyleName", OmitZeroName, - "StyleValue", OmitZeroValue, - "ReportNodeID", report_id); - }; - - auto create_no_pretty_print_obj = [&report_style_tbl]( - const simdb::DatabaseID report_id) - { - if (!report_style_tbl) { - return; - } - static const std::string PPrintDisabledName = "PrettyPrintDisabled"; - static const std::string PPrintDisabledValue = "true"; - report_style_tbl->createObjectWithArgs( - "StyleName", PPrintDisabledName, - "StyleValue", PPrintDisabledValue, - "ReportNodeID", report_id); - }; - - if (db_non_timeseries_ != nullptr) { - for (const auto & md : metadata) { - db_non_timeseries_->setStringMetadataByNameAndValue(md.first, md.second); - } - if (elapsed.isValid()) { - db_non_timeseries_->setStringMetadataByNameAndValue("Elapsed", elapsed); - } - - const bool omit_zero = formatter->statsWithValueZeroAreOmitted(); - const bool pprint_enabled = formatter->prettyPrintEnabled(); - if (omit_zero) { - create_omit_zero_obj( - db_non_timeseries_->getRootReportNodeDatabaseID()); - } - if (!pprint_enabled) { - create_no_pretty_print_obj( - db_non_timeseries_->getRootReportNodeDatabaseID()); - } - } - - if (db_timeseries_ != nullptr) { - db::ReportHeader & db_header = db_timeseries_->getTimeseriesHeader(); - for (const auto & md : metadata) { - db_header.setStringMetadata(md.first, md.second); - } - if (elapsed.isValid()) { - db_header.setStringMetadata("Elapsed", elapsed); - } - - const bool omit_zero = formatter->statsWithValueZeroAreOmitted(); - const bool pprint_enabled = formatter->prettyPrintEnabled(); - if (omit_zero) { - create_omit_zero_obj( - db_timeseries_->getRootReportNodeDatabaseID()); - } - if (!pprint_enabled) { - create_no_pretty_print_obj( - db_timeseries_->getRootReportNodeDatabaseID()); - } - - auto header_lines = formatter->getWrittenHeaderLines(); - - std::ostringstream header_oss; - for (size_t idx = 0; idx < header_lines.size(); ++idx) { - header_oss << header_lines[idx] << "\n"; - } - db_header.setStringMetadata("RawHeader", header_oss.str()); - } - - formatter->doPostProcessingBeforeReportValidation(); - } - - if (db_non_timeseries_ != nullptr) { - sparta_assert(sim_db != nullptr); - auto report_id = db_non_timeseries_->getRootReportNodeDatabaseID(); - - //The only time the db::SingleUpdateReport object would - //have a root report ID of zero (invalid/unset) at the - //end of the simulation is if it has a start trigger - //that has not fired yet. We still flush those report - //descriptors in our destructor. - if (report_id == 0) { - return; - } - - std::unique_ptr report_verif_meta_writer( - new db::ReportVerifMetaWriter(sim_db, report_id, dest_file, format)); - - if (task_queue != nullptr) { - task_queue->addWorkerTask(std::move(report_verif_meta_writer)); - } else { - report_verif_meta_writer->completeTask(); - } - - //Reset the DB object. We'll need to short circuit the call - //to this doPostProcessing() method if it gets called again - //from our destructor. - db_non_timeseries_.reset(); - } -} - std::map> ReportDescriptor::getFormattersByFilename() const { @@ -549,17 +279,6 @@ report::format::BaseFormatter* ReportDescriptor::addInstantiation(Report* r, if(formatter->supportsUpdate()){ //TODO: Deprecate "during simulation" formatters formatter->writeHeader(); - - if (db_timeseries_ != nullptr) { - db::ReportHeader & db_header = db_timeseries_->getTimeseriesHeader(); - auto header_tbl = GET_DB_FOR_COMPONENT(Stats, sim)->getTable("ReportHeader"); - - db_header.getObjectRef().getObjectManager().safeTransaction([&]() { - header_tbl->updateRowValues("StartTime", r->getStart(), - "EndTime", r->getEnd()). - forRecordsWhere("Id", simdb::constraints::equal, db_header.getId()); - }); - } } return formatter; @@ -581,12 +300,6 @@ ReportDescriptor::~ReportDescriptor() this->updateOutput(nullptr); this->writeOutput(nullptr); - - if (db_non_timeseries_ != nullptr) { - simdb::AsyncTaskEval * null_task_queue = nullptr; - auto sim_db = db_non_timeseries_->getSimulationDatabase(); - doPostProcessing(null_task_queue, sim_db); - } } } catch (...) { //destructors should never throw @@ -619,12 +332,8 @@ uint32_t ReportDescriptor::writeOutput(std::ostream* out) for(auto & inst : getInstantiations()){ const bool report_active = this->updateReportActiveState_(inst.first); if (report_active && false == inst.second->supportsUpdate()) { - sparta_assert(db_timeseries_ == nullptr); //TODO: Deprecate "during simulation" formatters inst.second->write(); - if (db_non_timeseries_ != nullptr) { - db_non_timeseries_->writeCurrentValues(); - } num_saved++; // User information @@ -669,26 +378,11 @@ uint32_t ReportDescriptor::updateOutput(std::ostream* out) for(auto & inst : getInstantiations()){ const bool report_active = this->updateReportActiveState_(inst.first); if(report_active && inst.second->supportsUpdate()){ - sparta_assert(db_non_timeseries_ == nullptr); bool capture_update_values = true; if (skipped_annotator_ != nullptr) { if (skipped_annotator_->currentSkipCount() > 0) { //TODO: Deprecate "during simulation" formatters inst.second->skip(skipped_annotator_.get()); - if (db_timeseries_ != nullptr) { - //Timeseries .csv files annotate rows in the report - //that show how many updates were skipped over due - //to a report being temporarily disabled (this can - //happen with reports that use toggle triggers). - // - //Async/database timeseries with these kinds of - //triggers needs some more thought. This is featured - //off by default, so this should never get hit in - //production. - throw SpartaException( - "Asynchronous timeseries generation is currently not " - "supported when using report toggle triggers."); - } inst.first->start(); capture_update_values = false; } @@ -697,9 +391,6 @@ uint32_t ReportDescriptor::updateOutput(std::ostream* out) if (capture_update_values) { //TODO: Deprecate "during simulation" formatters inst.second->update(); - if (db_timeseries_ != nullptr) { - db_timeseries_->writeCurrentValues(); - } } num_updated++; @@ -1718,7 +1409,7 @@ utils::ValidValue getNotifSourceForStopTrigger( void ReportDescriptor::inspectSimulatorFeatureValues( const FeatureConfiguration * feature_config) { - simdb_feature_opts_ = GetFeatureOptions(feature_config, "simdb"); + (void) feature_config; } } // namespace sparta diff --git a/sparta/src/ReportHeader.cpp b/sparta/src/ReportHeader.cpp deleted file mode 100644 index 591b56b290..0000000000 --- a/sparta/src/ReportHeader.cpp +++ /dev/null @@ -1,195 +0,0 @@ -// -*- C++ -*- - -#include "sparta/report/db/ReportHeader.hpp" - -#include - -#include "sparta/report/db/ReportTimeseries.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/Constraints.hpp" - -namespace sparta { -namespace db { - -/*! - * \brief This class is just a user-friendly wrapper around - * report metadata, so it simply forwards set/get requests - * right to its ObjectRef (record wrapper) - */ - -ReportHeader::ReportHeader(std::unique_ptr obj_ref) : - obj_ref_(std::move(obj_ref)) -{ -} - -ReportHeader::ReportHeader(const simdb::ObjectManager & obj_mgr) -{ - obj_ref_ = obj_mgr.getTable("ReportHeader")->createObject(); -} - -uint64_t ReportHeader::getId() const -{ - return obj_ref_->getId(); -} - -const simdb::ObjectRef & ReportHeader::getObjectRef()const -{ - return *obj_ref_; -} - -simdb::ObjectRef & ReportHeader::getObjectRef() -{ - return *obj_ref_; -} - -void ReportHeader::setOwningTimeseries(const ReportTimeseries & ts) -{ - obj_ref_->setPropertyInt32("TimeseriesID", ts.getId()); -} - -void ReportHeader::setReportName(const std::string & report_name) -{ - obj_ref_->setPropertyString("ReportName", report_name); -} - -void ReportHeader::setReportStartTime(const uint64_t start_time) -{ - obj_ref_->setPropertyUInt64("StartTime", start_time); -} - -void ReportHeader::setReportEndTime(const uint64_t end_time) -{ - obj_ref_->setPropertyUInt64("EndTime", end_time); -} - -void ReportHeader::setSourceReportDescDestFile( - const std::string & fname) -{ - obj_ref_->setPropertyString("DestFile", fname); -} - -void ReportHeader::setCommaSeparatedSILocations( - const std::string & si_locations) -{ - obj_ref_->setPropertyString("SILocations", si_locations); -} - -void ReportHeader::setSourceReportNumStatInsts( - const uint32_t num_stat_insts) -{ - obj_ref_->setPropertyInt32("NumStatInsts", num_stat_insts); -} - -std::string ReportHeader::getReportName() const -{ - return obj_ref_->getPropertyString("ReportName"); -} - -uint64_t ReportHeader::getReportStartTime() const -{ - return obj_ref_->getPropertyUInt64("StartTime"); -} - -uint64_t ReportHeader::getReportEndTime() const -{ - return obj_ref_->getPropertyUInt64("EndTime"); -} - -std::string ReportHeader::getSourceReportDescDestFile() const -{ - return obj_ref_->getPropertyString("DestFile"); -} - -std::string ReportHeader::getCommaSeparatedSILocations() const -{ - return obj_ref_->getPropertyString("SILocations"); -} - -void ReportHeader::setStringMetadata( - const std::string & name, - const std::string & value) -{ - const simdb::ObjectManager & obj_mgr = obj_ref_->getObjectManager(); - auto metadata_tbl = obj_mgr.getTable("StringMetadata"); - constexpr auto equals = simdb::constraints::equal; - - if (metadata_tbl->updateRowValues("MetadataValue", value). - forRecordsWhere("ReportHeaderID", equals, getId(), - "MetadataName", equals, name)) - { - return; - } else { - metadata_tbl->createObjectWithArgs( - "ReportHeaderID", getId(), - "MetadataName", name, - "MetadataValue", value); - } -} - -std::string ReportHeader::getStringMetadata(const std::string & name) const -{ - const simdb::ObjectManager & obj_mgr = obj_ref_->getObjectManager(); - - simdb::ObjectQuery query(obj_mgr, "StringMetadata"); - query.addConstraints( - "ReportHeaderID", simdb::constraints::equal, getId(), - "MetadataName", simdb::constraints::equal, name); - - std::string metadata_value; - query.writeResultIterationsTo("MetadataValue", &metadata_value); - query.executeQuery()->getNext(); - - return metadata_value; -} - -std::map locGetAllStringMetadata( - const ReportHeader & header) -{ - const simdb::ObjectManager & obj_mgr = - header.getObjectRef().getObjectManager(); - - simdb::ObjectQuery query(obj_mgr, "StringMetadata"); - query.addConstraints("ReportHeaderID", simdb::constraints::equal, header.getId()); - - std::string metadata_name, metadata_value; - query.writeResultIterationsTo("MetadataName", &metadata_name, - "MetadataValue", &metadata_value); - - std::map metadata_pairs; - auto result_iter = query.executeQuery(); - while (result_iter->getNext()) { - metadata_pairs[metadata_name] = metadata_value; - } - return metadata_pairs; -} - -std::map ReportHeader::getAllStringMetadata() const -{ - std::map all_metadata = locGetAllStringMetadata(*this); - std::map visible_metadata; - for (const auto & md : all_metadata) { - if (md.first.find("__") != 0) { - visible_metadata[md.first] = md.second; - } - } - return visible_metadata; -} - -std::map ReportHeader::getAllHiddenStringMetadata() const -{ - std::map all_metadata = locGetAllStringMetadata(*this); - std::map hidden_metadata; - for (const auto & md : all_metadata) { - if (md.first.find("__") == 0 && md.first.size() > 2) { - const std::string culled_metadata_name = md.first.substr(2); - hidden_metadata[culled_metadata_name] = md.second; - } - } - return hidden_metadata; -} - -} // namespace db -} // namespace sparta diff --git a/sparta/src/ReportRepository.cpp b/sparta/src/ReportRepository.cpp index e7a607dbe2..fd60d7b411 100644 --- a/sparta/src/ReportRepository.cpp +++ b/sparta/src/ReportRepository.cpp @@ -26,13 +26,7 @@ #include "sparta/statistics/dispatch/archives/ReportStatisticsArchive.hpp" #include "sparta/statistics/dispatch/archives/StatisticsArchives.hpp" #include "sparta/statistics/dispatch/streams/StatisticsStreams.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/ObjectManager.hpp" -#include "sparta/report/db/ReportHeader.hpp" #include "sparta/app/FeatureConfiguration.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/schema/DatabaseRoot.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/statistics/CounterBase.hpp" #include "sparta/simulation/GlobalTreeNode.hpp" @@ -130,13 +124,6 @@ class Directory on_triggered_notifier_ = on_triggered_notifier; } - void doPostProcessing( - simdb::AsyncTaskEval * task_queue, - simdb::ObjectManager * sim_db) - { - desc_.doPostProcessing(task_queue, sim_db); - } - std::vector> saveReports(size_t & num_written) { if (formatters_.empty()) { @@ -335,7 +322,6 @@ class Directory referenced_directories_[referenced_directory_key_] = this; } start_expression_ = start_expression; - needs_header_overwrite_ = true; } }; @@ -510,21 +496,6 @@ class Directory this->initializeReportInstantiations_(); - if (needs_header_overwrite_) { - db::ReportHeader * db_header = desc_.getTimeseriesDatabaseHeader(); - if (db_header != nullptr) { - sparta_assert(reports_.size() == 1); - if (reports_[0]->hasHeader()) { - auto & header = reports_[0]->getHeader(); - const std::map stringified = header.getAllStringified(); - for (const auto & meta : stringified) { - db_header->setStringMetadata(meta.first, meta.second); - } - } - } - needs_header_overwrite_ = false; - } - if (is_cumulative_) { for (auto & r : reports_) { r->accumulateStats(); @@ -886,7 +857,6 @@ class Directory bool update_reports_when_enabled_ = false; bool reports_have_started_ = false; bool update_descriptor_when_asked_ = true; - bool needs_header_overwrite_ = false; bool is_cumulative_ = false; utils::ValidValue update_delta_; @@ -1018,10 +988,6 @@ class ReportRepository::Impl } rd.inspectSimulatorFeatureValues(feature_config); } - - if (IsFeatureValueEnabled(feature_config, "simdb")) { - configureAsyncReportStreams_(); - } } statistics::StatisticsArchives * getStatsArchives() @@ -1118,7 +1084,6 @@ class ReportRepository::Impl for (auto & report : reports) { saved_reports.emplace_back(report.release()); } - doPostProcessing_(directories_[handle].get()); } //Do not print anything if we didn't have any reports @@ -1130,32 +1095,8 @@ class ReportRepository::Impl << std::endl << std::endl; } - //Before deleting the directories and ReportDescriptor's, - //flush the async database engine. This is done like this - //so that any objects using the engine do not have to rely - //on their destructors for important last-minute work to do. - simdb::AsyncTaskController * task_controller = nullptr; - if (sim_) { // %%% See note below about nulling out sim_ - if (auto db_root = sim_->getDatabaseRoot()) { - task_controller = db_root->getTaskController(); - } - } - - if (task_controller != nullptr) { - task_controller->emitPreFlushNotification(); - task_controller->flushQueue(); - } - directories_.clear(); - //In the cases where a report trigger was disabled at the - //time of the report write/update(s), the descriptor's - //destructor may have put one last item in the task queue. - if (task_controller != nullptr) { - task_controller->emitPreFlushNotification(); - task_controller->flushQueue(); - } - // %%% ... if(sim_) { ... %%% // This same method can get called again from our own // destructor, *after* the Simulation has already been @@ -1175,120 +1116,6 @@ class ReportRepository::Impl { } - //! One-time setup of asynchronous report streams / worker threads - void configureAsyncReportStreams_() - { - - if(!sim_) { - return; - } - - //The async timeseries object needs a few things from us: - // - the app::Simulation's scheduler and root clock - // - the app::Simulation's shared database object - const Scheduler * scheduler = nullptr; - const Clock * root_clk = nullptr; - - auto obj_db = GET_DB_FOR_COMPONENT(stats, sim_); - auto sim_db = obj_db ? obj_db->getObjectManager() : nullptr; - sparta_assert(sim_db != nullptr); - - simdb::AsyncTaskController * task_controller = nullptr; - - if (auto db_root = sim_->getDatabaseRoot()) { - task_controller = db_root->getTaskController(); - } - - //As well as: - // - a shared worker thread (simdb::AsyncTaskEval) - // (we'll create one for all of our descriptors, and - // everyone will get a shared_ptr) - for (auto & dir : directories_) { - auto & rd = dir.second->getDescriptor(); - if (!rd.isEnabled()) { - continue; - } - - //Report formats such as JSON do not add much overhead - //to the simulation since they only have one data point - //across their SI's, and some metadata. There isn't that - //much metadata to warrant making them asynchronous (yet!). - //For now, we will just put timeseries reports on a worker - //thread since they support update/interval triggers, and - //can produce arbitrarily large amounts of SI data values. - if (rd.isSingleTimeseriesReport()) { - - if (scheduler == nullptr) { - sparta_assert(sim_); - scheduler = sim_->getScheduler(); - } - - if (root_clk == nullptr) { - sparta_assert(sim_); - root_clk = sim_->getRootClock(); - } - - sim_db->addToTaskController(task_controller); - auto task_queue = sim_db->getTaskQueue(); - task_queues_by_dir_[dir.second.get()] = task_queue; - - //We have one consumer thread per simulation, and one - //report descriptor per report. Ask the report descriptor - //to create its own async timeseries report object, which - //will forward report metadata DB writes and SI blob writes - //to the one shared worker object that we just created. - rd.configureAsyncTimeseriesReport( - task_queue, - sim_db, - *root_clk); - - //Write any report trigger metadata into the header object - auto db_header = rd.getTimeseriesDatabaseHeader(); - if (db_header) { - //Simulation name is written as hidden metadata by - //by prefixing the name with "__" - db_header->setStringMetadata( - "__simulationName", sim_->getSimName()); - } - } else if (rd.isSingleNonTimeseriesReport()) { - sim_db->addToTaskController(task_controller); - auto task_queue = sim_db->getTaskQueue(); - task_queues_by_dir_[dir.second.get()] = task_queue; - - rd.configureAsyncNonTimeseriesReport( - task_queue, - sim_db); - } - } - } - - void doPostProcessing_(Directory * dir) { - if (directories_.empty()) { - return; - } - - auto obj_db = GET_DB_FOR_COMPONENT(stats, sim_); - simdb::ObjectManager * sim_db = - obj_db ? obj_db->getObjectManager() : nullptr; - - auto post_processing = [&]() { - auto task_queue_iter = task_queues_by_dir_.find(dir); - auto task_queue = - task_queue_iter != task_queues_by_dir_.end() ? - task_queue_iter->second : - nullptr; - dir->doPostProcessing(task_queue, sim_db); - }; - - if (sim_db != nullptr) { - sim_db->safeTransaction([&]() { - post_processing(); - }); - } else { - post_processing(); - } - } - app::Simulation * sim_ = nullptr; TreeNode *const context_; std::shared_ptr sub_container_; @@ -1301,10 +1128,6 @@ class ReportRepository::Impl std::shared_ptr> on_triggered_notifier_; std::unique_ptr stats_archives_; std::unique_ptr stats_streams_; - std::unordered_map< - Directory*, - simdb::AsyncTaskEval* - > task_queues_by_dir_; }; ReportRepository::ReportRepository(app::Simulation * sim) : diff --git a/sparta/src/ReportTimeseries.cpp b/sparta/src/ReportTimeseries.cpp deleted file mode 100644 index b71dbe72d4..0000000000 --- a/sparta/src/ReportTimeseries.cpp +++ /dev/null @@ -1,943 +0,0 @@ -// -*- C++ -*- - -#include "sparta/report/db/ReportTimeseries.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/report/db/Schema.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/utils/BlobHelpers.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -//SQLite-specific headers -#include "simdb/impl/sqlite/TransactionUtils.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/DbConnProxy.hpp" -#include "simdb/Errors.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" - -namespace sparta { -namespace db { - -//! Template helper for getting a particular property value -//! from a database record (row). The onFound() method will -//! be called by sqlite for every record that matches the -//! query you run. -template -class MultiSelectPropertyQuery { -public: - int onFound(int argc, char ** argv, char **) { - assert(argc == 1); - - std::vector & prop_values = property_values_; - // TODO swap for our own - prop_values.emplace_back(boost::lexical_cast(argv[0])); - - //Property is valid, return SQLITE_OK for success - return SQLITE_OK; - } - - bool found() const { - return !property_values_.empty(); - } - - const std::vector & getPropertyValues() const { - assert(found()); - return property_values_; - } - -private: - std::vector property_values_; -}; - -ReportTimeseries::ReportTimeseries(std::unique_ptr obj_ref) : - obj_ref_(std::move(obj_ref)) -{ - //Try to find a ReportHeader record whose TimeseriesID is - //*our* database ID. - std::unique_ptr our_header_obj = - obj_ref_->getObjectManager().findObject("ReportHeader", obj_ref_->getId()); - - //There is currently no reason why a timeseries object would - //exist without any header record to go with it. - assert(our_header_obj != nullptr); - header_.reset(new ReportHeader(std::move(our_header_obj))); -} - -ReportTimeseries::ReportTimeseries(const simdb::ObjectManager & obj_mgr) -{ - //Create a brand new timeseries object in the database - std::unique_ptr timeseries_tbl = obj_mgr.getTable("Timeseries"); - obj_ref_ = timeseries_tbl->createObject(); - - //And create a new header object to go with it - header_.reset(new ReportHeader(obj_ref_->getObjectManager())); - header_->setOwningTimeseries(*this); -} - -uint64_t ReportTimeseries::getId() const -{ - return obj_ref_->getId(); -} - -ReportHeader & ReportTimeseries::getHeader() -{ - return *header_; -} - -void LOCAL_ReportTimeseries_writeStatisticInstValuesInTimeRange( - const uint64_t beginning_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t beginning_cycle, - const uint64_t ending_cycle, - const simdb::Blob & blob_descriptor, - const size_t num_si_values_in_blob, - const bool si_values_blob_was_compressed, - const db::MajorOrdering major_ordering, - const simdb::DatabaseID timeseries_id, - const simdb::ObjectManager & obj_mgr) -{ - assert(ending_picoseconds >= beginning_picoseconds); - assert(ending_cycle >= beginning_cycle); - - //We currently do not support indexed chunks at time values - //that are larger than int64 max. This is due to SQLite not - //supporting uint64_t out of the box. More investigation - //needs to go into how we can index chunks against uint64_t's. - constexpr uint64_t max_time_t = std::numeric_limits::max(); - if (ending_picoseconds > max_time_t || ending_cycle > max_time_t) { - throw simdb::DBException("Simulation database does not currently accept SI ") - << "chunks at time values larger than int64_t max. " - << "This error occured at " << ending_picoseconds - << " simulated picoseconds (" << ending_cycle << " root clock cycles)."; - } - - obj_mgr.safeTransaction([&]() { - //Create a new chunk record - std::unique_ptr chunk_tbl = obj_mgr.getTable("TimeseriesChunk"); - std::unique_ptr chunk_ref = chunk_tbl->createObject(); - - //Tell that record which timeseries it belongs to (foreign key) - chunk_ref->setPropertyInt32("TimeseriesID", timeseries_id); - - //When we are *not* using SI compression, we write each report update - //into the database in its own chunk. In that case, StartPS==EndPS - //and StartCycle==EndCycle - // - //However, if we *are* using SI compression, then multiple report - //updates will be buffered in memory for a certain amount of time. - //When it is time to go to the database, the start/end ps/cycle - //values may look like this: - // - // Update #1 Occurred at start = 100ps / 600 cycles - // Update #2 Occurred at start = 200ps / 1200 cycles - // Update #3 Occurred at start = 300ps / 1800 cycles - // **send to the database now** - // -> beginning_picoseconds = 100 - // -> ending_picoseconds = 300 - // -> beginning_cycle = 600 - // -> ending_cycle = 1800 - chunk_ref->setPropertyUInt64("StartPS", beginning_picoseconds); - chunk_ref->setPropertyUInt64("EndPS", ending_picoseconds); - chunk_ref->setPropertyUInt64("StartCycle", beginning_cycle); - chunk_ref->setPropertyUInt64("EndCycle", ending_cycle); - - //Create a new SI blob record - std::unique_ptr chunk_values_tbl = obj_mgr.getTable("StatInstValues"); - std::unique_ptr chunk_values_ref = chunk_values_tbl->createObject(); - - //Write the SI values as a raw blob to this record - chunk_values_ref->setPropertyBlob("RawBytes", blob_descriptor); - - //Tell this blob how many double SI values are in this - //blob. If compression was used for this SI blob, we will - //need this bit of information later when we go to inflate - //the blob back into a vector. - chunk_values_ref->setPropertyInt32("NumPts", num_si_values_in_blob); - - //Tell this blob if it was compressed or not. If so, it will - //need to send the raw char's through zlib when these SI's - //are read back out from the database. - chunk_values_ref->setPropertyInt32( - "WasCompressed", - si_values_blob_was_compressed ? 1 : 0); - chunk_values_ref->setPropertyInt32( - "MajorOrdering", (uint32_t)major_ordering); - - //Tell the SI blob record which TimeseriesChunk it belongs - //to (foreign key) - chunk_values_ref->setPropertyInt32("TimeseriesChunkID", chunk_ref->getId()); - }); -} - -void ReportTimeseries::writeStatisticInstValuesAtTimeT( - const uint64_t current_picoseconds, - const uint64_t current_cycle, - const std::vector & si_values, - const db::MajorOrdering major_ordering) -{ - writeStatisticInstValuesInTimeRange( - current_picoseconds, - current_picoseconds, - current_cycle, - current_cycle, - si_values, - major_ordering); -} - -void ReportTimeseries::writeCompressedStatisticInstValuesAtTimeT( - const uint64_t current_picoseconds, - const uint64_t current_cycle, - const std::vector & compressed_si_values, - const db::MajorOrdering major_ordering, - const uint32_t original_num_si_values) -{ - writeCompressedStatisticInstValuesInTimeRange( - current_picoseconds, - current_picoseconds, - current_cycle, - current_cycle, - compressed_si_values, - major_ordering, - original_num_si_values); -} - -void ReportTimeseries::writeStatisticInstValuesInTimeRange( - const uint64_t beginning_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t beginning_cycle, - const uint64_t ending_cycle, - const std::vector & si_values, - const db::MajorOrdering major_ordering) -{ - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &si_values[0]; - blob_descriptor.num_bytes = si_values.size() * sizeof(double); - - const size_t num_si_values_in_blob = si_values.size(); - const bool si_values_blob_was_compressed = false; - - const simdb::DatabaseID timeseries_id = getId(); - const simdb::ObjectManager & obj_mgr = obj_ref_->getObjectManager(); - - LOCAL_ReportTimeseries_writeStatisticInstValuesInTimeRange( - beginning_picoseconds, - ending_picoseconds, - beginning_cycle, - ending_cycle, - blob_descriptor, - num_si_values_in_blob, - si_values_blob_was_compressed, - major_ordering, - timeseries_id, - obj_mgr); -} - -void ReportTimeseries::writeCompressedStatisticInstValuesInTimeRange( - const uint64_t beginning_picoseconds, - const uint64_t ending_picoseconds, - const uint64_t beginning_cycle, - const uint64_t ending_cycle, - const std::vector & compressed_si_values, - const db::MajorOrdering major_ordering, - const uint32_t original_num_si_values) -{ - if (compressed_si_values.empty()) { - return; - } - - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &compressed_si_values[0]; - blob_descriptor.num_bytes = compressed_si_values.size(); - - const size_t num_si_values_in_blob = original_num_si_values; - const bool si_values_blob_was_compressed = true; - - const simdb::DatabaseID timeseries_id = getId(); - const simdb::ObjectManager & obj_mgr = obj_ref_->getObjectManager(); - - LOCAL_ReportTimeseries_writeStatisticInstValuesInTimeRange( - beginning_picoseconds, - ending_picoseconds, - beginning_cycle, - ending_cycle, - blob_descriptor, - num_si_values_in_blob, - si_values_blob_was_compressed, - major_ordering, - timeseries_id, - obj_mgr); -} - -//! Local helper method to get a range of SI values. Whether the -//! range is between simulated picoseconds, clock cycles, etc. -//! doesn't matter to us here, since the string command that is -//! passed in is all we need. -//! -//! This query returns a vector of blobs, where each blob (vector -//! element) contains all the SI values at one report update. -void LOCAL_getStatisticInstValuesFromPreparedSqlCommand( - const simdb::ObjectManager & obj_mgr, - const std::string & command, - std::vector> & si_values, - const uint32_t num_stat_insts) -{ - const simdb::SQLiteConnProxy * db_proxy = - dynamic_cast(obj_mgr.getDbConn()); - - if (!db_proxy) { - std::cout << " [simdb] ReportTimeseries object can only " - << "be used with SQLite databases" << std::endl; - return; - } - - obj_mgr.safeTransaction([&]() { - //Create a callback handler that will collect all - //database ID's for the chunks we are asking for - MultiSelectPropertyQuery query; - - //Run the SELECT statement - eval_sql_select(db_proxy, - command, - +[](void * callback_obj, int argc, char ** argv, char ** col_names) { - return static_cast*>(callback_obj)-> - onFound(argc, argv, col_names); - }, - &query); - - //Did we find any? It is not an error if 'No, there are - //no chunks found'. Asking for data that falls outside - //our range is not invalid, we just return nothing. - if (query.found()) { - const auto & chunk_ids = query.getPropertyValues(); - si_values.reserve(chunk_ids.size()); - - std::vector> chunk_refs; - obj_mgr.findObjects("StatInstValues", chunk_ids, chunk_refs); - assert(chunk_refs.size() == chunk_ids.size()); - - for (const auto & chunk_ref : chunk_refs) { - assert(chunk_ref != nullptr); - - const bool was_compressed = chunk_ref->getPropertyInt32("WasCompressed"); - const db::MajorOrdering major_ordering = - (db::MajorOrdering)chunk_ref->getPropertyInt32("MajorOrdering"); - const bool needs_transpose = - (major_ordering == db::MajorOrdering::COLUMN_MAJOR); - - //If compression was used, get the raw char bytes back from the - //database, inflate them using zlib, and put the final values - //into a new vector at the back of 'si_values'. - if (was_compressed) { - std::vector compressed_blob; - chunk_ref->getPropertyBlob("RawBytes", compressed_blob); - assert(!compressed_blob.empty()); - - const size_t num_si_values = chunk_ref->getPropertyInt32("NumPts"); - assert(num_si_values > 0); - - //Re-inflate the compressed SI blob - z_stream infstream; - infstream.zalloc = Z_NULL; - infstream.zfree = Z_NULL; - infstream.opaque = Z_NULL; - - //Setup the source stream - infstream.avail_in = (uInt)(compressed_blob.size()); - infstream.next_in = (Bytef*)(&compressed_blob[0]); - - //Setup the destination stream - std::vector interleaved_sis; - interleaved_sis.resize(num_si_values); - infstream.avail_out = (uInt)(interleaved_sis.size() * sizeof(double)); - infstream.next_out = (Bytef*)(&interleaved_sis[0]); - - //Inflate it! - inflateInit(&infstream); - inflate(&infstream, Z_FINISH); - inflateEnd(&infstream); - - //If this SI chunk was written in row-major format, simply rearrange - //the single vector into vector> like so: - // - // blob from database = [3, 5, 3, 4, 4, 8, 9, 7, 1] - // num SI's per update = 3 - // output SI's = [[3, 5, 3], [4, 4, 8], [9, 7, 1]] - if (!needs_transpose) { - assert(interleaved_sis.size() % num_stat_insts == 0); - const uint32_t num_report_updates_here = - interleaved_sis.size() / num_stat_insts; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(num_stat_insts); - std::vector & original_si_values = si_values.back(); - memcpy(&original_si_values[0], - &interleaved_sis[relative_update_idx * num_stat_insts], - original_si_values.size() * sizeof(double)); - } - continue; - } - - //SI blobs that are in column-major order are a little different. - //We're not done yet - we still have to rearrange the SI - //values. Instead of being interleaved like this: - // - // [SI_1a, SI_1b, SI_1c, SI_2a, SI_2b, SI_2c] - // ------------------- ------------------- - // (1) (2) <- organized by SI - // (three report updates, where SI values are - // next to their previous value) - // - //They need to be back in their original order like this: - // - // [SI_1a, SI_2a, SI_1b, SI_2b, SI_1c, SI_2c] - // - // ------------ ------------ ------------ - // (1) (2) (3) <- organized by - // (three report updates, where SI values report update - // are next to the SI's that are adjacent - // to them in the physical tree when traversed - // depth-first) - assert(interleaved_sis.size() % num_stat_insts == 0); - const uint32_t num_report_updates_here = interleaved_sis.size() / num_stat_insts; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(num_stat_insts); - std::vector & original_si_values = si_values.back(); - - //Perform the column-major-to-row-major transpose. This is the - //last step to get back the original SI values from a compressed, - //column-major ordered dataset. - uint32_t read_idx = relative_update_idx; - for (uint32_t write_idx = 0; write_idx < num_stat_insts; ++write_idx) { - original_si_values[write_idx] = interleaved_sis[read_idx]; - read_idx += num_report_updates_here; - } - } - } - - //No compression for this blob - just ask for the blob raw bytes - //and deinterleave them into their original SI ordering. - else { - std::vector interleaved_sis; - chunk_ref->getPropertyBlob("RawBytes", interleaved_sis); - assert(interleaved_sis.size() % num_stat_insts == 0); - const uint32_t num_report_updates_here = - interleaved_sis.size() / num_stat_insts; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(std::vector()); - std::vector & original_si_values = si_values.back(); - original_si_values.resize(num_stat_insts); - - if (!needs_transpose) { - memcpy(&original_si_values[0], - &interleaved_sis[relative_update_idx * num_stat_insts], - original_si_values.size() * sizeof(double)); - continue; - } - - //Continuing with the example from above, we might now - //have a original_si_values vector sized to 2 elements - //ready to go: - // - // original_si_values = [---, ---] - uint32_t read_idx = relative_update_idx; - for (uint32_t write_idx = 0; write_idx < num_stat_insts; /*see below*/) { - original_si_values[write_idx] = interleaved_sis[read_idx]; - - //Advance the read and write indices - read_idx += num_report_updates_here; - ++write_idx; - } - } - } - } - } - - //Clear and shrink the results just to be safe - else { - si_values.clear(); - si_values.shrink_to_fit(); - } - }); -} - -//! Retrieve all SI data value chunks between "time points" A and B: -//! - between simulated time values A and B (picoseconds) -void ReportTimeseries::getStatisticInstValuesBetweenSimulatedPicoseconds( - const uint64_t start_picoseconds, - const uint64_t end_picoseconds, - std::vector> & si_values) -{ - const auto & obj_mgr = obj_ref_->getObjectManager(); - const std::string table_name = obj_mgr.getQualifiedTableName( - "TimeseriesChunk", "Stats"); - - std::ostringstream oss; - oss << " SELECT Id FROM " << table_name << " WHERE " - << start_picoseconds << " <= StartPS AND " - << end_picoseconds << " >= EndPS AND " - << "TimeseriesID == " << getId(); - - const std::string command = oss.str(); - - //TODO: This piece of metadata is needed to decompress SI data. - //Find another way to decompress blobs without requiring this. - const uint32_t num_stat_insts = - getHeader().getObjectRef().getPropertyInt32("NumStatInsts"); - - LOCAL_getStatisticInstValuesFromPreparedSqlCommand( - obj_mgr, command, si_values, num_stat_insts); -} - -//! Retrieve all SI data value chunks between "time points" A and B: -//! - between root clock cycles A and B -void ReportTimeseries::getStatisticInstValuesBetweenRootClockCycles( - const uint64_t start_cycle, - const uint64_t end_cycle, - std::vector> & si_values) -{ - const auto & obj_mgr = obj_ref_->getObjectManager(); - const std::string table_name = obj_mgr.getQualifiedTableName( - "TimeseriesChunk", "Stats"); - - std::ostringstream oss; - oss << " SELECT Id FROM " << table_name << " WHERE " - << start_cycle << " <= StartCycle AND " - << end_cycle << " >= EndCycle AND " - << "TimeseriesID == " << getId(); - - const std::string command = oss.str(); - - //TODO: This piece of metadata is needed to decompress SI data. - //Find another way to decompress blobs without requiring this. - const uint32_t num_stat_insts = - getHeader().getObjectRef().getPropertyInt32("NumStatInsts"); - - LOCAL_getStatisticInstValuesFromPreparedSqlCommand( - obj_mgr, command, si_values, num_stat_insts); -} - -//! Implementation class for RangeIterator. This class allows -//! us to incrementally process SI blobs across an arbitrarily -//! large dataset without needing to bring the whole dataset -//! into memory. -class ReportTimeseries::RangeIterator::Impl -{ -public: - explicit Impl(ReportTimeseries & db_timeseries) : - db_timeseries_(db_timeseries) - {} - - //! Setup the database cursor at the beginning of - //! a specified range in simulated picoseconds. - bool positionRangeAroundSimulatedPicoseconds( - const uint64_t start_picoseconds, - const uint64_t end_picoseconds) - { - const auto & obj_mgr = db_timeseries_.getHeader(). - getObjectRef().getObjectManager(); - - const std::string table_name = obj_mgr.getQualifiedTableName( - "TimeseriesChunk", "Stats"); - - std::ostringstream oss; - oss << " SELECT Id FROM " << table_name << " WHERE " - << start_picoseconds << " <= StartPS AND " - << end_picoseconds << " >= EndPS AND " - << "TimeseriesID == " << db_timeseries_.getId(); - - const std::string command = oss.str(); - return runQueryAndPositionRangeIterator_(command); - } - - //! Setup the database cursor at the beginning of - //! a specified range in root clock cycles. - bool positionRangeAroundRootClockCycles( - const uint64_t start_cycle, - const uint64_t end_cycle) - { - const auto & obj_mgr = db_timeseries_.getHeader(). - getObjectRef().getObjectManager(); - - const std::string table_name = obj_mgr.getQualifiedTableName( - "TimeseriesChunk", "Stats"); - - std::ostringstream oss; - oss << " SELECT Id FROM " << table_name << " WHERE " - << start_cycle << " <= StartCycle AND " - << end_cycle << " >= EndCycle AND " - << "TimeseriesID == " << db_timeseries_.getId(); - - const std::string command = oss.str(); - return runQueryAndPositionRangeIterator_(command); - } - - //! Advance the iterator to the next set of values. Return - //! true if successful. - bool getNext() - { - if (iterator_ == nullptr) { - return false; - } - - ++current_deserialized_vector_idx_; - if (current_deserialized_vector_idx_ >= deserialized_si_values_.size()) { - //! We've exhausted the blob we had in memory. Now we - //! need to advance the DB iterator, which will bring - //! in the next SI blob. Note this is not necessarily - //! one SI *row* (update) - a blob can contain many - //! SI rows. - const bool valid_advance = iterator_->getNext(); - if (valid_advance) { - //! We have a new blob with one or more SI rows in - //! it. Deserialize it and reset our read index - //! back to zero. Say this blob has three rows' - //! worth of SI data in it. Our "read index" goes - //! from 0, to 1, to 2, and then back to zero since - //! we'll have used up this three-row blob. - deserializeCurrentBlob_(); - current_deserialized_vector_idx_ = 0; - return true; - } else { - //! No more blobs, or the DB iterator failed for - //! some other reason. - current_deserialized_vector_idx_ = - std::numeric_limits::max(); - return false; - } - } - return true; - } - - //! Get a pointer to the current SI range's data values. - const double * getCurrentSliceDataValuesPtr() const - { - if (current_deserialized_vector_idx_ >= deserialized_si_values_.size()) { - return nullptr; - } - - const std::vector & values = - deserialized_si_values_[current_deserialized_vector_idx_]; - - return !values.empty() ? &values[0] : nullptr; - } - - //! Get the number of SI data points in the current slice. - //! This is how many points you can read off of the returned - //! pointer from getCurrentSliceDataValuesPtr() - size_t getCurrentSliceNumDataValues() const - { - if (current_deserialized_vector_idx_ >= deserialized_si_values_.size()) { - return 0; - } - - const std::vector & values = - deserialized_si_values_[current_deserialized_vector_idx_]; - - return values.size(); - } - -private: - //! Setup the database cursor at the beginning of - //! a specified range. The input string is a SQL - //! statement we've already computed. - bool runQueryAndPositionRangeIterator_(const std::string & command) - { - const simdb::ObjectRef & header_ref = db_timeseries_.getHeader().getObjectRef(); - const simdb::ObjectManager & obj_mgr = header_ref.getObjectManager(); - - const simdb::SQLiteConnProxy * db_proxy = - dynamic_cast(obj_mgr.getDbConn()); - - if (!db_proxy) { - std::cout << " [simdb] ReportTimeseries object can only " - << "be used with SQLite databases" << std::endl; - return false; - } - - num_stat_insts_ = header_ref.getPropertyInt32("NumStatInsts"); - - struct FindHelper { - explicit FindHelper(std::vector & chunk_ids) : - chunk_ids(chunk_ids) - {} - - int onFound(int argc, char ** argv, char ** col_names) { - assert(argc == 1); - assert(strcmp(col_names[0], "Id") == 0); - chunk_ids.emplace_back(atoi(argv[0])); - return SQLITE_OK; - } - - std::vector & chunk_ids; - }; - - //! We run two queries to set up this database cursor: - //! - First query finds all chunk database IDs we need - //! - The second uses ObjectQuery to give us a cursor - //! at the beginning of the StatInstValues rows that - //! we are looking for (but it does so without reading - //! any of those rows' blobs into memory). - FindHelper finder(chunk_ids_); - eval_sql_select(db_proxy, command, - +[](void * caller, int argc, char ** argv, char ** col_names) { - return static_cast(caller)->onFound( - argc, argv, col_names); - }, - &finder); - - iterator_.reset(); - if (chunk_ids_.empty()) { - return false; - } - - //SELECT RawBytes,NumPts,WasCompressed,MajorOrdering - //FROM StatInstValues - //WHERE TimeseriesChunkID IN (12,15,88,106) - // ************** - // chunk_ids_ - // - simdb::ObjectQuery query(obj_mgr, "StatInstValues"); - - query.addConstraints("TimeseriesChunkID", - simdb::constraints::in_set, - chunk_ids_); - - query.writeResultIterationsTo("RawBytes", &raw_bytes_, - "NumPts", &num_pts_, - "WasCompressed", &was_compressed_, - "MajorOrdering", &major_ordering_); - - iterator_ = query.executeQuery(); - return (iterator_ != nullptr); - } - - //! SI blobs are stored in one of a number of configurations. - //! These include compression on or off, and row-major vs. - //! column-major SI ordering. This method puts SI blobs - //! back into a "human readable" vector of doubles: the - //! row-major raw double values that originally came from - //! the simulation/report. - void deserializeCurrentBlob_() - { - deserialized_si_values_.clear(); - std::vector> & si_values = deserialized_si_values_; - const bool needs_transpose = - (major_ordering_ == (uint32_t)db::MajorOrdering::COLUMN_MAJOR); - - //If compression was used, get the raw char bytes back from the - //database, inflate them using zlib, and put the final values - //into a new vector at the back of 'si_values'. - if (was_compressed_) { - std::vector & compressed_blob = raw_bytes_; - const size_t num_si_values = num_pts_; - assert(num_si_values > 0); - - //Re-inflate the compressed SI blob - z_stream infstream; - infstream.zalloc = Z_NULL; - infstream.zfree = Z_NULL; - infstream.opaque = Z_NULL; - - //Setup the source stream - infstream.avail_in = (uInt)(compressed_blob.size()); - infstream.next_in = (Bytef*)(&compressed_blob[0]); - - //Setup the destination stream - std::vector interleaved_sis; - interleaved_sis.resize(num_si_values); - infstream.avail_out = (uInt)(interleaved_sis.size() * sizeof(double)); - infstream.next_out = (Bytef*)(&interleaved_sis[0]); - - //Inflate it! - inflateInit(&infstream); - inflate(&infstream, Z_FINISH); - inflateEnd(&infstream); - - //If this SI chunk was written in row-major format, simply rearrange - //the single vector into vector> like so: - // - // blob from database = [3, 5, 3, 4, 4, 8, 9, 7, 1] - // num SI's per update = 3 - // output SI's = [[3, 5, 3], [4, 4, 8], [9, 7, 1]] - if (!needs_transpose) { - assert(interleaved_sis.size() % num_stat_insts_ == 0); - const uint32_t num_report_updates_here = - interleaved_sis.size() / num_stat_insts_; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(num_stat_insts_); - std::vector & original_si_values = si_values.back(); - memcpy(&original_si_values[0], - &interleaved_sis[relative_update_idx * num_stat_insts_], - original_si_values.size() * sizeof(double)); - } - //! Nothing left to do for blobs in row-major order... - return; - } - //!...but column-major ordering is a little different... - - //We're not done yet - we still have to rearrange the SI - //values. Instead of being interleaved like this: - // - // [SI_1a, SI_1b, SI_1c, SI_2a, SI_2b, SI_2c] - // ------------------- ------------------- - // (1) (2) <- organized by SI - // (three report updates, where SI values are - // next to their previous value) - // - //They need to be back in their original order like this: - // - // [SI_1a, SI_2a, SI_1b, SI_2b, SI_1c, SI_2c] - // - // ------------ ------------ ------------ - // (1) (2) (3) <- organized by - // (three report updates, where SI values report update - // are next to the SI's that are adjacent - // to them in the physical tree when traversed - // depth-first) - assert(interleaved_sis.size() % num_stat_insts_ == 0); - const uint32_t num_report_updates_here = - interleaved_sis.size() / num_stat_insts_; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(num_stat_insts_); - std::vector & original_si_values = si_values.back(); - - //Continuing with the example from above, we might now - //have a original_si_values vector sized to 2 elements - //ready to go: - // - // original_si_values = [---, ---] - uint32_t read_idx = relative_update_idx; - for (uint32_t write_idx = 0; write_idx < num_stat_insts_; ++write_idx) { - original_si_values[write_idx] = interleaved_sis[read_idx]; - read_idx += num_report_updates_here; - } - } - } - - //No compression for this blob - just ask for the blob raw bytes - //and deinterleave them into their original SI ordering. - else { - const simdb::VectorAlias interleaved_sis(raw_bytes_); - assert(interleaved_sis.size() % num_stat_insts_ == 0); - const uint32_t num_report_updates_here = - interleaved_sis.size() / num_stat_insts_; - - for (uint32_t relative_update_idx = 0; - relative_update_idx < num_report_updates_here; - ++relative_update_idx) - { - si_values.emplace_back(num_stat_insts_); - std::vector & original_si_values = si_values.back(); - - if (!needs_transpose) { - memcpy(&original_si_values[0], - &interleaved_sis[relative_update_idx * num_stat_insts_], - original_si_values.size() * sizeof(double)); - continue; - } - - //Continuing with the example from above, we might now - //have a original_si_values vector sized to 2 elements - //ready to go: - // - // original_si_values = [---, ---] - uint32_t read_idx = relative_update_idx; - for (uint32_t write_idx = 0; write_idx < num_stat_insts_; ++write_idx) { - original_si_values[write_idx] = interleaved_sis[read_idx]; - read_idx += num_report_updates_here; - } - } - } - } - - ReportTimeseries & db_timeseries_; - std::vector chunk_ids_; - std::unique_ptr iterator_; - - std::vector raw_bytes_; - std::vector> deserialized_si_values_; - - int32_t num_pts_ = 0; - uint32_t num_stat_insts_ = 0; - int32_t major_ordering_ = (int32_t)db::MajorOrdering::ROW_MAJOR; - uint32_t current_deserialized_vector_idx_ = 0; - - //In C++, the "was compressed or not" variable would be a - //bool, but sqlite does not support boolean data types. - //It is stored in the database as an unsigned int, so this - //variable here is an int32_t to match what the database - //query will return. - int32_t was_compressed_ = true; -}; - -ReportTimeseries::RangeIterator::RangeIterator(ReportTimeseries & db_timeseries) : - impl_(new ReportTimeseries::RangeIterator::Impl(db_timeseries)) -{ -} - -void ReportTimeseries::RangeIterator::positionRangeAroundSimulatedPicoseconds( - const uint64_t start_picoseconds, - const uint64_t end_picoseconds) -{ - impl_->positionRangeAroundSimulatedPicoseconds( - start_picoseconds, end_picoseconds); -} - -void ReportTimeseries::RangeIterator::positionRangeAroundRootClockCycles( - const uint64_t start_cycle, - const uint64_t end_cycle) -{ - impl_->positionRangeAroundRootClockCycles( - start_cycle, end_cycle); -} - -bool ReportTimeseries::RangeIterator::getNext() -{ - return impl_->getNext(); -} - -const double * ReportTimeseries::RangeIterator::getCurrentSliceDataValuesPtr() const -{ - return impl_->getCurrentSliceDataValuesPtr(); -} - -size_t ReportTimeseries::RangeIterator::getCurrentSliceNumDataValues() const -{ - return impl_->getCurrentSliceNumDataValues(); -} - -} // namespace db -} // namespace sparta diff --git a/sparta/src/ReportVerifier.cpp b/sparta/src/ReportVerifier.cpp deleted file mode 100644 index 62be887767..0000000000 --- a/sparta/src/ReportVerifier.cpp +++ /dev/null @@ -1,582 +0,0 @@ -// -*- C++ -*- - -#include "sparta/report/db/ReportVerifier.hpp" - -#include -#include -#include -#include -#include - -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/utils/uuids.hpp" -#include "sparta/app/ReportDescriptor.hpp" -#include "sparta/report/format/BaseFormatter.hpp" -#include "sparta/report/db/format/toCSV.hpp" -#include "sparta/report/Report.hpp" -#include "sparta/utils/SpartaTester.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/schema/DatabaseTypedefs.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "sparta/report/db/ReportTimeseries.hpp" - -namespace sparta { -namespace db { - -//! Static initializations -std::string ReportVerifier::verif_results_dir_ = "AccuracyCheckedDBs"; -bool ReportVerifier::verif_results_dir_is_changeable_ = true; - -//! Ask for the verification results directory. It could -//! be the default "AccuracyCheckedDBs" or it could have -//! been changed via writeVerifResultsTo(). -const std::string & ReportVerifier::getVerifResultsDir() -{ - sparta_assert(!ReportVerifier::verif_results_dir_.empty()); - return ReportVerifier::verif_results_dir_; -} - -//! Redirect the verification artifacts to be written -//! somewhere other than the REPORT_VERIF_DEFAULT_DIR. -//! This throws an exception if the post-simulation -//! verification process has already begun. -void ReportVerifier::writeVerifResultsTo(const std::string & dir) -{ - if (dir.empty()) { - std::cout << " [simdb] ReportVerifier::writeVerifResultsTo() called " - << "with an empty directory string. This will be ignored." - << std::endl; - return; - } - - if (!ReportVerifier::verif_results_dir_is_changeable_ && - dir != ReportVerifier::getVerifResultsDir()) - { - throw SpartaException("Redirecting the report verification ") - << "artifacts directory is disallowed after the verification " - << "process has already begun."; - } else if (dir == ReportVerifier::getVerifResultsDir()) { - return; - } - - //For testing purposes, we may assign a verification - //directory of "@", which means that we should create - //a randomly-generated directory name. - if (dir == "@") { - std::ostringstream oss; - oss << simdb::generateUUID(); - ReportVerifier::verif_results_dir_ = oss.str(); - } else { - ReportVerifier::verif_results_dir_ = dir; - } -} - -//! Turn the provided timeseries report file (baseline) into -//! an equivalent report file from the appropriate database -//! records. Use a SpartaTester to check for file differences, -//! and give those differences back to the caller in the -//! "failure_differences" output argument. -bool verifyTimeseriesReport( - const std::string & yaml_dest_file, - const std::string & simdb_dest_file, - const simdb::DatabaseID timeseries_id, - const simdb::ObjectManager & sim_db, - std::string & failure_differences) -{ - std::unique_ptr timeseries_ref = - sim_db.findObject("Timeseries", timeseries_id); - - if (timeseries_ref == nullptr) { - throw SpartaException("Unable to locate 'Timeseries' record with Id ") - << timeseries_id; - } - - ReportTimeseries timeseries(std::move(timeseries_ref)); - format::toCSV(×eries, simdb_dest_file); - - std::ostringstream cerr; - std::unique_ptr tester = - SpartaTester::makeTesterWithUserCError(cerr); - - tester->expectFilesEqual( - yaml_dest_file, simdb_dest_file, - true, __LINE__, __FILE__, false); - - failure_differences = cerr.str(); - if (!failure_differences.empty()) { - std::ifstream fin_expected(yaml_dest_file); - const std::string contents_expected{ - std::istreambuf_iterator{fin_expected}, {}}; - - std::ifstream fin_actual(simdb_dest_file); - const std::string contents_actual{ - std::istreambuf_iterator{fin_actual}, {}}; - - //Until we can be 100% sure that our SimInfo table has enough information - //in it to exactly reproduce verification failures, we'll just capture a - //deep copy of the two files (baseline and SimDB-generated) and stash them - //back in the database. This verification step is disabled in production - //simulators. - auto failed_deep_copy_tbl = sim_db.getTable("ReportVerificationDeepCopyFiles"); - - failed_deep_copy_tbl->createObjectWithArgs( - "DestFile", yaml_dest_file, - "Expected", contents_expected, - "Actual", contents_actual); - } - - return failure_differences.empty(); -} - -//! Turn the provided non-timeseries report file e.g. json, -//! txt, html, etc. into an equivalent report file from the -//! appropriate database records. Use a SpartaTester to check -//! for file differences, and give those differences back to -//! the caller in the "failure_differences" output argument. -bool verifyNonTimeseriesReport( - const std::string & yaml_dest_file, - const std::string & simdb_dest_file, - const std::string & format, - const simdb::DatabaseID report_id, - const simdb::ObjectManager & sim_db, - const Scheduler * scheduler, - std::string & failure_differences) -{ - if (!Report::createFormattedReportFromDatabase( - sim_db, report_id, simdb_dest_file, format, scheduler)) - { - throw SpartaException("Unable to create report from SimDB: \n") - << "\tdest_file: " << yaml_dest_file << "\n" - << "\tformat: " << format << "\n" - << "\treport_id: " << report_id; - } - - std::ostringstream cerr; - std::unique_ptr tester = - SpartaTester::makeTesterWithUserCError(cerr); - - tester->expectFilesEqual( - yaml_dest_file, simdb_dest_file, - true, __LINE__, __FILE__, false); - - failure_differences = cerr.str(); - if (!failure_differences.empty()) { - std::ifstream fin_expected(yaml_dest_file); - const std::string contents_expected{ - std::istreambuf_iterator{fin_expected}, {}}; - - std::ifstream fin_actual(simdb_dest_file); - const std::string contents_actual{ - std::istreambuf_iterator{fin_actual}, {}}; - - //Until we can be 100% sure that our SimInfo table has enough information - //in it to exactly reproduce verification failures, we'll just capture a - //deep copy of the two files (baseline and SimDB-generated) and stash them - //back in the database. This verification step is disabled in production - //simulators. - auto failed_deep_copy_tbl = sim_db.getTable("ReportVerificationDeepCopyFiles"); - - failed_deep_copy_tbl->createObjectWithArgs( - "DestFile", yaml_dest_file, - "Expected", contents_expected, - "Actual", contents_actual); - } - - return failure_differences.empty(); -} - -/*! - * \brief VerificationSummary implementation class - */ -class ReportVerifier::VerificationSummary::Impl -{ -public: - explicit Impl(const simdb::ObjectManager & sim_db) : - sim_db_(sim_db) - {} - - void setManagledDescriptorDefFilesToSimDbDefFiles( - std::map && mangled_fnames) - { - desc_dest_files_to_simdb_dest_files_ = std::move(mangled_fnames); - } - - void setMangledDescriptorDefFilesToYamlDestFiles( - std::map && mangled_fnames) - { - desc_dest_files_to_yaml_dest_files_ = std::move(mangled_fnames); - } - - const std::map & - getMangledDescriptorDefFilesToYamlDestFiles() const - { - return desc_dest_files_to_yaml_dest_files_; - } - - bool hasSummary() const - { - return !pass_fail_by_filename_.empty(); - } - - std::set getPassingReportFilenames() const - { - std::set passing; - for (const auto & pf : pass_fail_by_filename_) { - if (pf.second) { - passing.insert(pf.first); - } - } - return passing; - } - - std::set getFailingReportFilenames() const - { - std::set failing; - for (const auto & pf : pass_fail_by_filename_) { - if (!pf.second) { - failing.insert(pf.first); - } - } - return failing; - } - - bool reportIsTimeseries(const std::string & filename) const - { - if (tested_timeseries_report_filenames_.count(filename) > 0) { - return true; - } - if (tested_non_timeseries_report_filenames_.count(filename) == 0) { - throw SpartaException("Unrecognized filename given to a VerificationSummary ('") - << filename << "')"; - } - return false; - } - - std::string getFailureDifferences(const std::string & filename) const - { - auto iter = failure_diffs_by_filename_.find(filename); - return (iter != failure_diffs_by_filename_.end() ? iter->second : ""); - } - - void serializeSummary( - const simdb::ObjectManager & sim_db) const - { - sim_db.safeTransaction([&]() { - simdb::DatabaseID sim_info_id = 0; - simdb::ObjectQuery sim_info_query(sim_db, "SimInfo"); - sim_info_query.writeResultIterationsTo("Id", &sim_info_id); - - //Until there is a stronger link between simulations - //and the contents of the SimInfo table, we'll just - //see if this table has exactly one record in it. - //Until multiple simulations all feed the same SimDB - //file, SimInfo will only have one record (or none). - auto result_iter = sim_info_query.executeQuery(); - if (!result_iter->getNext() || result_iter->getNext()) { - sim_info_id = 0; - } - - auto verif_tbl = sim_db.getTable("ReportVerificationResults"); - const auto passing_files = getPassingReportFilenames(); - for (const auto & passing : passing_files) { - const std::string & dest_file = passing; - const bool is_timeseries = reportIsTimeseries(dest_file); - - verif_tbl->createObjectWithArgs( - "DestFile", dest_file, - "SimInfoID", sim_info_id, - "Passed", 1, - "IsTimeseries", (int)is_timeseries); - } - - auto failure_summary_tbl = sim_db.getTable("ReportVerificationFailureSummaries"); - const auto failing_files = getFailingReportFilenames(); - for (const auto & failing : failing_files) { - const std::string & dest_file = failing; - const bool is_timeseries = reportIsTimeseries(dest_file); - const std::string failure_diffs = getFailureDifferences(dest_file); - - auto failure_obj = verif_tbl->createObjectWithArgs( - "DestFile", dest_file, - "SimInfoID", sim_info_id, - "Passed", 0, - "IsTimeseries", (int)is_timeseries); - - failure_summary_tbl->createObjectWithArgs( - "ReportVerificationResultID", failure_obj->getId(), - "FailureSummary", failure_diffs); - } - }); - } - - std::map getFinalDestFiles() const - { - std::map simdb_to_yaml_dest_files; - for (const auto & mangled : desc_dest_files_to_simdb_dest_files_) { - const std::string & mangled_desc_dest_file = mangled.first; - const std::string & simdb_dest_file = mangled.second; - - auto mangled_iter = desc_dest_files_to_yaml_dest_files_.find( - mangled_desc_dest_file); - sparta_assert(mangled_iter != desc_dest_files_to_yaml_dest_files_.end()); - const std::string & yaml_dest_file = mangled_iter->second; - - simdb_to_yaml_dest_files[simdb_dest_file] = yaml_dest_file; - } - - //Trim away any leading '/' characters. They trip up - //the std::filesystem::copy_file() calls. - std::map trimmed_fnames; - for (auto & simdb_to_yaml : simdb_to_yaml_dest_files) { - namespace sfs = std::filesystem; - sfs::path p(simdb_to_yaml.second); - std::string to; - - auto parent_path = p.parent_path(); - if (!sfs::is_directory(parent_path)) { - auto not_slash = simdb_to_yaml.second.find_first_not_of("/"); - if (not_slash != std::string::npos) { - to = simdb_to_yaml.second.substr(not_slash); - } else { - //We could warn or throw, but the calling code - //is not going to make the copy_file() call - //without some adjustments to the dest_file - //name. - to = simdb_to_yaml.second; - } - } else { - to = simdb_to_yaml.second; - } - trimmed_fnames[simdb_to_yaml.first] = to; - } - - return trimmed_fnames; - } - - bool verifyReport(const std::string & filename, - const Scheduler * scheduler) - { - //Until the timeseries and non-timeseries backend is - //more streamlined, we need to run separate queries to infer - //which type of report this is. The code path for database- - //regenerated report files is currently quite different for - //a timeseries report and every other type of report. - simdb::ObjectQuery timeseries_query(sim_db_, "ReportHeader"); - simdb::DatabaseID timeseries_id = 0; - std::string desc_dest_file; - - timeseries_query.addConstraints( - "DestFile", simdb::constraints::equal, filename); - - timeseries_query.writeResultIterationsTo( - "TimeseriesID", ×eries_id, - "DestFile", &desc_dest_file); - - auto result_iter = timeseries_query.executeQuery(); - if (result_iter->getNext()) { - //This dest_file is a timeseries report. - auto mangled_iter = desc_dest_files_to_simdb_dest_files_.find( - desc_dest_file); - sparta_assert(mangled_iter != desc_dest_files_to_simdb_dest_files_.end()); - const std::string & simdb_dest_file = mangled_iter->second; - - std::string failure_differences; - const bool passed = verifyTimeseriesReport( - desc_dest_file, simdb_dest_file, - timeseries_id, sim_db_, - failure_differences); - - //Store some pass/fail metadata for later and return. - pass_fail_by_filename_[desc_dest_file] = passed; - tested_timeseries_report_filenames_.insert(desc_dest_file); - if (!passed) { - failure_diffs_by_filename_[filename] = failure_differences; - } - return passed; - } - - simdb::ObjectQuery non_timeseries_query( - sim_db_, "ReportVerificationMetadata"); - - simdb::DatabaseID report_id = 0; - std::string format; - - non_timeseries_query.addConstraints( - "DestFile", simdb::constraints::equal, filename); - - non_timeseries_query.writeResultIterationsTo( - "RootReportNodeID", &report_id, - "DestFile", &desc_dest_file, - "Format", &format); - - result_iter = non_timeseries_query.executeQuery(); - if (result_iter->getNext()) { - //This dest_file is a non-timeseries report. - auto mangled_iter = desc_dest_files_to_simdb_dest_files_.find( - desc_dest_file); - sparta_assert(mangled_iter != desc_dest_files_to_simdb_dest_files_.end()); - const std::string & simdb_dest_file = mangled_iter->second; - - std::string failure_differences; - const bool passed = verifyNonTimeseriesReport( - desc_dest_file, simdb_dest_file, - format, report_id, sim_db_, - scheduler, - failure_differences); - - //Store some pass/fail metadata for later and return. - pass_fail_by_filename_[desc_dest_file] = passed; - tested_non_timeseries_report_filenames_.insert(desc_dest_file); - if (!passed) { - failure_diffs_by_filename_[filename] = failure_differences; - } - return passed; - } - - //If the dest_file was not found to be a timeseries - //or a non-timeseries in this database, that means - //the report is not even in the database (or it got - //"lost" due to something like invalid foreign keys). - throw SpartaException("Unable to run report verification for file '") - << desc_dest_file << "'. File not found in the database (" - << sim_db_.getDatabaseFile() << ")."; - } - -private: - const simdb::ObjectManager & sim_db_; - std::map pass_fail_by_filename_; - std::set tested_timeseries_report_filenames_; - std::set tested_non_timeseries_report_filenames_; - std::map failure_diffs_by_filename_; - std::map desc_dest_files_to_simdb_dest_files_; - std::map desc_dest_files_to_yaml_dest_files_; -}; - -ReportVerifier::VerificationSummary::VerificationSummary( - const simdb::ObjectManager & sim_db) : - impl_(new ReportVerifier::VerificationSummary::Impl(sim_db)) -{ -} - -bool ReportVerifier::VerificationSummary::hasSummary() const -{ - return impl_->hasSummary(); -} - -std::set ReportVerifier::VerificationSummary::getPassingReportFilenames() const -{ - return impl_->getPassingReportFilenames(); -} - -std::set ReportVerifier::VerificationSummary::getFailingReportFilenames() const -{ - return impl_->getFailingReportFilenames(); -} - -bool ReportVerifier::VerificationSummary::reportIsTimeseries( - const std::string & filename) const -{ - return impl_->reportIsTimeseries(filename); -} - -std::string ReportVerifier::VerificationSummary::getFailureDifferences( - const std::string & filename) const -{ - return impl_->getFailureDifferences(filename); -} - -void ReportVerifier::VerificationSummary::serializeSummary( - const simdb::ObjectManager & sim_db) const -{ - impl_->serializeSummary(sim_db); -} - -std::map - ReportVerifier::VerificationSummary::getFinalDestFiles() const -{ - return impl_->getFinalDestFiles(); -} - -bool ReportVerifier::VerificationSummary::verifyReport_(const std::string & filename, - const Scheduler * scheduler) -{ - return impl_->verifyReport(filename, scheduler); -} - -void ReportVerifier::addReportToVerify(const app::ReportDescriptor & rd) -{ - to_verify_[rd.dest_file] = rd.getDescriptorOrigDestFile(); -} - -void ReportVerifier::addBaseFormatterForPreVerificationReset( - const std::string & filename, - report::format::BaseFormatter * formatter) -{ - formatters_[filename] = formatter; -} - -std::unique_ptr ReportVerifier::verifyAll( - const simdb::ObjectManager & sim_db, - const Scheduler * scheduler) -{ - //Immediately lock down the verification artifacts directory - //from changes for the rest of the program. - sparta_assert(!ReportVerifier::getVerifResultsDir().empty()); - ReportVerifier::verif_results_dir_is_changeable_ = false; - - //Since we typically use report verification in regression - //testing, we'll have many simulations concurrently writing - //to the filesystem. They could have identical dest_file's - //which could trip up the verification code (use the wrong - //dest_file as the baseline and get sporadic failures). - // - //We'll add a random uuid onto each dest_file to help prevent - //this from happening, or at least make it extremely unlikely. - std::map mangled_filenames; - for (const auto & file : to_verify_) { - std::ostringstream random_file_suffix; - random_file_suffix << simdb::generateUUID(); - mangled_filenames[file.first] = - file.first + "_" + random_file_suffix.str(); - } - - std::unique_ptr summary( - new VerificationSummary(sim_db)); - - if (to_verify_.empty()) { - return summary; - } - - summary->impl_->setManagledDescriptorDefFilesToSimDbDefFiles( - std::move(mangled_filenames)); - - summary->impl_->setMangledDescriptorDefFilesToYamlDestFiles( - std::move(to_verify_)); - - sim_db.safeTransaction([&]() { - const auto & desc_dest_files_to_yaml_dest_files = - summary->impl_->getMangledDescriptorDefFilesToYamlDestFiles(); - - for (const auto & file : desc_dest_files_to_yaml_dest_files) { - if (file.first == "1") { - std::cout << " [simdb] Skipping report validation check for " - << "std::cout report (dest_file: \"1\")\n"; - continue; - } - auto formatter_iter = formatters_.find(file.first); - if (formatter_iter != formatters_.end()) { - formatter_iter->second->doPostProcessingBeforeReportValidation(); - } - summary->verifyReport_(file.first, scheduler); - } - }); - - return summary; -} - -} // namespace db -} // namespace sparta diff --git a/sparta/src/SINodeHierarchy.cpp b/sparta/src/SINodeHierarchy.cpp deleted file mode 100644 index ad3d438e48..0000000000 --- a/sparta/src/SINodeHierarchy.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// -*- C++ -*- - -#include "sparta/statistics/db/SINodeHierarchy.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/report/Report.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "sparta/statistics/StatisticInstance.hpp" - -namespace sparta { -namespace statistics { - -//! Create a database record in the SINodeHierarchy table. -//! This record describes one SI node and its place in the -//! larger report it belongs to. -//! -//! Call SINodeCreator::createLeafSINode() for StatisticInstance -//! nodes at the leaves of an SI/Report tree. -//! -//! Call SINodeCreator::createMidLevelSINode() for mid-level and -//! top-level nodes of an SI/Report tree. -class SINodeCreator -{ -public: - static std::unique_ptr createLeafSINode( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID timeseries_id, - const simdb::DatabaseID parent_node_id, - const std::string & si_node_name, - int & si_relative_index) - { - auto leaf = SINodeCreator::createSINode_( - obj_mgr, timeseries_id, parent_node_id, - si_node_name, si_relative_index); - - ++si_relative_index; - return leaf; - } - - static std::unique_ptr createMidLevelSINode( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID timeseries_id, - const simdb::DatabaseID parent_node_id, - const std::string & si_node_name, - const int si_relative_index) - { - return SINodeCreator::createSINode_( - obj_mgr, timeseries_id, parent_node_id, - si_node_name, si_relative_index); - } - -private: - static std::unique_ptr createSINode_( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID timeseries_id, - const simdb::DatabaseID parent_node_id, - const std::string & si_node_name, - const int si_relative_index) - { - std::unique_ptr si_node_tbl = - obj_mgr.getTable("SINodeHierarchy"); - - return si_node_tbl->createObjectWithArgs( - "TimeseriesID", timeseries_id, - "ParentNodeID", parent_node_id, - "NodeName", si_node_name, - "RelativeSIIndex", si_relative_index); - } -}; - -//! Recursively create records in the SINodeHierarchy to -//! describe the overall SI tree in a report. This walks -//! the report's SI tree in a depth-first fashion. -void recursCreateSubreportSIHierarchy( - const simdb::ObjectManager & obj_mgr, - const simdb::ObjectRef & parent_hier_node, - const simdb::DatabaseID timeseries_id, - const Report & report, - int & si_relative_index) -{ - const auto & stats = report.getStatistics(); - const auto & subreports = report.getSubreports(); - - if (!stats.empty()) { - for (const auto & stat : stats) { - //Create leaf SI nodes for all StatisticInstance's - //we encounter. For example, "top.core0.rob.ipc" - const std::string name = !stat.first.empty() ? - stat.first : stat.second->getLocation(); - - SINodeCreator::createLeafSINode( - obj_mgr, timeseries_id, - parent_hier_node.getId(), - name, si_relative_index); - } - } - - if (!subreports.empty()) { - for (const auto & sr : subreports) { - std::vector dot_delimited; - boost::split(dot_delimited, sr.getName(), boost::is_any_of(".")); - const std::string & name = dot_delimited.back(); - - //Create another record in the SINodeHierarchy table - //for this mid-level report/subreport node. For example, - //"top.core0.rob" - std::unique_ptr sr_node = - SINodeCreator::createMidLevelSINode( - obj_mgr, timeseries_id, - parent_hier_node.getId(), - name, si_relative_index); - - //Recursively call this method to handle the next - //subreport level of nodes. - recursCreateSubreportSIHierarchy( - obj_mgr, *sr_node, timeseries_id, - sr, si_relative_index); - } - } -} - -//! Implementation class for SINodeHierarchy. Populates the -//! database with metadata describing the overall SI tree -//! (names of nodes, how the nodes are grouped i.e. parent/ -//! child, etc.) -class SINodeHierarchy::Impl -{ -public: - Impl(db::ReportTimeseries & db_timeseries, - const Report & report) : - db_timeseries_(db_timeseries), - report_(report) - {} - - //! Serialize our ReportTimeseries' SI tree into the - //! provided ObjectManager. - simdb::DatabaseID serializeHierarchy(simdb::ObjectManager & obj_mgr) - { - simdb::DatabaseID root_report_id = 0; - - obj_mgr.safeTransaction([&]() { - int si_relative_index = 0; - - std::unique_ptr si_root = - SINodeCreator::createMidLevelSINode( - obj_mgr, db_timeseries_.getId(), 0, - report_.getName(), si_relative_index); - - auto & db_header_obj_ref = db_timeseries_.getHeader().getObjectRef(); - db_header_obj_ref.setPropertyInt32("SIRootNodeID", si_root->getId()); - - recursCreateSubreportSIHierarchy( - obj_mgr, *si_root, db_timeseries_.getId(), - report_, si_relative_index); - - root_report_id = si_root->getId(); - }); - - return root_report_id; - } - -private: - db::ReportTimeseries & db_timeseries_; - const Report & report_; -}; - -SINodeHierarchy::SINodeHierarchy(db::ReportTimeseries & db_timeseries, - const Report & report) : - impl_(new SINodeHierarchy::Impl(db_timeseries, report)) -{ -} - -simdb::DatabaseID SINodeHierarchy::serializeHierarchy(simdb::ObjectManager & obj_mgr) -{ - return impl_->serializeHierarchy(obj_mgr); -} - -} // namespace statistics -} // namespace sparta diff --git a/sparta/src/Simulation.cpp b/sparta/src/Simulation.cpp index b021caba46..73989b39bc 100644 --- a/sparta/src/Simulation.cpp +++ b/sparta/src/Simulation.cpp @@ -41,20 +41,8 @@ #include "src/State.tpp" #include "sparta/kernel/MemoryProfiler.hpp" #include "sparta/statistics/dispatch/streams/StatisticsStreams.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/impl/hdf5/HDF5ConnProxy.hpp" -#include "simdb/async/AsyncTaskEval.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/schema/DatabaseRoot.hpp" -#include "simdb/Errors.hpp" -#include "sparta/report/db/Schema.hpp" -#include "sparta/report/db/SimInfoSerializer.hpp" -#include "sparta/report/db/ReportVerifier.hpp" #include "sparta/app/FeatureConfiguration.hpp" #include "sparta/kernel/PhasedObject.hpp" -#include "sparta/report/DatabaseInterface.hpp" -#include "simdb/schema/Schema.hpp" -#include "simdb/schema/TableSummaries.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/simulation/ClockManager.hpp" #include "sparta/utils/Colors.hpp" @@ -307,16 +295,6 @@ void printSchedulerPerformanceInfo(std::ostream& o, const boost::timer::cpu_time << scheduler->getNumFired() << std::endl; } -simdb::DatabaseRoot * Simulation::getDatabaseRoot() const -{ - return db_root_.get(); -} - -const DatabaseAccessor * Simulation::getSimulationDatabaseAccessor() const -{ - return sim_db_accessor_.get(); -} - Simulation::Simulation(const std::string& sim_name, Scheduler * scheduler) : clk_manager_(scheduler), @@ -339,17 +317,6 @@ Simulation::Simulation(const std::string& sim_name, // Sanity check - simulations cannot exist without a scheduler sparta_assert(scheduler_, "All simulators must be given a non-null scheduler"); - - REGISTER_SIMDB_NAMESPACE(Stats, SQLite); - REGISTER_SIMDB_SCHEMA_BUILDER(Stats, db::buildSimulationDatabaseSchema); - - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(SQLite, []() { - return new simdb::SQLiteConnProxy; - }); - - REGISTER_SIMDB_PROXY_CREATE_FUNCTION(HDF5, []() { - return new simdb::HDF5ConnProxy; - }); } Simulation::~Simulation() @@ -389,27 +356,7 @@ Simulation::~Simulation() // Deregister root_.getNodeAttachedNotification().DEREGISTER_FOR_THIS(rootDescendantAdded_); - - // ReportRepository's destructor may call back into - // SimDB objects, such as AsyncTaskController, to do - // any last-minute data flushes to the database. We - // should streamline these various classes that need - // to coordinate post-simulation / post-processing - // between themselves, and not have to worry about - // the calling order of their destructors. - // - // (Temporary: This is a quick fix for the v1.7.1 release) - // - Bug found went like this: - // 1. Delete the DatabaseRoot - // 2. Delete the ReportRepository - // 3. ReportRepository::saveReports() gets implicitly - // called from its own destructor - // 4. Code in RR::saveReports() got a *raw* pointer - // to the Simulation's (this) DatabaseRoot, which - // by that point had been deleted - // 5. Use of the dead DatabaseRoot seg faulted report_repository_.reset(); - db_root_.reset(); } void Simulation::configure(const int argc, @@ -506,87 +453,6 @@ void Simulation::configure(const int argc, setupProfilers_(); simulation_state_.configure(); - inspectFeatureValues_(); -} - -// At the very end of the simulation's configure() method, -// take a first look at the feature values we were given. -void Simulation::inspectFeatureValues_() -{ - const bool db_featured_on = IsFeatureValueEnabled(feature_config_, "simdb"); - const bool needs_db = !db_root_ || !stats_db_ || !sim_db_accessor_; - - if (db_featured_on && needs_db) { - if (!sim_db_accessor_) { - sim_db_accessor_.reset(new DatabaseAccessor(&root_)); - } - - if (!db_root_) { - std::string db_dir = sim_config_->getSimulationDatabaseLocation(); - if (db_dir.empty()) { - auto tempdir = std::filesystem::temp_directory_path(); - db_dir = tempdir.string(); - } - - db_root_.reset(new simdb::DatabaseRoot(db_dir)); - } - - if (!stats_db_) { - auto db_root = getDatabaseRoot(); - sparta_assert(db_root, "Could not find database root"); - - auto stats_namespace = db_root->getNamespace("Stats"); - sparta_assert(stats_namespace, "Could not find Stats database namespace"); - - auto stats_db = stats_namespace->getDatabase(); - sparta_assert(stats_db, "Could not find Stats database"); - - stats_db_ = stats_db->getObjectManager(); - sparta_assert(stats_db_, "Could not get ObjectManager from the Stats database"); - } - - simdb::TableSummaries si_summaries; - db::configureDatabaseTableSummaries(si_summaries); - - if (isReportValidationEnabled_()) { - si_summaries.excludeTables( - "ReportVerificationMetadata", - "ReportVerificationResults", - "ReportVerificationFailureSummaries", - "ReportVerificationDeepCopyFiles"); - } - - //TODO - Table summary configurations should be registered - //via a macro that looks like this: - // - // REGISTER_SIMDB_SCHEMA_SUMMARY(Stats, [](simdb::Schema & schema) { - // simdb::TableSummaries si_summaries; - // db::configureDatabaseTableSummaries(si_summaries); - // schema.setTableSummaryConfig(std::move(si_summaries)); - // }); - // - //Or perhaps like this: - // - // REGISTER_SIMDB_SCHEMA_SUMMARY( - // Stats, db::configureDatabaseTableSummaries); - // - //Where the second argument is a std::function - auto & si_schema = stats_db_->getSchema(); - si_schema.setTableSummaryConfig(std::move(si_summaries)); - } -} - -bool Simulation::isReportValidationEnabled_() const -{ - if (stats_db_ == nullptr) { - return false; - } - - if (IsFeatureValueEnabled(feature_config_, "simdb-verify")) { - return true; - } - - return false; } void Simulation::addReport(const ReportDescriptor & rep) @@ -870,9 +736,6 @@ void Simulation::run(uint64_t run_time) "See Simulation::finalizeFramework"); } - // Create any SimDB triggers the simulation was configured to use - this->setupDatabaseTriggers_(); - // Setup Pevent instruction warmup if (pevent_warmup_icount_ > 0) { // We are waiting, so we must setup a trigger. @@ -902,11 +765,6 @@ void Simulation::run(uint64_t run_time) // Actually run the simulation (or allow it to be controlled) PHASE_PROFILER(memory_profiler_, MemoryProfiler::Phase::Simulate); runRaw_(run_time); - } catch(const simdb::DatabaseInterrupt &) { - // Nothing to do. These are not "real" exceptions - nothing - // bad happened, there was just a request to stop database - // work from continuing. Don't add this exception info to - // the 'eptr' variable. } catch (...) { eptr = std::current_exception(); } @@ -1078,8 +936,6 @@ void Simulation::saveReports() { std::cout << "Saving reports..." << std::endl; - db::ReportVerifier formatted_reports_to_verify; - // Print summary report when there is no exception if(auto_summary_report_){ sparta::report::format::Text summary_fmt(auto_summary_report_.get()); @@ -1095,286 +951,15 @@ void Simulation::saveReports() summary_fmt.setShowDescriptions(true); } std::cout << summary_fmt << std::endl;; - - if (stats_db_) { - std::cout << " [simdb] Validation of database-generated " - "auto summary is currently unavailable\n"; - } - } - - std::map> formatters_in_use; - - auto save_reports = [&]() { - //Write out the SimulationInfo global data to the database. - //This table holds metadata that is common to all reports - //that came from this simulation. - if (stats_db_ != nullptr) { - formatters_in_use = report_repository_->getFormattersByFilename(); - db::SimInfoSerializer serializer(SimulationInfo::getInstance()); - serializer.serialize(*stats_db_); - std::cout << "SimDB file is: " << stats_db_->getDatabaseFile() << std::endl; - } - - auto saved_reports = report_repository_->saveReports(); - (void) saved_reports; - }; - - if (stats_db_ != nullptr) { - stats_db_->safeTransaction([&]() { - save_reports(); - stats_db_->captureTableSummaries(); - }); - } else { - save_reports(); } - simdb::AsyncTaskEval * db_thread = - stats_db_ ? stats_db_->getTaskQueue() : nullptr; - -#ifdef SPARTA_PYTHON_SUPPORT - if (pyshell_) { - //Remove the DB worker variable from the Python namespace. - //This allows the last Python interaction to access the database - //as much as it wants without incurring useless synchronization - //and queue flushes. - if (db_thread) { - pyshell_->removePublishedObject(db_thread); - } - - std::cout << "\n* * * Simulation is finished. Type 'quit' to exit " - "the Python shell. * * *\n" << std::endl; - pyshell_->interact(); - } -#endif - - //Reports have all been saved to disk / database / etc. - //Stop the database consumer thread and delete it now. - if (db_thread != nullptr) { - db_thread->stopThread(); - } + report_repository_->saveReports(); #ifdef SPARTA_TCMALLOC_SUPPORT if (memory_profiler_) { memory_profiler_->saveReport(); } #endif - - if (isReportValidationEnabled_()) { - stats_db_->safeTransaction([&]() { - for (const auto & rd : rep_descs_) { - if (rd.isEnabled()) { - formatted_reports_to_verify.addReportToVerify(rd); - } - } - - for (const auto & formatter : formatters_in_use) { - formatted_reports_to_verify.addBaseFormatterForPreVerificationReset( - formatter.first, formatter.second.get()); - } - - auto verification_summary = - formatted_reports_to_verify.verifyAll(*stats_db_, - scheduler_); - - if (verification_summary->hasSummary()) { - verification_summary->serializeSummary(*stats_db_); - } - - namespace sfs = std::filesystem; - std::map final_dest_files = - verification_summary->getFinalDestFiles(); - - //Store some basic info about any failures that occurred while - //checking these report files. We'll be given a chance to error - //out or produce a warning shortly. - report_verif_failed_fnames_ = - verification_summary->getFailingReportFilenames(); - - //Copy any report files that were renamed or moved somewhere - //inside the verification directory. The SPARTA simulator is - //still expecting the report files where their yaml file - //specified. - for (const auto & to_copy : final_dest_files) { - const std::string & simdb_dest_file = to_copy.first; - const std::string & orig_yaml_dest_file = to_copy.second; - if (sfs::exists(simdb_dest_file)) { - try { - if (sfs::exists(orig_yaml_dest_file)) { - sfs::remove(orig_yaml_dest_file); - } - sfs::copy_file(simdb_dest_file, orig_yaml_dest_file); - } catch (const std::exception & ex) { - std::cout << " [simdb] Unable to copy report file '" - << simdb_dest_file << "' to '" - << orig_yaml_dest_file << "'" - << std::endl; - } - } - } - }); - } -} - -void Simulation::postProcessingLastCall() -{ - std::ostringstream oss; - - if (!report_verif_failed_fnames_.empty()) { - //The report filename might have been something like: - // AccuracyCheckedDBs/abcd-1234/out.json - // - //Let's turn these strings into something like this: - // AccuracyCheckedDBs/abcd-1234.db/out.json - // - //Which is shorthand for "database file abcd-1234.db, found - //in the AccuracyCheckedDBs subfolder, failed to verify the - //contents of dest_file out.json" - auto make_err_string_from_report_fname = [](const std::string & fname) { - auto slash = fname.find_last_of("/"); - if (slash == std::string::npos) { - return fname; - } - - std::string err = "'" + fname.substr(0, slash); - err += ".db" + fname.substr(slash) + "'"; - return err; - }; - - oss << "Simulation failed to verify reports: "; - if (report_verif_failed_fnames_.size() == 1) { - oss << make_err_string_from_report_fname( - *report_verif_failed_fnames_.begin()); - } else { - size_t idx = 0; - auto iter = report_verif_failed_fnames_.begin(); - while (idx < report_verif_failed_fnames_.size() - 1) { - oss << make_err_string_from_report_fname(*iter) << ", "; - ++iter; - ++idx; - } - oss << make_err_string_from_report_fname(*iter); - } - oss << "\n"; - } - - //Make sure the ReportDescriptor report files all ended - //up in their *original* dest_file locations. We may have - //overwritten their dest_file values, but we should have - //put those report files back to where the outside world - //(production SPARTA simulators, sparta_core_example tests) - //expected to find them. - if (isReportValidationEnabled_()) { - for (const auto & rd : rep_descs_) { - if (rd.isEnabled()) { - std::string orig_dest_file = rd.getDescriptorOrigDestFile(); - if (!std::filesystem::exists(orig_dest_file)) { - auto not_slash = orig_dest_file.find_first_not_of("/"); - if (not_slash != std::string::npos) { - orig_dest_file = orig_dest_file.substr(not_slash); - } - - if (!std::filesystem::exists(orig_dest_file)) { - oss << "A report descriptor's dest_file was not " - << "found at the end of a SimDB verification-enabled simulation " - << "('" << orig_dest_file << "')\n"; - } - } - } - } - } - - const std::string last_call_exception_str = oss.str(); - if (!last_call_exception_str.empty()) { - throw SpartaException(last_call_exception_str); - } - - if (sim_config_ && stats_db_) { - namespace sfs = std::filesystem; - const std::string dest_path_str1 = sim_config_->getLegacyReportsCopyDir(); - if (!dest_path_str1.empty()) { - const std::string dest_path_str2 = - sfs::path(stats_db_->getDatabaseFile()).stem().string(); - - const std::set & collected_formats = - sim_config_->getLegacyReportsCollectedFormats(); - - if (!dest_path_str1.empty() && !dest_path_str2.empty()) { - for (const auto & rd : rep_descs_) { - if (!rd.isEnabled()) { - continue; - } - - if (!collected_formats.empty()) { - utils::lowercase_string lower_format = rd.format; - if (collected_formats.find(lower_format.getString()) == - collected_formats.end()) - { - continue; - } - } - - const std::string dest_path_str3 = rd.format; - if (dest_path_str3.empty()) { - continue; - } - - sfs::path dest_path = sfs::path(dest_path_str1 + "/" + - dest_path_str2 + "/" + - dest_path_str3); - - if (!sfs::exists(dest_path)) { - try { - sfs::create_directories(dest_path); - } catch (const std::exception & ex) { - std::cout << " [simdb] Error occurred while collecting " - << "legacy reports: '" << ex.what() << "'" - << std::endl; - continue; - } catch (...) { - } - } else if (!sfs::is_directory(dest_path)) { - throw SpartaException("Path exists, but is not a directory: '") - << dest_path.string() << "'"; - } - - std::string orig_dest_file = rd.getDescriptorOrigDestFile(); - if (!sfs::exists(orig_dest_file)) { - auto not_slash = orig_dest_file.find_first_not_of("/"); - if (not_slash != std::string::npos) { - orig_dest_file = orig_dest_file.substr(not_slash); - } - } - - if (orig_dest_file == "1") { - continue; - } - - if (sfs::exists(orig_dest_file)) { - const std::string src_file = orig_dest_file; - const std::string dest_file = - dest_path_str1 + "/" + - dest_path_str2 + "/" + - dest_path_str3 + "/" + - orig_dest_file; - - try { - sfs::copy_file(src_file, dest_file); - } catch (const std::exception & ex) { - std::cout << " [simdb] Error occurred while collecting " - << "legacy reports: '" << ex.what() << "'" - << std::endl; - continue; - } catch (...) { - } - } - } - } - } - } - - if (stats_db_ && root_clk_) { - root_clk_->serializeTo(*stats_db_); - } } void Simulation::dumpMetaParameterTable(std::ostream& out) const @@ -1757,26 +1342,6 @@ void Simulation::setupReports_() { validateReportDescriptors_(rep_descs_); - //If the SimDB report validation post-processing step is - //enabled, let's use this simulator's unique database filename - //to generate a temporary subfolder we can put our report files - //in. We'll move those files to their original intended location - //at the end of simulation, after we have performed the validation. - namespace sfs = std::filesystem; - std::string dest_file_subfolder; - if (isReportValidationEnabled_()) { - dest_file_subfolder = db::ReportVerifier::getVerifResultsDir(); - dest_file_subfolder += "/"; - - sfs::path path = stats_db_->getDatabaseFile(); - dest_file_subfolder += path.filename().string(); - - auto dot = dest_file_subfolder.find_last_of("."); - sparta_assert(dot != std::string::npos); - sparta_assert(dest_file_subfolder.substr(dot) == ".db"); - dest_file_subfolder = dest_file_subfolder.substr(0, dot); - } - // Set up reports now that the entire device tree is finalized for(auto& rd : rep_descs_){ //Report descriptors may have been disabled from Python during @@ -1795,54 +1360,6 @@ void Simulation::setupReports_() replacements); } - //When using SimDB, point the report files to a new subfolder - //under the verification subdirectory in the current pwd, so - //we get a directory structure like this: - // - // ./ - // abc-123/ - // foo.csv - // bar.json - // def-456/ - // foo.csv - // bar.json - // baz.txt - // - //This allows us to have deterministic post-simulation report - //verification by putting 'this' simulation's report dest_file's - //in a subfolder that does not clash with other concurrently - //running simulations. - if (rd.dest_file != "1" && isReportValidationEnabled_()) { - auto first_char = rd.dest_file.find_first_not_of("/"); - if (first_char != std::string::npos) { - rd.dest_file = rd.dest_file.substr(first_char); - } - rd.dest_file = dest_file_subfolder + "/" + rd.dest_file; - - //A descriptor's dest_file can be given as something like: - // def_file: simple_stats.yaml - // dest_file: /tmp/out.json - // format: json_reduced - // - //We need to create the subfolder /tmp inside the verif - //subfolder when this is the case, or std::ofstream - //will throw. - auto last_slash = rd.dest_file.find_last_of("/"); - sparta_assert(last_slash != std::string::npos); - const std::string rd_dest_dir = rd.dest_file.substr(0, last_slash); - - try { - sfs::create_directories(rd_dest_dir); - } catch (const std::exception & ex) { - std::cout << " [simdb] An exception was encountered when " - << "attempting to create a report verification " - << "subfolder '" << rd_dest_dir << "'. The " - << "exception message was '" << ex.what() - << "'." << std::endl; - continue; - } - } - sparta::ReportRepository::DirectoryHandle directoryH = report_repository_->createDirectory(rd); @@ -1906,29 +1423,6 @@ void Simulation::setupReports_() //One-time inspection of the simulator's feature values (if any) report_repository_->inspectSimulatorFeatureValues(feature_config_); - -#ifdef SPARTA_PYTHON_SUPPORT - //If we are using the Python shell and the "simdb" feature is - //featured on, publish our simulation database object to the - //Python namespace now. This lets users run queries against the - //database during simulation and access raw data (including .csv - //report generation if requested). - if (pyshell_ && stats_db_) { - pyshell_->publishSimulationDatabase(stats_db_); - } -#endif -} - -void Simulation::setupDatabaseTriggers_() -{ - if (!sim_config_ || !sim_db_accessor_) { - return; - } - - const auto & access_opt_files = sim_config_->getDatabaseAccessOptsFiles(); - for (const auto & opt_file : access_opt_files) { - sim_db_accessor_->setAccessOptsFromFile_(opt_file); - } } ReportDescVec Simulation::expandReportDescriptor_(const ReportDescriptor & rd) const diff --git a/sparta/src/SimulationInfo.cpp b/sparta/src/SimulationInfo.cpp index 2b91837fab..12fa1eca7d 100644 --- a/sparta/src/SimulationInfo.cpp +++ b/sparta/src/SimulationInfo.cpp @@ -3,9 +3,6 @@ #include #include "sparta/app/SimulationInfo.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/utils/ObjectQuery.hpp" - #include "rapidjson/document.h" #include "rapidjson/istreamwrapper.h" #include "rapidjson/stringbuffer.h" @@ -16,80 +13,8 @@ namespace sparta{ SimulationInfo SimulationInfo::sim_inst_; // Must be constructed after TimeManager - std::stack SimulationInfo::sim_inst_stack_; const char SimulationInfo::sparta_version[] = SPARTA_VERSION; - SimulationInfo::SimulationInfo(const simdb::ObjectManager & sim_db, - const simdb::DatabaseID obj_mgr_db_id, - const simdb::DatabaseID report_node_id) : - SimulationInfo() - { - sim_db.safeTransaction([&]() { - simdb::ObjectQuery global_meta_query(sim_db, "SimInfo"); - - global_meta_query.addConstraints( - "ObjMgrID", simdb::constraints::equal, obj_mgr_db_id); - - global_meta_query.writeResultIterationsTo( - "Name", &sim_name_, - "Cmdline", &command_line_, - "Exe", &executable_, - "SimulatorVersion", &simulator_version_, - "Repro", &reproduction_info_, - "SpartaVersion", &sparta_version_, - "Start", &start_time_); - - auto result_iter = global_meta_query.executeQuery(); - if (!result_iter->getNext()) { - throw SpartaException("Unable to locate a SimInfo record with ") - << "ObjMgrID equal to " << obj_mgr_db_id; - } - - //Clear out any strings that were "unset" in the database. - //SimDB's "unset" is equal to std::string's empty(). - auto clear_unset_string = [](std::string & str) { - if (str == "unset") { - str.clear(); - } - }; - clear_unset_string(sim_name_); - clear_unset_string(command_line_); - clear_unset_string(executable_); - clear_unset_string(simulator_version_); - clear_unset_string(reproduction_info_); - clear_unset_string(start_time_); - - //Apply any report-specific metadata we find. - if (report_node_id > 0) { - simdb::ObjectQuery report_meta_query(sim_db, "RootReportNodeMetadata"); - - report_meta_query.addConstraints( - "ReportNodeID", simdb::constraints::equal, report_node_id, - "Name", simdb::constraints::equal, "Elapsed"); - - std::string elapsed; - report_meta_query.writeResultIterationsTo("Value", &elapsed); - result_iter = report_meta_query.executeQuery(); - if (result_iter->getNext()) { - //Note that even if the "Elapsed" column was not set, - //we still want our ValidValue to be isValid(). Clear - //the 'elapsed' string if needed, but set the ValidValue - //either way. - clear_unset_string(elapsed); - db_elapsed_time_ = elapsed; - } - } - - //We currently only allow at most *one* non-singleton - //SimulationInfo object, strictly for the purpose of - //generating reports from a SimDB outside of a simulation. - sparta_assert(sim_inst_stack_.empty(), "You cannot create more than " - "one SimulationInfo object outside of a simulation."); - - sim_inst_stack_.push(this); - }); - } - /*! * \brief Instantiate a SimulationInfo object from a json, json_reduced, * json_detail, or js_json report file. @@ -106,11 +31,6 @@ namespace sparta{ return; } - sparta_assert(sim_inst_stack_.empty(), "You cannot create more than " - "one SimulationInfo object outside of a simulation."); - - sim_inst_stack_.push(this); - namespace rj = rapidjson; rj::IStreamWrapper isw(json_fin); rj::Document doc; diff --git a/sparta/src/SingleUpdateReport.cpp b/sparta/src/SingleUpdateReport.cpp deleted file mode 100644 index 38ca014979..0000000000 --- a/sparta/src/SingleUpdateReport.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// -*- C++ -*- - -#include "sparta/report/db/SingleUpdateReport.hpp" - -#include -#include -#include - -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "simdb/Constraints.hpp" -#include "simdb/schema/ColumnTypedefs.hpp" - -namespace sparta { -namespace db { - -SingleUpdateReport::SingleUpdateReport( - std::unique_ptr obj_ref) : - obj_ref_(std::move(obj_ref)) -{ -} - -SingleUpdateReport::SingleUpdateReport( - const simdb::ObjectManager & obj_mgr, - const simdb::DatabaseID root_report_node_id) : - root_report_node_id_(root_report_node_id) -{ - std::unique_ptr si_values_tbl = - obj_mgr.getTable("SingleUpdateStatInstValues"); - - obj_ref_ = si_values_tbl->createObjectWithArgs( - "RootReportNodeID", root_report_node_id); -} - -int SingleUpdateReport::getId() const -{ - return obj_ref_->getId(); -} - -void SingleUpdateReport::writeStatisticInstValues( - const std::vector & si_values) -{ - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &si_values[0]; - blob_descriptor.num_bytes = si_values.size() * sizeof(double); - - auto table = obj_ref_->getObjectManager().getTable( - "SingleUpdateStatInstValues"); - - if (!table->updateRowValues("RawBytes", blob_descriptor, - "NumPts", (int)si_values.size(), - "WasCompressed", 0). - forRecordsWhere("RootReportNodeID", - simdb::constraints::equal, - root_report_node_id_)) - { - throw SpartaException( - "Unable to write uncompressed SI blob to the database"); - } -} - -void SingleUpdateReport::writeCompressedStatisticInstValues( - const std::vector & compressed_si_values, - const uint32_t original_num_si_values) -{ - simdb::Blob blob_descriptor; - blob_descriptor.data_ptr = &compressed_si_values[0]; - blob_descriptor.num_bytes = original_num_si_values * sizeof(double); - - auto table = obj_ref_->getObjectManager().getTable( - "SingleUpdateStatInstValues"); - - if (!table->updateRowValues("RawBytes", blob_descriptor, - "NumPts", (int)original_num_si_values, - "WasCompressed", 1). - forRecordsWhere("RootReportNodeID", - simdb::constraints::equal, - root_report_node_id_)) - { - throw SpartaException( - "Unable to write compressed SI blob to the database"); - } -} - -void SingleUpdateReport::getStatisticInstValues( - std::vector & si_values) -{ - const auto & obj_mgr = obj_ref_->getObjectManager(); - - //SELECT NumPts,WasCompressed,RawBytes - //FROM SingleUpdateStatInstValues - //WHERE Id= - simdb::ObjectQuery si_query(obj_mgr, "SingleUpdateStatInstValues"); - si_query.addConstraints("Id", simdb::constraints::equal, getId()); - - int num_si_values = 0, was_compressed = 0; - std::vector compressed_blob; - - si_query.writeResultIterationsTo( - "NumPts", &num_si_values, - "WasCompressed", &was_compressed, - "RawBytes", &compressed_blob); - - auto result_iter = si_query.executeQuery(); - assert(result_iter != nullptr); - - if (!result_iter->getNext()) { - //There is no SI data for this record. Clear and - //shrink the output vector. - si_values.clear(); - si_values.shrink_to_fit(); - return; - } - - //Single-update records should have only one SI row. - if (result_iter->getNext()) { - throw SpartaException("Unexpectedly found multiple records ") - << "in the SingleUpdateStatInstValues table with row Id " - << getId(); - } - - //We currently *only* support single-update reports (json, html, etc.) - //in compressed format. - if (!was_compressed) { - throw SpartaException("Unexpectedly found a single-update report which ") - << "had uncompressed SI values stored in the database"; - } - - //Re-inflate the compressed SI blob - z_stream infstream; - infstream.zalloc = Z_NULL; - infstream.zfree = Z_NULL; - infstream.opaque = Z_NULL; - - //Setup the source stream - infstream.avail_in = (uInt)(compressed_blob.size()); - infstream.next_in = (Bytef*)(&compressed_blob[0]); - - //Setup the destination stream - si_values.resize(num_si_values); - infstream.avail_out = (uInt)(si_values.size() * sizeof(double)); - infstream.next_out = (Bytef*)(&si_values[0]); - - //Inflate it! - inflateInit(&infstream); - inflate(&infstream, Z_FINISH); - inflateEnd(&infstream); -} - -} // namespace db -} // namespace sparta diff --git a/sparta/src/StatisticInstance.cpp b/sparta/src/StatisticInstance.cpp index a7e77605f5..e1f3b45836 100644 --- a/sparta/src/StatisticInstance.cpp +++ b/sparta/src/StatisticInstance.cpp @@ -7,8 +7,6 @@ */ #include "sparta/statistics/StatisticInstance.hpp" -#include "sparta/report/db/StatInstValueLookup.hpp" -#include "sparta/report/db/StatInstRowIterator.hpp" namespace sparta { @@ -101,6 +99,22 @@ namespace sparta sparta_assert(false == node_ref_.expired()); } + //! \brief Copy Constructor + StatisticInstance::StatisticInstance(const StatisticInstance& rhp) : + node_ref_(rhp.node_ref_), + sdef_(rhp.sdef_), + ctr_(rhp.ctr_), + par_(rhp.par_), + stat_expr_(rhp.stat_expr_), + start_tick_(rhp.start_tick_), + end_tick_(rhp.end_tick_), + scheduler_(rhp.scheduler_), + initial_(rhp.initial_), + result_(rhp.result_), + sub_statistics_(rhp.sub_statistics_) + { + } + StatisticInstance::StatisticInstance(std::shared_ptr & calculator, std::vector& used) : StatisticInstance(nullptr, nullptr, nullptr, calculator->getNode(), &used) @@ -117,43 +131,6 @@ namespace sparta user_calculated_si_value_ = calculator; } - //! \brief Copy Constructor - StatisticInstance::StatisticInstance(const StatisticInstance& rhp) : - node_ref_(rhp.node_ref_), - sdef_(rhp.sdef_), - ctr_(rhp.ctr_), - par_(rhp.par_), - stat_expr_(rhp.stat_expr_), - start_tick_(rhp.start_tick_), - end_tick_(rhp.end_tick_), - scheduler_(rhp.scheduler_), - initial_(rhp.initial_), - result_(rhp.result_), - sub_statistics_(rhp.sub_statistics_), - user_calculated_si_value_(rhp.user_calculated_si_value_), - direct_lookup_si_value_(rhp.direct_lookup_si_value_), - provided_metadata_(rhp.provided_metadata_) - { - if (rhp.provided_location_.isValid()) { - provided_location_ = rhp.provided_location_.getValue(); - } - if (rhp.provided_description_.isValid()) { - provided_description_ = rhp.provided_description_.getValue(); - } - if (rhp.provided_expr_string_.isValid()) { - provided_expr_string_ = rhp.provided_expr_string_.getValue(); - } - if (rhp.provided_value_semantic_.isValid()) { - provided_value_semantic_ = rhp.provided_value_semantic_.getValue(); - } - if (rhp.provided_visibility_.isValid()) { - provided_visibility_ = rhp.provided_visibility_.getValue(); - } - if (rhp.provided_class_.isValid()) { - provided_class_ = rhp.provided_class_.getValue(); - } - } - //! \brief Move Constructor StatisticInstance::StatisticInstance(StatisticInstance&& rhp) : node_ref_(std::move(rhp.node_ref_)), @@ -165,77 +142,12 @@ namespace sparta end_tick_(rhp.end_tick_), scheduler_(rhp.scheduler_), initial_(rhp.initial_), - result_(rhp.result_), - sub_statistics_(std::move(rhp.sub_statistics_)), - user_calculated_si_value_(std::move(rhp.user_calculated_si_value_)), - direct_lookup_si_value_(std::move(rhp.direct_lookup_si_value_)), - provided_metadata_(std::move(rhp.provided_metadata_)) + result_(rhp.result_) { rhp.sdef_ = nullptr; rhp.ctr_ = nullptr; rhp.par_ = nullptr; rhp.result_ = NAN; - - if (rhp.provided_location_.isValid()) { - provided_location_ = rhp.provided_location_.getValue(); - } - if (rhp.provided_description_.isValid()) { - provided_description_ = rhp.provided_description_.getValue(); - } - if (rhp.provided_expr_string_.isValid()) { - provided_expr_string_ = rhp.provided_expr_string_.getValue(); - } - if (rhp.provided_value_semantic_.isValid()) { - provided_value_semantic_ = rhp.provided_value_semantic_.getValue(); - } - if (rhp.provided_visibility_.isValid()) { - provided_visibility_ = rhp.provided_visibility_.getValue(); - } - if (rhp.provided_class_.isValid()) { - provided_class_ = rhp.provided_class_.getValue(); - } - - rhp.provided_location_.clearValid(); - rhp.provided_description_.clearValid(); - rhp.provided_expr_string_.clearValid(); - rhp.provided_value_semantic_.clearValid(); - rhp.provided_visibility_.clearValid(); - rhp.provided_class_.clearValid(); - } - - StatisticInstance::StatisticInstance(const std::string & location, - const std::string & description, - const std::string & expression_str, - const StatisticDef::ValueSemantic value_semantic, - const InstrumentationNode::visibility_t visibility, - const InstrumentationNode::class_t cls, - const std::vector> & metadata) : - provided_location_(location), - provided_description_(description), - provided_expr_string_(expression_str), - provided_value_semantic_(value_semantic), - provided_visibility_(visibility), - provided_class_(cls), - provided_metadata_(metadata) - {} - - StatisticInstance::StatisticInstance(const std::string & location, - const std::string & description, - const std::shared_ptr & calculator, - const InstrumentationNode::visibility_t visibility, - const InstrumentationNode::class_t cls, - const std::vector> & metadata) : - user_calculated_si_value_(calculator), - provided_visibility_(visibility), - provided_class_(cls), - provided_metadata_(metadata) - { - if (!location.empty()) { - provided_location_ = location; - } - if (!description.empty()) { - provided_description_ = description; - } } StatisticInstance& StatisticInstance::operator=(const StatisticInstance& rhp) { @@ -252,27 +164,14 @@ namespace sparta result_ = rhp.result_; sub_statistics_ = rhp.sub_statistics_; - user_calculated_si_value_ = rhp.user_calculated_si_value_; - direct_lookup_si_value_ = rhp.direct_lookup_si_value_; - provided_metadata_ = rhp.provided_metadata_; return *this; } void StatisticInstance::start() { - sparta_assert(direct_lookup_si_value_ == nullptr, - "You cannot call StatisticInstance::start() for an SI " - "that was recreated from a SimDB record"); - start_tick_ = getScheduler_()->getElapsedTicks(); end_tick_ = Scheduler::INDEFINITE; - if(user_calculated_si_value_){ - initial_.resetValue(user_calculated_si_value_->getCurrentValue()); - result_ = NAN; - return; - } - if(sdef_ != nullptr){ if(node_ref_.expired() == true){ throw SpartaException("Cannot start() a StatisticInstance referring to a " @@ -301,10 +200,6 @@ namespace sparta } void StatisticInstance::end(){ - sparta_assert(direct_lookup_si_value_ == nullptr, - "You cannot call StatisticInstance::end() for an SI " - "that was recreated from a SimDB record"); - end_tick_ = getScheduler_()->getElapsedTicks(); if(sdef_ != nullptr){ @@ -334,10 +229,6 @@ namespace sparta } double StatisticInstance::getValue() const { - if (direct_lookup_si_value_ != nullptr) { - return computeValue_(); - } - if(SPARTA_EXPECT_FALSE(end_tick_ < start_tick_)) { throw ReversedStatisticRange("Range is reversed. End < start"); } @@ -396,9 +287,6 @@ namespace sparta } bool StatisticInstance::supportsCompression() const { - if (user_calculated_si_value_) { - return false; - } if (sdef_) { if (node_ref_.expired()) { return false; @@ -455,9 +343,6 @@ namespace sparta std::string StatisticInstance::getExpressionString(bool show_range, bool resolve_subexprs) const { - if(provided_expr_string_.isValid()) { - return provided_expr_string_.getValue(); - } if(sdef_){ if(node_ref_.expired() == false){ // Print the fully rendered expression string instead of the @@ -485,9 +370,6 @@ namespace sparta } std::string StatisticInstance::getDesc(bool show_stat_node_expressions) const { - if(provided_description_.isValid()) { - return provided_description_.getValue(); - } if(sdef_){ if(node_ref_.expired() == false){ std::string result = sdef_->getDesc(); @@ -545,9 +427,6 @@ namespace sparta } std::string StatisticInstance::getLocation() const { - if(provided_location_.isValid()) { - return provided_location_.getValue(); - } if(sdef_){ if(node_ref_.expired() == false){ return node_ref_.lock()->getLocation(); @@ -572,9 +451,6 @@ namespace sparta } StatisticDef::ValueSemantic StatisticInstance::getValueSemantic() const { - if(provided_value_semantic_.isValid()) { - return provided_value_semantic_.getValue(); - } if(sdef_){ if(node_ref_.expired() == false){ return sdef_->getValueSemantic(); @@ -591,9 +467,6 @@ namespace sparta } InstrumentationNode::visibility_t StatisticInstance::getVisibility() const { - if(provided_visibility_.isValid()) { - return provided_visibility_.getValue(); - } if(node_ref_.expired()) { return InstrumentationNode::VIS_NORMAL; } @@ -609,9 +482,6 @@ namespace sparta } InstrumentationNode::class_t StatisticInstance::getClass() const { - if(provided_class_.isValid()) { - return provided_class_.getValue(); - } if(node_ref_.expired()) { return InstrumentationNode::DEFAULT_CLASS; } @@ -649,48 +519,7 @@ namespace sparta } } - void StatisticInstance::setSIValueDirectLookupPlaceholder( - const std::shared_ptr & direct_lookup) - { - direct_lookup_si_value_ = direct_lookup; - } - - void StatisticInstance::realizeSIValueDirectLookup( - const StatInstRowIterator & si_row_iterator) - { - if (direct_lookup_si_value_ != nullptr) { - auto realized_lookup = direct_lookup_si_value_-> - realizePlaceholder(si_row_iterator.getRowAccessor()); - - sparta_assert(realized_lookup != nullptr); - direct_lookup_si_value_.reset(realized_lookup); - } - } - - bool StatisticInstance::isSIValueDirectLookupValid() const - { - if (direct_lookup_si_value_ == nullptr) { - return false; - } - - //The following function call throws if this direct - //lookup object is a placeholders::StatInstValueLookup - //which has not yet been realized. - try { - return direct_lookup_si_value_->isIndexValidForCurrentRow(); - } catch (...) { - } - - return false; - } - double StatisticInstance::computeValue_() const { - if(user_calculated_si_value_){ - return user_calculated_si_value_->getCurrentValue() - getInitial(); - } - if(direct_lookup_si_value_){ - return getCurrentValueFromDirectLookup_(); - } if(sdef_){ if(node_ref_.expired() == true){ return NAN; @@ -742,27 +571,4 @@ namespace sparta return scheduler_; } - double StatisticInstance::getCurrentValueFromDirectLookup_() const - { - if (direct_lookup_si_value_ == nullptr) { - throw SpartaException("StatisticInstance asked for its SI ") - << "value from a null direct-lookup object"; - } - - sparta_assert(getInitial() == 0, - "Unexpectedly encountered a StatisticInstance that " - "was created from a SimDB record, but whose SI offset " - "value (SI::getInitial()) was not zero. This is a bug."); - - //The following function call throws if this direct - //lookup object is a placeholders::StatInstValueLookup - //which has not yet been realized. - try { - return direct_lookup_si_value_->getCurrentValue(); - } catch (...) { - } - - return NAN; - } - } diff --git a/sparta/src/TemporaryRunController.cpp b/sparta/src/TemporaryRunController.cpp index 6c556e9ef7..39e0388aec 100644 --- a/sparta/src/TemporaryRunController.cpp +++ b/sparta/src/TemporaryRunController.cpp @@ -17,7 +17,6 @@ #include "sparta/kernel/Scheduler.hpp" #include "sparta/trigger/SingleTrigger.hpp" #include "sparta/statistics/dispatch/streams/StreamController.hpp" -#include "simdb/async/AsyncTaskEval.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/statistics/CounterBase.hpp" #include "sparta/utils/SpartaAssert.hpp" @@ -56,10 +55,7 @@ void TemporaryRunControl::runIcountEnd_() { if (stream_controller_) { stream_controller_->processStreams(); } - if (db_task_thread_) { - db_task_thread_->emitPreFlushNotification(); - db_task_thread_->flushQueue(); - } + //! \todo This stopRunning may need to skip or allow scheduler to finish up it's trigger //! stuff for this cycle. This may not have an ideal implementation sched_->stopRunning(); @@ -99,12 +95,6 @@ std::shared_ptr & return stream_controller_; } -void TemporaryRunControl::setDatabaseTaskThread( - simdb::AsyncTaskEval * db_thread) -{ - db_task_thread_ = db_thread; -} - uint64_t TemporaryRunControl::getCurrentCycle(const std::string& clk_name) const { const sparta::Clock * runtime_clk = findClock_(clk_name); return runtime_clk->currentCycle(); diff --git a/sparta/test/Array/Array_test.cpp b/sparta/test/Array/Array_test.cpp index 7e12271719..9d43a6204e 100644 --- a/sparta/test/Array/Array_test.cpp +++ b/sparta/test/Array/Array_test.cpp @@ -17,15 +17,12 @@ #include "sparta/resources/Array.hpp" #include "sparta/resources/FrontArray.hpp" #include "sparta/simulation/ClockManager.hpp" - #include "sparta/collection/PipelineCollector.hpp" #include TEST_INIT -#define PIPEOUT_GEN - struct dummy_struct { uint16_t int16_field; @@ -137,15 +134,9 @@ int main() root_node.enterConfiguring(); root_node.enterFinalized(); -#ifdef PIPEOUT_GEN - sparta::collection::PipelineCollector pc("test_collection_", 1000, &clk, &root); -#endif - + sparta::collection::PipelineCollector pc("test_collection_", {}, 10, &root_node, nullptr); sched.finalize(); - -#ifdef PIPEOUT_GEN - pc.startCollection(&root_node); -#endif + pc.startCollecting(); // Test perfect forwarding arrays move { @@ -467,17 +458,13 @@ int main() EXPECT_EQUAL(aged_array_test.getAge(i), age_vec_0[idxx++]); } -#ifdef PIPEOUT_GEN sched.run(1); -#endif aged_collected_array.erase(0); //+9 records. aged_collected_array.erase(1); //+8 records. aged_collected_array.write(0, 0); -#ifdef PIPEOUT_GEN sched.run(1); -#endif EXPECT_EQUAL(aged_array.abegin().getIndex(), aged_array.getOldestIndex().getIndex()); bit = aged_array.abegin(); @@ -516,17 +503,13 @@ int main() } EXPECT_EQUAL(cnt, 7); -#ifdef PIPEOUT_GEN sched.run(1); -#endif aged_array.write(4, 4); aged_array.write(2, 2); aged_array.write(1, 1); -#ifdef PIPEOUT_GEN sched.run(1); -#endif AgedArray::iterator it = aged_array.getCircularIterator(); while(it != aged_array.getCircularIterator(aged_array.capacity() - 1)) @@ -539,9 +522,7 @@ int main() EXPECT_EQUAL(*dat, 9); aged_array.erase(it); -#ifdef PIPEOUT_GEN sched.run(1); -#endif //set up the pipeline collection. @@ -552,9 +533,7 @@ int main() ns_array.write(1, 1); ns_array.write(2, 2); -#ifdef PIPEOUT_GEN sched.run(1); -#endif //ns_array.getOldestIndex(0); THROWS a static assert message since //ns_array is not aged. @@ -568,18 +547,13 @@ int main() EXPECT_EQUAL(ns_array.numValid(), 1); EXPECT_EQUAL(ns_array.capacity(), 10); -#ifdef PIPEOUT_GEN sched.run(1); -#endif - ns_array.write(5, 5); ns_array.write(3, 3); ns_array.write(0, 0); -#ifdef PIPEOUT_GEN sched.run(1); -#endif MyArray::iterator iter = ns_array.begin(); uint32_t i = 0; @@ -595,9 +569,8 @@ int main() ++i; } -#ifdef PIPEOUT_GEN sched.run(1); -#endif + iter = ns_array.begin(); std::advance(iter, 3); @@ -703,10 +676,8 @@ int main() // std::cout << "valid: " << iter.isValid() << std::endl; // } -#ifdef PIPEOUT_GEN sched.run(10); - pc.destroy(); -#endif + pc.stopCollecting(); //it's now safe to tear down our dummy tree root_node.enterTeardown(); diff --git a/sparta/test/Buffer/Buffer_test.cpp b/sparta/test/Buffer/Buffer_test.cpp index 66d24b61a7..51b5377b04 100644 --- a/sparta/test/Buffer/Buffer_test.cpp +++ b/sparta/test/Buffer/Buffer_test.cpp @@ -16,11 +16,10 @@ #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/CycleCounter.hpp" +#include "sparta/collection/PipelineCollector.hpp" TEST_INIT -#define PIPEOUT_GEN - #define QUICK_PRINT(x) \ std::cout << x << std::endl @@ -96,9 +95,7 @@ void generalTest() &buf10_stats); rtn.setClock(root_clk.get()); -#ifdef PIPEOUT_GEN buf10.enableCollection(&rtn); -#endif rtn.enterConfiguring(); rtn.enterFinalized(); @@ -106,16 +103,9 @@ void generalTest() // Get info messages from the scheduler node and send them to this file sparta::log::Tap t2(root_clk.get()->getScheduler(), "debug", "scheduler.log.debug"); -#ifdef PIPEOUT_GEN - sparta::collection::PipelineCollector pc("testBuffer", 1000000, - root_clk.get(), &rtn); -#endif - + sparta::collection::PipelineCollector pc("testBuffer", {}, 10, &rtn, nullptr); sched.finalize(); - -#ifdef PIPEOUT_GEN - pc.startCollection(&rtn); -#endif + pc.startCollecting(); //////////////////////////////////////////////////////////// sched.run(1); @@ -581,9 +571,7 @@ void generalTest() sched.run(5); rtn.enterTeardown(); -#ifdef PIPEOUT_GEN - pc.destroy(); -#endif + pc.stopCollecting(); } struct B { diff --git a/sparta/test/CMakeLists.txt b/sparta/test/CMakeLists.txt index c4912bf6b4..89ad1ae9f0 100644 --- a/sparta/test/CMakeLists.txt +++ b/sparta/test/CMakeLists.txt @@ -54,7 +54,6 @@ add_subdirectory (cache) add_subdirectory (Clock) add_subdirectory (CircularBuffer) add_subdirectory (Color) -add_subdirectory (Collection) add_subdirectory (CommandLineSimulator) add_subdirectory (Counter) add_subdirectory (ContextCounter) @@ -85,13 +84,11 @@ add_subdirectory (Rational) add_subdirectory (SpartaSharedPointer) add_subdirectory (Register) add_subdirectory (Report) -add_subdirectory (ReportVerifier) add_subdirectory (ResourceAssert) add_subdirectory (SpartaException) add_subdirectory (Scheduler) add_subdirectory (Scoreboard) add_subdirectory (SharedData) -add_subdirectory (SimDB) add_subdirectory (SmartLexCast) add_subdirectory (State) add_subdirectory (StaticInit) @@ -116,7 +113,6 @@ add_subdirectory (Utils) add_subdirectory (BitArray) add_subdirectory (StateTimer) add_subdirectory (ValidValue) -add_subdirectory (PairCollector) add_subdirectory (NestedPEvents) add_subdirectory (StateResidencyTracker) add_subdirectory (MetaTypeList) diff --git a/sparta/test/CircularBuffer/CircularBuffer_test.cpp b/sparta/test/CircularBuffer/CircularBuffer_test.cpp index 2abbb19f47..86fbf30878 100644 --- a/sparta/test/CircularBuffer/CircularBuffer_test.cpp +++ b/sparta/test/CircularBuffer/CircularBuffer_test.cpp @@ -14,7 +14,6 @@ TEST_INIT -//#define PIPEOUT_GEN struct dummy_struct { uint16_t int16_field; @@ -463,9 +462,6 @@ void testCollection() rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testCircBuffer", 1000000, - root_clk.get(), &rtn); - sched.finalize(); for(uint32_t i = 0; i < BUF_SIZE/2; ++i) { diff --git a/sparta/test/Collection/CMakeLists.txt b/sparta/test/Collection/CMakeLists.txt deleted file mode 100644 index 283ef4fcd2..0000000000 --- a/sparta/test/Collection/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -project(Collection_test) - -include(${SPARTA_CMAKE_MACRO_PATH}/SpartaTestingMacros.cmake) - -sparta_add_test_executable(Collection_test Collection_test.cpp) - -sparta_test(Collection_test Collection_test_RUN) diff --git a/sparta/test/Collection/Collection_test.cpp b/sparta/test/Collection/Collection_test.cpp deleted file mode 100644 index 07756caa40..0000000000 --- a/sparta/test/Collection/Collection_test.cpp +++ /dev/null @@ -1,72 +0,0 @@ - - -#include "sparta/collection/Collectable.hpp" -#include "sparta/collection/PipelineCollector.hpp" - -#include "sparta/utils/SpartaTester.hpp" - -#include "sparta/simulation/Clock.hpp" -#include "sparta/simulation/ClockManager.hpp" -#include "sparta/kernel/Scheduler.hpp" - -#include - -struct EmptyData {}; -std::ostream & operator<<(std::ostream & os, const EmptyData &) { - return os; -} - -void testEmptyCollection() -{ - - // Test communication between blocks using ports - sparta::Scheduler sched; - sparta::ClockManager cm(&sched); - sparta::RootTreeNode rtn; - sparta::Clock::Handle root_clk; - root_clk = cm.makeRoot(&rtn, "root_clk"); - cm.normalize(); - rtn.setClock(root_clk.get()); - - sparta::collection::Collectable collector(&rtn, "empty_collection_test"); - - rtn.enterConfiguring(); - rtn.enterFinalized(); - - sparta::collection::PipelineCollector pc("emptyPipe", 1000000, - root_clk.get(), &rtn); - - sched.finalize(); - - // Order matters -- need to finalize the scheduler before we start - // collecting. - pc.startCollection(&rtn); - - EmptyData dat; - - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - collector.collect(dat); - sched.run(1, true); - - rtn.enterTeardown(); - pc.destroy(); - - std::fstream record_file("emptyPiperecord.bin", std::fstream::in | std::fstream::binary ); - EXPECT_TRUE(record_file.peek() == std::ifstream::traits_type::eof()); -} - -int main() -{ - testEmptyCollection(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/test/CommandLineSimulator/CommandLineSimulator_test.cpp b/sparta/test/CommandLineSimulator/CommandLineSimulator_test.cpp index 88b01e49c1..858ebc6ae5 100644 --- a/sparta/test/CommandLineSimulator/CommandLineSimulator_test.cpp +++ b/sparta/test/CommandLineSimulator/CommandLineSimulator_test.cpp @@ -58,108 +58,8 @@ class MySimulator : public sparta::app::Simulation } }; -void testFeatureConfig() -{ - PRINT_ENTER_TEST - - sparta::app::FeatureConfiguration features; - EXPECT_EQUAL(features.getFeatureValue("simdb"), 0); - - features.setFeatureValue("simdb", 2); - EXPECT_EQUAL(features.getFeatureValue("simdb"), 2); - - EXPECT_NOTEQUAL(features.getFeatureOptions("simdb"), nullptr); - features.setFeatureOptionsFromFile("simdb", "sample_feat_opts.yaml"); - - auto opts = features.getFeatureOptions("simdb"); - EXPECT_NOTEQUAL(opts, nullptr); - - //The sample options yaml file we just applied has - //values like this: - // foo: hello - // bar: 56.8 - // - //Let's try a variety of getOptionValue() calls, - //including a few calls where we mix up the feature - //option data type (foo is a double, bar is a string). - - //When we ask for a feature option that does not exist, - //it should return the default value we pass in. - std::string default_opt_str = opts->getOptionValue( - "nonexistent", "none"); - EXPECT_EQUAL(default_opt_str, "none"); - - double default_opt_dbl = opts->getOptionValue( - "nonexistent", 4.6); - EXPECT_EQUAL(default_opt_dbl, 4.6); - - //Asking for a named option which exists in the yaml - //file should just return the value, either as a string - //or as a double depending on the data type. - std::string custom_opt_str = opts->getOptionValue( - "foo", "none"); - EXPECT_EQUAL(custom_opt_str, "hello"); - - double custom_opt_dbl = opts->getOptionValue( - "bar", 4.6); - EXPECT_WITHIN_EPSILON(custom_opt_dbl, 56.8); - - //In this sample options file, "foo" was a string ("hello"), - //so this call site is not valid. It should return - //the default double we pass in. - default_opt_dbl = opts->getOptionValue( - "foo", 4.6); - EXPECT_WITHIN_EPSILON(default_opt_dbl, 4.6); - - //However, even though the "bar" option *looks* like - //a double (56.8) it is still picked up from the yaml - //file as a string ("56.8"), and therefore asking for - //the "bar" option as a string should return the option - //value that was found in the file *as a string*. - default_opt_str = opts->getOptionValue( - "bar", "hello"); - EXPECT_EQUAL(default_opt_str, "56.8"); - - //Test one of the utility free functions for various types of - //FeatureConfiguration pointers: raw, shared_ptr, unique_ptr - { - sparta::app::FeatureConfiguration * feature_cfg = nullptr; - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg = new sparta::app::FeatureConfiguration; - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg->setFeatureValue("simdb", 5); - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 5)); - - delete feature_cfg; - } - { - std::shared_ptr feature_cfg; - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg.reset(new sparta::app::FeatureConfiguration); - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg->setFeatureValue("simdb", 5); - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 5)); - } - { - std::unique_ptr feature_cfg; - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg.reset(new sparta::app::FeatureConfiguration); - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 0)); - - feature_cfg->setFeatureValue("simdb", 5); - EXPECT_TRUE(sparta::IsFeatureValueEqualTo(feature_cfg, "simdb", 5)); - } -} - int main(int argc, char **argv) { - testFeatureConfig(); - // Defaults for command line simulator sparta::app::DefaultValues DEFAULTS; #ifndef __APPLE__ diff --git a/sparta/test/PairCollector/CMakeLists.txt b/sparta/test/PairCollector/CMakeLists.txt deleted file mode 100644 index eed083c14c..0000000000 --- a/sparta/test/PairCollector/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -project(Collectable_test) -sparta_add_test_executable(Collectable_test Collectable_test.cpp) -sparta_test(Collectable_test Collectable_test_RUN) diff --git a/sparta/test/PairCollector/Collectable_test.cpp b/sparta/test/PairCollector/Collectable_test.cpp deleted file mode 100644 index f7cf4e347a..0000000000 --- a/sparta/test/PairCollector/Collectable_test.cpp +++ /dev/null @@ -1,579 +0,0 @@ - -#include "sparta/utils/SpartaTester.hpp" -#include "sparta/collection/PipelineCollector.hpp" -#include "sparta/collection/Collectable.hpp" -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/simulation/ClockManager.hpp" -#include "sparta/events/Event.hpp" -#include "sparta/events/EventSet.hpp" -#include "sparta/events/StartupEvent.hpp" -#include "sparta/log/Tap.hpp" -#include "sparta/pairs/SpartaKeyPairs.hpp" -#include "yaml-cpp/yaml.h" - -/* - * This test basically tests the Flattening of Nested Pairs in Collectable - * classes while collection is on. The purpose of this test is to be certain - * that the Flattening runs correctly on any depth of Nested Classes. This - * test is also required to make sure that not only we can flatten nested - * pairs, but also, we are collecting the correct values of the Nested classes. - * - * For testing that we are flattening the nested pairs and collecting the correct - * values, we are using a Debugging API called sparta::Collectable::dumpNameValuePairs() - * which takes as the only parameter, an object of the type we are collecting and returns - * all the name value pairs we want to collect in a nicely formatted string. - * - * We take that object and register its own name value pairs, and then we start - * the Flattening process, going one level deeper into the Nested classes and - * processing its pairs and so on, going deeper and deeper till hit the base class - * or the standalone class. - * - * For this test, I am flattening nested pairs with a maximum degree of 8 and - * this works of N levels. -*/ - -/* - * This is the Level 1 class or Standalone class. - * This class does not have any Nested Pair Classes - * inside of itself. This is the most basic and simplest - * case of Pair Collection. -*/ -class Level_1_PairDef; -class Level_1{ -public: - using SpartaPairDefinitionType = Level_1_PairDef; - Level_1(uint64_t uid, uint64_t vaddr, uint64_t raddr, - const std::vector& vec) : - uid_(uid), vaddr_(vaddr), raddr_(raddr), vec_(vec) {} - uint64_t getUid() const { return uid_; } - uint64_t getVaddr() const { return vaddr_; } - uint64_t getRaddr() const { return raddr_; } - std::vector getVec() const { return vec_; } -private: - uint64_t uid_; - uint64_t vaddr_; - uint64_t raddr_; - std::vector vec_; -}; -typedef std::shared_ptr Level_1_Ptr; - -class Level_1_PairDef : public sparta::PairDefinition{ -public: - - Level_1_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_1); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("uid", &Level_1::getUid), - SPARTA_ADDPAIR("vaddr", &Level_1::getVaddr), - SPARTA_ADDPAIR("raddr", &Level_1::getRaddr), - SPARTA_ADDPAIR("vector", &Level_1::getVec)) -}; - -inline std::ostream& operator << (std::ostream& os, const std::vector& vec){ - for(const auto& items : vec){ - os << items << " "; - } - return os; -} - -/* - * This is the Level 2 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 1 class, which - * means, Level 1 Class is nested inside this class. So, Level 2 classes - * Name Value pairs consists of its own Name Value pairs + Level 1 Name Value - * pairs. -*/ -class Level_2_PairDef; -class Level_2{ -public: - using SpartaPairDefinitionType = Level_2_PairDef; - enum class TargetUnit : std::uint8_t{ - ALU0, - ALU1, - FPU, - BR, - LSU, - ROB, - N_TARGET_UNITS}; - Level_2(const Level_1_Ptr& ptr, uint32_t latency, - bool complete, Level_2::TargetUnit unit) : - level_1_ptr_(ptr), latency_(latency), - complete_(complete), unit_(unit) {} - - const Level_1_Ptr & getNestedPtr() const { return level_1_ptr_; } - uint32_t getLatency() const { return latency_; } - bool getComplete() const { return complete_; } - const TargetUnit& getUnit() const { return unit_; } -private: - Level_1_Ptr level_1_ptr_ = nullptr; - uint32_t latency_; - bool complete_; - TargetUnit unit_; -}; -typedef std::shared_ptr Level_2_Ptr; - -inline std::ostream & operator<<(std::ostream & os, const Level_2::TargetUnit & unit) { - switch(unit) - { - case Level_2::TargetUnit::ALU0: - os << "ALU0"; - break; - case Level_2::TargetUnit::ALU1: - os << "ALU1"; - break; - case Level_2::TargetUnit::FPU: - os << "FPU"; - break; - case Level_2::TargetUnit::BR: - os << "BR"; - break; - case Level_2::TargetUnit::LSU: - os << "LSU"; - break; - case Level_2::TargetUnit::ROB: - os << "ROB"; - break; - case Level_2::TargetUnit::N_TARGET_UNITS: - os << "ERROR!!!"; - break; - } - return os; -} - -class Level_2_PairDef : public sparta::PairDefinition{ -public: - Level_2_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_2); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("latency", &Level_2::getLatency), - SPARTA_ADDPAIR("complete", &Level_2::getComplete), - SPARTA_FLATTEN(&Level_2::getNestedPtr), - SPARTA_ADDPAIR("unit", &Level_2::getUnit)) -}; - -/* - * This is the Level 3 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 2 class, which - * means, Level 2 Class is nested inside this class. So, Level 3 classes - * Name Value pairs consists of its own Name Value pairs + Level 2 Name Value - * pairs. -*/ -class Level_3_PairDef; -class Level_3{ -public: - using SpartaPairDefinitionType = Level_3_PairDef; - enum class MMUState : std::uint8_t { - NO_ACCESS, - MISS, - HIT, - NUM_STATES}; - enum class CacheState : std::uint8_t { - NO_ACCESS, - MISS, - HIT, - NUM_STATES}; - - Level_3(const Level_2_Ptr& ptr, Level_3::MMUState mmustate, - Level_3::CacheState cachestate) : - level_2_ptr_(ptr), mmustate_(mmustate), - cachestate_(cachestate) {} - const Level_2_Ptr & getNestedPtr() const { return level_2_ptr_; } - const MMUState& getMMUState() const { return mmustate_; } - const CacheState& getCacheState() const { return cachestate_; } -private: - Level_2_Ptr level_2_ptr_ = nullptr; - MMUState mmustate_; - CacheState cachestate_; -}; -typedef std::shared_ptr Level_3_Ptr; - -inline std::ostream & operator<<(std::ostream & os, const Level_3::MMUState & mmuaccessstate){ - switch(mmuaccessstate){ - case Level_3::MMUState::NO_ACCESS: - os << "no_access"; - break; - case Level_3::MMUState::MISS: - os << "miss"; - break; - case Level_3::MMUState::HIT: - os << "hit"; - break; - default: - os << "N/A"; - } - return os; -} - -inline std::ostream & operator<<(std::ostream & os, const Level_3::CacheState & cacheaccessstate){ - switch(cacheaccessstate){ - case Level_3::CacheState::NO_ACCESS: - os << "no_access"; - break; - case Level_3::CacheState::MISS: - os << "miss"; - break; - case Level_3::CacheState::HIT: - os << "hit"; - break; - default: - os << "N/A"; - } - return os; -} - -class Level_3_PairDef : public sparta::PairDefinition{ -public: - Level_3_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_3); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("mmu", &Level_3::getMMUState), - SPARTA_FLATTEN(&Level_3::getNestedPtr), - SPARTA_ADDPAIR("cache", &Level_3::getCacheState)) -}; - -/* - * This is the Level 4 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 3 class, which - * means, Level 3 Class is nested inside this class. So, Level 4 classes - * Name Value pairs consists of its own Name Value pairs + Level 3 Name Value - * pairs. -*/ -class Level_4_PairDef; -class Level_4{ -public: - using SpartaPairDefinitionType = Level_4_PairDef; - enum class IssuePriority : std::uint8_t { - HIGHEST, - CACHE_RELOAD, // Receive mss ack, waiting for cache re-access - CACHE_PENDING, // Wait for another outstanding miss finish - MMU_RELOAD, // Receive for mss ack, waiting for mmu re-access - MMU_PENDING, // Wait for another outstanding miss finish - NEW_DISP, // Wait for new issue - LOWEST, - NUM_OF_PRIORITIES - }; - enum class IssueState : std::uint8_t { - READY, // Ready to be issued - ISSUED, // On the flight somewhere inside Load/Store Pipe - NOT_READY, // Not ready to be issued - NUM_STATES - }; - - Level_4(const Level_3_Ptr& ptr, Level_4::IssuePriority rank, - Level_4::IssueState state) : - level_3_ptr_(ptr), rank_(rank), state_(state) {} - const Level_3_Ptr & getNestedPtr() const { return level_3_ptr_; } - const IssuePriority& getRank() const { return rank_; } - const IssueState& getState() const { return state_; } -private: - Level_3_Ptr level_3_ptr_ = nullptr; - IssuePriority rank_; - IssueState state_; -}; -typedef std::shared_ptr Level_4_Ptr; - -inline std::ostream& operator<<(std::ostream& os, const Level_4::IssuePriority& rank){ - switch(rank){ - case Level_4::IssuePriority::HIGHEST: - os << "highest"; - break; - case Level_4::IssuePriority::CACHE_RELOAD: - os << "$_reload"; - break; - case Level_4::IssuePriority::CACHE_PENDING: - os << "$_pending"; - break; - case Level_4::IssuePriority::MMU_RELOAD: - os << "mmu_reload"; - break; - case Level_4::IssuePriority::MMU_PENDING: - os << "mmu_pending"; - break; - case Level_4::IssuePriority::NEW_DISP: - os << "new_disp"; - break; - case Level_4::IssuePriority::LOWEST: - os << "lowest"; - break; - default: - os << "N/A"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const Level_4::IssueState& state){ - // Print instruction issue state - switch(state){ - case Level_4::IssueState::NOT_READY: - os << "not_ready"; - break; - case Level_4::IssueState::READY: - os << "ready"; - break; - case Level_4::IssueState::ISSUED: - os << "issued"; - break; - default: - os << "N/A"; - } - return os; -} - -class Level_4_PairDef : public sparta::PairDefinition{ -public: - Level_4_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_4); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("rank", &Level_4::getRank), - SPARTA_FLATTEN(&Level_4::getNestedPtr), - SPARTA_ADDPAIR("state", &Level_4::getState)) -}; - -/* - * This is the Level 5 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 4 class, which - * means, Level 4 Class is nested inside this class. So, Level 5 classes - * Name Value pairs consists of its own Name Value pairs + Level 4 Name Value - * pairs. -*/ -class Level_5_PairDef; -class Level_5{ -public: - using SpartaPairDefinitionType = Level_5_PairDef; - enum class Mnemonic : std::uint8_t { - ADC, - CLZ, - ADD, - CMN, - VABA, - CMP, - SUB - }; - - Level_5(const Level_4_Ptr& ptr, Level_5::Mnemonic mnemonic) : - level_4_ptr_(ptr), mnemonic_(mnemonic) {} - const Level_4_Ptr & getNestedPtr() const { return level_4_ptr_; } - const Mnemonic& getMnemonic() const { return mnemonic_; } -private: - Level_4_Ptr level_4_ptr_ = nullptr; - Mnemonic mnemonic_; -}; -typedef std::shared_ptr Level_5_Ptr; - -inline std::ostream& operator<<(std::ostream& os, const Level_5::Mnemonic& mnemonic){ - switch(mnemonic){ - case Level_5::Mnemonic::ADC: - os << "adc"; - break; - case Level_5::Mnemonic::CLZ: - os << "clz"; - break; - case Level_5::Mnemonic::ADD: - os << "add"; - break; - case Level_5::Mnemonic::CMN: - os << "cmn"; - break; - case Level_5::Mnemonic::VABA: - os << "vaba"; - break; - case Level_5::Mnemonic::CMP: - os << "cmp"; - break; - case Level_5::Mnemonic::SUB: - os << "sub"; - break; - default: - os << "N/A"; - } - return os; -} - -class Level_5_PairDef : public sparta::PairDefinition{ -public: - Level_5_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_5); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("mnemonic", &Level_5::getMnemonic), - SPARTA_FLATTEN(&Level_5::getNestedPtr)) -}; - -/* - * This is the Level 6 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 5 class, which - * means, Level 5 Class is nested inside this class. So, Level 6 classes - * Name Value pairs consists of its own Name Value pairs + Level 5 Name Value - * pairs. -*/ -class Level_6_PairDef; -class Level_6{ -public: - using SpartaPairDefinitionType = Level_6_PairDef; - Level_6(const Level_5_Ptr& ptr, uint16_t randomValue) : - level_5_ptr_(ptr), randomValue_(randomValue) {} - const Level_5_Ptr & getNestedPtr() const { return level_5_ptr_; } - const uint16_t& getRandomValue() const { return randomValue_; } -private: - Level_5_Ptr level_5_ptr_ = nullptr; - uint16_t randomValue_; -}; -typedef std::shared_ptr Level_6_Ptr; - -class Level_6_PairDef : public sparta::PairDefinition{ -public: - Level_6_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_6); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("RandomValue", &Level_6::getRandomValue), - SPARTA_FLATTEN(&Level_6::getNestedPtr)) -}; - -/* - * This is the Level 7 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 6 class, ehich - * means, Level 6 Class is nested inside this class. So, Level 7 classes - * Name Value pairs consists of its own Name Value pairs + Level 6 Name Value - * pairs. -*/ -class Level_7_PairDef; -class Level_7{ -public: - using SpartaPairDefinitionType = Level_7_PairDef; - Level_7(const Level_6_Ptr& ptr, uint16_t randomValue, - bool a, uint64_t b) : - level_6_ptr_(ptr), randomValue_(randomValue), - pair_(std::make_pair(a, b)) {} - const Level_6_Ptr & getNestedPtr() const { return level_6_ptr_; } - const uint16_t& getRandomValue() const { return randomValue_; } - const std::pair& getPair() const { return pair_; } -private: - Level_6_Ptr level_6_ptr_ = nullptr; - uint16_t randomValue_; - std::pair pair_; -}; -typedef std::shared_ptr Level_7_Ptr; - -class Level_7_PairDef : public sparta::PairDefinition{ -public: - Level_7_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_7); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("SomeValue", &Level_7::getRandomValue), - SPARTA_FLATTEN(&Level_7::getNestedPtr), - SPARTA_ADDPAIR("ran1", &Level_7::getPair), - SPARTA_ADDPAIR("ran2", &Level_7::getPair)) -}; - -/* - * This is the Level 8 class which contains its own Name Value pairs - * as well as an instance or pointer to instance of Level 7 class, which - * means, Level 7 Class is nested inside this class. So, Level 8 classes - * Name Value pairs consists of its own Name Value pairs + Level 7 Name Value - * pairs. -*/ -class Level_8_PairDef; -class Level_8{ -public: - using SpartaPairDefinitionType = Level_8_PairDef; - Level_8(const Level_7_Ptr& ptr, uint16_t randomValue, - uint32_t a, uint32_t b) : - level_7_ptr_(ptr), randomValue_(randomValue), - pair_(std::make_pair(a, b)) {} - const Level_7_Ptr & getNestedPtr() const { return level_7_ptr_; } - const uint16_t& getRandomValue() const { return randomValue_; } - const std::pair& getPair() const { return pair_; } -private: - Level_7_Ptr level_7_ptr_ = nullptr; - uint16_t randomValue_; - std::pair pair_; -}; -typedef std::shared_ptr Level_3_Ptr; - -class Level_8_PairDef : public sparta::PairDefinition{ -public: - Level_8_PairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(Level_8); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("ArbitaryValue", &Level_8::getRandomValue), - SPARTA_FLATTEN(&Level_8::getNestedPtr), - SPARTA_ADDPAIR("val1", &Level_8::getPair), - SPARTA_ADDPAIR("val2", &Level_8::getPair)) -}; - -int main() -{ - sparta::Scheduler sched; - sparta::RootTreeNode root_node("root"); - sparta::RootTreeNode root_clks("clocks", "Clock Tree Root", root_node.getSearchScope()); - sparta::ClockManager cm(&sched); - sparta::Clock::Handle root_clk = cm.makeRoot(&root_clks); - sparta::Clock::Handle clk_1000000 = cm.makeClock("clk_1000000", root_clk, 1000000.0); - sparta::Clock::Handle clk_100000 = cm.makeClock("clk_100000", root_clk, 100000.0); - sparta::Clock::Handle clk_10000 = cm.makeClock("clk_10000", root_clk, 10000.0); - sparta::Clock::Handle clk_1000 = cm.makeClock("clk_1000", root_clk, 1000.0); - sparta::Clock::Handle clk_100 = cm.makeClock("clk_100", root_clk, 100.0); - sparta::Clock::Handle clk_10 = cm.makeClock("clk_10", root_clk, 10.0); - cm.normalize(); - root_node.setClock(root_clk.get()); - sparta::TreeNode obj1000000_tn(&root_node, "obj1000000", "obj1000000 desc"); - sparta::TreeNode obj100000_tn(&root_node, "obj100000", "obj100000 desc"); - sparta::TreeNode obj10000_tn(&root_node, "obj10000", "obj10000 desc"); - sparta::TreeNode obj1000_tn(&root_node, "obj1000", "obj1000 desc"); - sparta::TreeNode obj100_tn(&root_node, "obj100", "obj100 desc"); - sparta::TreeNode obj10_tn(&root_node, "obj10", "obj10 desc"); - obj1000000_tn.setClock(clk_1000000.get()); - obj100000_tn.setClock(clk_100000.get()); - obj10000_tn.setClock(clk_10000.get()); - obj1000_tn.setClock(clk_1000.get()); - obj100_tn.setClock(clk_100.get()); - obj10_tn.setClock(clk_10.get()); - - std::vector vec {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - std::shared_ptr l_1 {std::make_shared(12, 1024, 4966, vec)}; - sparta::collection::Collectable Level_1_Collector(&obj1000000_tn, "level1_0"); - std::string expectedLogString = "uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) "; - EXPECT_EQUAL(Level_1_Collector.dumpNameValuePairs(*l_1), expectedLogString); - - std::shared_ptr l_2 {std::make_shared(l_1, 4, true, Level_2::TargetUnit::FPU)}; - sparta::collection::Collectable Level_2_Collector(&obj100000_tn, "level2_0"); - expectedLogString = "latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) "; - EXPECT_EQUAL(Level_2_Collector.dumpNameValuePairs(*l_2), expectedLogString); - - std::shared_ptr l_3 {std::make_shared(l_2, Level_3::MMUState::MISS, Level_3::CacheState::HIT)}; - sparta::collection::Collectable Level_3_Collector(&obj10000_tn, "level3_0"); - expectedLogString = "mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) "; - EXPECT_EQUAL(Level_3_Collector.dumpNameValuePairs(*l_3), expectedLogString); - - std::shared_ptr l_4 {std::make_shared(l_3, Level_4::IssuePriority::CACHE_RELOAD, Level_4::IssueState::NOT_READY)}; - sparta::collection::Collectable Level_4_Collector(&obj1000_tn, "level4_0"); - expectedLogString = "rank($_reload) mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) state(not_ready) "; - EXPECT_EQUAL(Level_4_Collector.dumpNameValuePairs(*l_4), expectedLogString); - - std::shared_ptr l_5 {std::make_shared(l_4, Level_5::Mnemonic::ADC)}; - sparta::collection::Collectable Level_5_Collector(&obj100_tn, "level5_0"); - expectedLogString = "mnemonic(adc) rank($_reload) mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) state(not_ready) "; - EXPECT_EQUAL(Level_5_Collector.dumpNameValuePairs(*l_5), expectedLogString); - - std::shared_ptr l_6 {std::make_shared(l_5, 1991)}; - sparta::collection::Collectable Level_6_Collector(&obj10_tn, "level6_0"); - expectedLogString = "RandomValue(1991) mnemonic(adc) rank($_reload) mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) state(not_ready) "; - EXPECT_EQUAL(Level_6_Collector.dumpNameValuePairs(*l_6), expectedLogString); - - std::shared_ptr l_7 {std::make_shared(l_6, 2018, true, 714)}; - sparta::collection::Collectable Level_7_Collector(&obj10_tn, "level7_0"); - expectedLogString = - "SomeValue(2018) RandomValue(1991) mnemonic(adc) rank($_reload) mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) state(not_ready) ran1(true) ran2(714) "; - EXPECT_EQUAL(Level_7_Collector.dumpNameValuePairs(*l_7), expectedLogString); - - std::shared_ptr l_8 {std::make_shared(l_7, 2017, 18, 69)}; - sparta::collection::Collectable Level_8_Collector(&obj10_tn, "level8_0"); - expectedLogString = - "ArbitaryValue(2017) SomeValue(2018) RandomValue(1991) mnemonic(adc) rank($_reload) mmu(miss) latency(4) complete(true) uid(12) vaddr(1024) raddr(4966) vector([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) unit(FPU) cache(hit) state(not_ready) ran1(true) ran2(714) val1(18) val2(69) "; - EXPECT_EQUAL(Level_8_Collector.dumpNameValuePairs(*l_8), expectedLogString); - - root_node.enterTeardown(); - root_clks.enterTeardown(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/test/Pipe/Pipe_test.cpp b/sparta/test/Pipe/Pipe_test.cpp index 021934a344..b9f2c7a17f 100644 --- a/sparta/test/Pipe/Pipe_test.cpp +++ b/sparta/test/Pipe/Pipe_test.cpp @@ -10,9 +10,9 @@ #include "sparta/sparta.hpp" #include "sparta/events/EventSet.hpp" #include "sparta/events/PayloadEvent.hpp" +#include "sparta/collection/PipelineCollector.hpp" TEST_INIT -#define PIPEOUT_GEN /* * This test creates a producer and a consumer for two staged pipes. @@ -68,10 +68,9 @@ int main () // sparta::log::Tap scheduler_debug(sparta::TreeNode::getVirtualGlobalNode(), // sparta::log::categories::DEBUG, std::cout); -#ifdef PIPEOUT_GEN pipe1.enableCollection(&rtn); - pipe2.enableCollection(&rtn); -#endif + // TODO cnyce: pipe2.enableCollection(&rtn); + sparta::PayloadEvent ev(&es, "dummy_ev", CREATE_SPARTA_HANDLER_WITH_DATA_WITH_OBJ(sparta::Pipe, &pipe1, push_front, uint32_t)); @@ -79,17 +78,12 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); -#ifdef PIPEOUT_GEN - sparta::collection::PipelineCollector pc("testPipe", 1000000, - root_clk.get(), &rtn); -#endif + sparta::collection::PipelineCollector pc("testPipe", {}, 10, &rtn, nullptr); sched.finalize(); -#ifdef PIPEOUT_GEN EXPECT_THROW(pipe2.resize(5)); EXPECT_EQUAL(pipe2.capacity(), 10); // Make sure it really didn't get resized - pc.startCollection(&rtn); -#endif + pc.startCollecting(); // Check initials EXPECT_EQUAL(pipe1.capacity(), 10); @@ -320,9 +314,7 @@ int main () EXPECT_EQUAL(pipe2.size(), 0); rtn.enterTeardown(); -#ifdef PIPEOUT_GEN - pc.destroy(); -#endif + pc.stopCollecting(); // Returns error if one REPORT_ERROR; diff --git a/sparta/test/Pipeline/Pipeline_test.cpp b/sparta/test/Pipeline/Pipeline_test.cpp index f732196779..e69d6066a5 100644 --- a/sparta/test/Pipeline/Pipeline_test.cpp +++ b/sparta/test/Pipeline/Pipeline_test.cpp @@ -12,9 +12,10 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/simulation/ClockManager.hpp" #include "sparta/utils/SpartaTester.hpp" +#include "sparta/events/PayloadEvent.hpp" +#include "sparta/collection/PipelineCollector.hpp" TEST_INIT -#define PIPEOUT_GEN #define TEST_MANUAL_UPDATE @@ -310,7 +311,7 @@ int main () sparta::PayloadEvent ev_flush_one (&es, "ev_flush_one", CREATE_SPARTA_HANDLER_WITH_DATA_WITH_OBJ(DummyClass2, &dummyObj2, flushOne, uint32_t)); -#ifdef PIPEOUT_GEN +#if 0 // TODO cnyce EXPECT_FALSE(examplePipeline1.isCollected()); examplePipeline1.enableCollection(&rtn); EXPECT_FALSE(examplePipeline1.isCollected()); @@ -323,6 +324,19 @@ int main () examplePipeline8.enableCollection(&rtn); examplePipeline9.enableCollection(&rtn); stwr_pipe.enableCollection(&rtn); +#else + EXPECT_FALSE(examplePipeline1.isCollected()); + examplePipeline1.enableCollection(&rtn); + EXPECT_FALSE(examplePipeline1.isCollected()); + examplePipeline2.enableCollection(&rtn); + examplePipeline3.enableCollection(&rtn); + examplePipeline4.enableCollection(&rtn); + examplePipeline5.enableCollection(&rtn); + examplePipeline6.enableCollection(&rtn); + examplePipeline7.enableCollection(&rtn); + examplePipeline8.enableCollection(&rtn); + examplePipeline9.enableCollection(&rtn); + stwr_pipe.enableCollection(&rtn); #endif //////////////////////////////////////////////////////////////////////////////// @@ -541,10 +555,7 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); -#ifdef PIPEOUT_GEN - sparta::collection::PipelineCollector pc("examplePipeline1", 1000000, root_clk.get(), &rtn); -#endif - + sparta::collection::PipelineCollector pc("examplePipeline1", {}, 10, &rtn, nullptr); //////////////////////////////////////////////////////////////////////////////// // Pipeline stage handling event precedence setup @@ -668,12 +679,9 @@ int main () sched.finalize(); -#ifdef PIPEOUT_GEN EXPECT_FALSE(examplePipeline1.isCollected()); - pc.startCollection(&rtn); + pc.startCollecting(); EXPECT_TRUE(examplePipeline1.isCollected()); -#endif - //////////////////////////////////////////////////////////////////////////////// // Pipeline Forward Progression Test @@ -1627,10 +1635,7 @@ int main () std::cout << "[FINISH] Pipeline Data Event Handling Test\n"; rtn.enterTeardown(); - -#ifdef PIPEOUT_GEN - pc.destroy(); -#endif + pc.stopCollecting(); // Returns error if one REPORT_ERROR; diff --git a/sparta/test/Port/Port_test.cpp b/sparta/test/Port/Port_test.cpp index 5c028d2e58..f50a22e140 100644 --- a/sparta/test/Port/Port_test.cpp +++ b/sparta/test/Port/Port_test.cpp @@ -123,9 +123,6 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testPipe", 1000000, - root_clk.get(), &rtn); - sched.finalize(); #ifdef TEST_PIPEOUT_COLLECTION diff --git a/sparta/test/Queue/Queue_test.cpp b/sparta/test/Queue/Queue_test.cpp index 67be7550f2..3d0a4d5d5a 100644 --- a/sparta/test/Queue/Queue_test.cpp +++ b/sparta/test/Queue/Queue_test.cpp @@ -17,9 +17,9 @@ #include "sparta/statistics/CycleCounter.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" -TEST_INIT +#include "sparta/collection/PipelineCollector.hpp" -#define PIPEOUT_GEN +TEST_INIT void testIteratorValidity(); void testIteratorValidity2(); @@ -63,23 +63,15 @@ int main() sparta::Queue dummy_struct_queue_alloc("dummy_struct_queue_alloc", 5, root_clk.get(), &queue10_stats); rtn.setClock(root_clk.get()); - -#ifdef PIPEOUT_GEN queue10_untimed.enableCollection(&rtn); -#endif rtn.enterConfiguring(); rtn.enterFinalized(); -#ifdef PIPEOUT_GEN - sparta::collection::PipelineCollector pc("testPipe", 1000000, root_clk.get(), &rtn); -#endif - + sparta::collection::PipelineCollector pc("testPipe", {}, 10, &rtn, nullptr); sched.finalize(); -#ifdef PIPEOUT_GEN - pc.startCollection(&rtn); -#endif + pc.startCollecting(); //////////////////////////////////////////////////////////// sched.run(1); @@ -389,9 +381,7 @@ int main() testPopBack(); rtn.enterTeardown(); -#ifdef PIPEOUT_GEN - pc.destroy(); -#endif + pc.stopCollecting(); REPORT_ERROR; return ERROR_CODE; diff --git a/sparta/test/ReportVerifier/CMakeLists.txt b/sparta/test/ReportVerifier/CMakeLists.txt deleted file mode 100644 index f6763e0268..0000000000 --- a/sparta/test/ReportVerifier/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -project(ReportVerifier_test) - -sparta_add_test_executable(ReportVerifier_test ReportVerifier.cpp) - -sparta_test(ReportVerifier_test ReportVerifier_test_RUN) diff --git a/sparta/test/ReportVerifier/ReportVerifier.cpp b/sparta/test/ReportVerifier/ReportVerifier.cpp deleted file mode 100644 index 19343d0259..0000000000 --- a/sparta/test/ReportVerifier/ReportVerifier.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/*! - * \file ReportVerifier.cpp - * \brief Test for ReportVerifier functionality - */ - -#include "sparta/report/db/ReportVerifier.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/TableRef.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "sparta/report/db/Schema.hpp" -#include "sparta/utils/SpartaTester.hpp" - -#include -#include - -TEST_INIT - -using sparta::SpartaTester; -namespace db = sparta::db; - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -std::string makeLoremText() -{ - std::ostringstream lorem; - lorem << "Lorem ipsum dolor sit amet, consectetur adipiscing elit, " << std::endl; - lorem << "sed do eiusmod tempor incididunt ut labore et dolore magna " << std::endl; - lorem << "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " << std::endl; - lorem << "ullamco laboris nisi ut aliquip ex ea commodo consequat. " << std::endl; - lorem << "Duis aute irure dolor in reprehenderit in voluptate velit " << std::endl; - lorem << "esse cillum dolore eu fugiat nulla pariatur. Excepteur sint " << std::endl; - lorem << "occaecat cupidatat non proident, sunt in culpa qui officia " << std::endl; - lorem << "deserunt mollit anim id est laborum. " << std::endl; - return lorem.str(); -} - -void spartaTesterEquivalentFiles() -{ - PRINT_ENTER_TEST - - std::ostringstream cerr; - std::unique_ptr tester = - SpartaTester::makeTesterWithUserCError(cerr); - - const std::string fname1 = "lorem1.txt"; - const std::string fname2 = "lorem2.txt"; - const std::string header = "# foo=5,bar=asdf"; - - { - std::ofstream fout(fname1); - fout << header << "\n"; - fout << makeLoremText(); - } - - { - std::ofstream fout(fname2); - fout << header << "\n"; - fout << makeLoremText(); - } - - tester->expectFilesEqual(fname1, fname2, true, __LINE__, __FILE__, false); - EXPECT_TRUE(cerr.str().empty()); - EXPECT_EQUAL(SpartaTester::getErrorCode(tester.get()), 0); -} - -void spartaTesterDifferentFiles() -{ - PRINT_ENTER_TEST - - std::ostringstream cerr; - std::unique_ptr tester = - SpartaTester::makeTesterWithUserCError(cerr); - - const std::string fname1 = "lorem1.txt"; - const std::string fname2 = "lorem2.txt"; - const std::string header = "# foo=5,bar=asdf"; - - { - std::ofstream fout(fname1); - fout << header << "\n"; - fout << makeLoremText(); - } - - { - std::ofstream fout(fname2); - std::string lorem = makeLoremText(); - boost::replace_all(lorem, "aliqua", "aliquip"); - boost::replace_all(lorem, "consequat", "consectetur"); - fout << header << "\n"; - fout << lorem; - } - - tester->expectFilesEqual(fname1, fname2, true, __LINE__, __FILE__, false); - EXPECT_FALSE(cerr.str().empty()); - EXPECT_NOTEQUAL(SpartaTester::getErrorCode(tester.get()), 0); -} - -void testVerificationTables() -{ - PRINT_ENTER_TEST - - simdb::ObjectManager obj_mgr("."); - - simdb::Schema schema; - db::buildSimulationDatabaseSchema(schema); - - std::unique_ptr db_proxy(new simdb::SQLiteConnProxy); - obj_mgr.createDatabaseFromSchema(schema, std::move(db_proxy)); - - const std::string dest_file = "AccuracyCheckedDBs/abcd-1234/out2.csv"; - const simdb::DatabaseID sim_info_id = 14; - const bool passed = false; - const bool is_timeseries = true; - - auto verif_tbl = obj_mgr.getTable("ReportVerificationResults"); - - verif_tbl->createObjectWithArgs("DestFile", dest_file, - "SimInfoID", sim_info_id, - "Passed", (int)passed, - "IsTimeseries", (int)is_timeseries); - - simdb::ObjectQuery query(obj_mgr, "ReportVerificationResults"); - query.addConstraints("Passed", simdb::constraints::equal, 0); - EXPECT_EQUAL(query.countMatches(), 1); - - std::string record_dest_file; - simdb::DatabaseID record_sim_info_id; - int record_passed; - int record_is_timeseries; - - query.writeResultIterationsTo("DestFile", &record_dest_file, - "SimInfoID", &record_sim_info_id, - "Passed", &record_passed, - "IsTimeseries", &record_is_timeseries); - - EXPECT_EQUAL(query.countMatches(), 1); - query.executeQuery()->getNext(); - - EXPECT_EQUAL(record_dest_file, dest_file); - EXPECT_EQUAL(record_sim_info_id, sim_info_id); - EXPECT_EQUAL(record_passed, (int)passed); - EXPECT_EQUAL(record_is_timeseries, (int)is_timeseries); -} - -int main() -{ - spartaTesterEquivalentFiles(); - spartaTesterDifferentFiles(); - testVerificationTables(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/test/SimDB/CMakeLists.txt b/sparta/test/SimDB/CMakeLists.txt deleted file mode 100644 index 5da84a6a41..0000000000 --- a/sparta/test/SimDB/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -project(SimDB_test) - -sparta_add_test_executable(SimDB_test SimDB_test.cpp) - -sparta_copy(SimDB_test *.db) - -sparta_test(SimDB_test SimDB_test_RUN) diff --git a/sparta/test/SimDB/SimDB_test.cpp b/sparta/test/SimDB/SimDB_test.cpp deleted file mode 100644 index 81da6d3620..0000000000 --- a/sparta/test/SimDB/SimDB_test.cpp +++ /dev/null @@ -1,675 +0,0 @@ -/*! - * \file SimDB_test.cpp - * - * \brief Tests functionality of SPARTA's statistics database - */ - -#include "sparta/report/db/ReportHeader.hpp" -#include "sparta/report/db/ReportTimeseries.hpp" -#include "sparta/report/db/Schema.hpp" -#include "sparta/simulation/ClockManager.hpp" -#include "simdb/ObjectManager.hpp" -#include "simdb/ObjectRef.hpp" -#include "simdb/impl/sqlite/SQLiteConnProxy.hpp" -#include "simdb/utils/ObjectQuery.hpp" -#include "sparta/report/Report.hpp" -#include "sparta/utils/ValidValue.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "sparta/utils/SpartaTester.hpp" - -#include -#include - -TEST_INIT - -//! Put all the temporary .db files into one directory -//! right in the build folder where tests are run from. -//! We will delete them all at the end. -const std::string DB_DIR = "./temp_dbs"; - -//! RAII for creating and deleting the temp db directory -struct DirDeleter { - DirDeleter() { - std::filesystem::create_directories(DB_DIR); - } - ~DirDeleter() { - std::filesystem::remove_all(DB_DIR); - } -}; - -#define ScopedDatabaseDir \ - DirDeleter deleter; \ - (void) deleter; - -//! ostream operator so we can do EXPECT_EQUAL(vec1, vec2) -template -inline std::ostream & operator<<(std::ostream & os, - const std::vector & data) -{ - if (data.empty()) { - return os; - } - - if (data.size() == 1) { - os << "[" << data[0] << "]"; - return os; - } - - //These vectors can be too long to really print to - //stdout in a useful way... let's just truncate to - //something like "[6.5, 3.4, 5.6, 7.8, 1.2, ...]" - // - //If the vector is less than or equal to 5 elements, - //they will all get printed out. - const size_t num_data_to_print = std::min(5ul, data.size()); - std::ostringstream oss; - oss << "["; - for (size_t idx = 0; idx < num_data_to_print-1; ++idx) { - oss << data[idx] << ","; - } - oss << data[num_data_to_print-1]; - if (data.size() > num_data_to_print) { - oss << ",..."; - } - oss << "]"; - os << oss.str(); - return os; -} - -#define PRINT_ENTER_TEST \ - std::cout << std::endl; \ - std::cout << "*************************************************************" \ - << "*** Beginning '" << __FUNCTION__ << "'" \ - << "*************************************************************" \ - << std::endl; - -void testReportHeaders() -{ - PRINT_ENTER_TEST - - namespace db = sparta::db; - - simdb::ObjectManager obj_mgr(DB_DIR); - - //Before opening the database, verify that getDatabaseFile() - //just returns the DB_DIR path. - EXPECT_EQUAL(obj_mgr.getDatabaseFile(), DB_DIR); - - simdb::Schema si_schema; - db::buildSimulationDatabaseSchema(si_schema); - - std::unique_ptr db_proxy(new simdb::SQLiteConnProxy); - obj_mgr.createDatabaseFromSchema(si_schema, std::move(db_proxy)); - - //The database file should now be set, and not just DB_DIR - EXPECT_NOTEQUAL(obj_mgr.getDatabaseFile(), DB_DIR); - EXPECT_FALSE(obj_mgr.getDatabaseFile().empty()); - - const std::string report_name = "simple_stats.yaml on _SPARTA_global_node_"; - const uint64_t start_time = 1200; - - //Use an end time that is large enough to overflow int64_t (signed). - //SQLite does not support uint64_t out of the box, so we have to cast - //to and from int64_t to overcome this. - const uint64_t end_time = std::numeric_limits::max(); - - const std::string start_trigger_expr = - "top.core0.rob.stats.total_number_retired >= 1200"; - - const std::string update_trigger_expr = - "notif.stats_profiler == 10"; - - const std::string stop_trigger_expr = - "notif.shutdown == 100"; - - const std::string dest_file = "foo.csv"; - - const std::string si_locations = "foo,bar,biz,baz"; - - //Ensure the connection from the ReportHeader table - //to the StringMetadata table is working - const std::string user_metadata_name = "UserMetaFoo"; - const std::string user_metadata_value = "OrigValue"; - const std::string user_metadata_overwritten_value = "NewValue"; - - //Make sure hidden metadata can be written/read - const std::string user_metadata_name2 = "MyHiddenFoo"; - const std::string hidden_metadata_name = "__" + user_metadata_name2; - const std::string hidden_metadata_value = "you_cannot_see_me"; - - sparta::utils::ValidValue header_db_id; - - { - //Put the header object in its own scope - db::ReportHeader header(obj_mgr); - - //Save this object's database ID for later... - header_db_id = header.getId(); - - //Populate the values. Let's do this inside a single - //transaction for better performance. - obj_mgr.safeTransaction([&]() { - header.setReportName(report_name); - header.setReportStartTime(start_time); - header.setReportEndTime(end_time); - header.setSourceReportDescDestFile(dest_file); - header.setSourceReportNumStatInsts(4); - header.setCommaSeparatedSILocations(si_locations); - header.setStringMetadata(user_metadata_name, user_metadata_value); - header.setStringMetadata(hidden_metadata_name, hidden_metadata_value); - }); - - //Let's check the string metadata value... and then overwrite it. - EXPECT_EQUAL(header.getStringMetadata(user_metadata_name), - user_metadata_value); - - header.setStringMetadata( - user_metadata_name, user_metadata_overwritten_value); - } - - { - //The previous header object is destroyed, but the - //database is still open. We should be able to connect - //to it again using the database ID that we got earlier. - - //Start by getting a wrapper around the row in the - //header table. - std::unique_ptr obj_ref = - obj_mgr.findObject("ReportHeader", header_db_id); - - //Now give that object reference to a ReportHeader - //object, who will give us more friendly read API's - //around the record's property values. - db::ReportHeader header(std::move(obj_ref)); - - //Verify the values are all correct - EXPECT_EQUAL(header.getReportName(), report_name); - EXPECT_EQUAL(header.getReportStartTime(), start_time); - EXPECT_EQUAL(header.getReportEndTime(), end_time); - EXPECT_EQUAL(header.getSourceReportDescDestFile(), dest_file); - EXPECT_EQUAL(header.getCommaSeparatedSILocations(), si_locations); - EXPECT_EQUAL(header.getStringMetadata(user_metadata_name), - user_metadata_overwritten_value); - EXPECT_TRUE(header.getStringMetadata("nonexistent").empty()); - - auto all_metadata = header.getAllStringMetadata(); - EXPECT_TRUE(all_metadata.find(user_metadata_name2) == all_metadata.end()); - - auto all_hidden_metadata = header.getAllHiddenStringMetadata(); - auto hidden_iter = all_hidden_metadata.find(user_metadata_name2); - EXPECT_TRUE(hidden_iter != all_hidden_metadata.end()); - EXPECT_EQUAL(hidden_iter->second, hidden_metadata_value); - - //Ensure that the hidden metadata does not contain anything *but* - //the one hidden value we added to this report header - all_hidden_metadata.erase(hidden_iter); - EXPECT_TRUE(all_hidden_metadata.empty()); - } -} - -void testReportTimeseries() -{ - PRINT_ENTER_TEST - - namespace db = sparta::db; - - simdb::ObjectManager obj_mgr(DB_DIR); - - simdb::Schema si_schema; - db::buildSimulationDatabaseSchema(si_schema); - - std::unique_ptr db_proxy(new simdb::SQLiteConnProxy); - obj_mgr.createDatabaseFromSchema(si_schema, std::move(db_proxy)); - - sparta::utils::ValidValue timeseries1_id; - sparta::utils::ValidValue timeseries2_id; - - //Create a few random SI values vectors and time values - //to go with them. These exist outside the scope of the - //writer code below so we can use them to verify the - //data values were written to & retrieved from the - //database correctly. - const uint32_t num_stat_insts_in_timeseries1 = rand() % 2000 + 50; - const uint32_t num_stat_insts_in_timeseries2 = rand() % 2000 + 50; - - auto random_si_values = [=](const uint32_t num_pts) { - std::vector si_values(num_pts); - for (auto & val : si_values) { - val = rand() * M_PI; - } - return si_values; - }; - - //Create some header metadata - const std::string report1_name = "MyFirstTimeseriesReport"; - const std::string report2_name = "MySecondTimeseriesReport"; - - //Pick random SI values and some time values to go with them... - - //...Timeseries 1 - const std::vector ts1_si1_values = - random_si_values(num_stat_insts_in_timeseries1); - - const std::vector ts1_si2_values = - random_si_values(num_stat_insts_in_timeseries1); - - const std::vector ts1_si3_values = - random_si_values(num_stat_insts_in_timeseries1); - - //...Timeseries 2 - const std::vector ts2_si1_values = - random_si_values(num_stat_insts_in_timeseries2); - - const std::vector ts2_si2_values = - random_si_values(num_stat_insts_in_timeseries2); - - const std::vector ts2_si3_values = - random_si_values(num_stat_insts_in_timeseries2); - - //Typically, a timeseries report will have evenly spaced - //"time values" since report updates are usually captured - //on a counter trigger, cycle trigger, or time trigger. - //These fire at regular intervals. - // - //However, timeseries reports can also be generated using - //sparta::NotificationSource's as the update trigger, and for - //those types of reports the update rate is essentially - //random. - // - //In order to address all use cases, we store the simulated - //picoseconds (from the Scheduler) as well as the current - //cycle (from the root Clock) for each SI blob we write to - //the database. - // - //These four values are all part of the TimeseriesChunk - //index for fast retrieval later on. - const uint64_t sim_picoseconds_time1 = 130; - const uint64_t sim_picoseconds_time2 = 920; - const uint64_t sim_picoseconds_time3 = 1835; - - const uint64_t root_clk_cur_cycles_time1 = 3450; - const uint64_t root_clk_cur_cycles_time2 = 9004; - const uint64_t root_clk_cur_cycles_time3 = 12408; - - { - //Create two timeseries objects - db::ReportTimeseries ts1(obj_mgr); - db::ReportTimeseries ts2(obj_mgr); - - //Save these objects' database IDs for later... - timeseries1_id = ts1.getId(); - timeseries2_id = ts2.getId(); - - //Populate the values. Note that we are NOT doing all three - //commands in one SQL statement (ObjectManager::safeTransaction) - //because real simulations will be feeding data into the database - //periodically or even asynchronously. Building up all of that - //pending data inside a single transaction / commit in the hopes - //of faster runtime performance would cause memory problems or - //even exhaust memory entirely in the worst case. - - //Timeseries 1, SI vector 1 - ts1.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time1, root_clk_cur_cycles_time1, ts1_si1_values, - db::MajorOrdering::ROW_MAJOR); - - //Timeseries 2, SI vector 1 - ts2.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time1, root_clk_cur_cycles_time1, ts2_si1_values, - db::MajorOrdering::ROW_MAJOR); - - //Timeseries 1, SI vector 2 - ts1.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time2, root_clk_cur_cycles_time2, ts1_si2_values, - db::MajorOrdering::ROW_MAJOR); - - //Timeseries 1, SI vector 3 - ts1.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time3, root_clk_cur_cycles_time3, ts1_si3_values, - db::MajorOrdering::ROW_MAJOR); - - //Timeseries 2, SI vector 2 - ts2.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time2, root_clk_cur_cycles_time2, ts2_si2_values, - db::MajorOrdering::ROW_MAJOR); - - //Timeseries 2, SI vector 3 - ts2.writeStatisticInstValuesAtTimeT( - sim_picoseconds_time3, root_clk_cur_cycles_time3, ts2_si3_values, - db::MajorOrdering::ROW_MAJOR); - - //Verify an exception is thrown if we attempt to write - //SI values at time "t" that is larger than int64 max. - EXPECT_THROW(ts2.writeStatisticInstValuesAtTimeT( - std::numeric_limits::max(), - std::numeric_limits::max(), - ts2_si3_values, - db::MajorOrdering::ROW_MAJOR)); - - //********************************************************** - //There is a unit test dedicated to the ReportHeader object. - //But let's add a little bit of header data through the - //timeseries object anyway. This will test that the connection - //between the header object (table) and timeseries object - //(another table) is working. - ts1.getHeader().setReportName(report1_name); - ts2.getHeader().setReportName(report2_name); - - //TODO: This piece of metadata is needed to decompress SI data. - //Find another way to decompress blobs without requiring this. - //We aren't even compressing blobs in this unit test, so this - //at least should not be required if compression is not even - //enabled. - ts1.getHeader().setSourceReportNumStatInsts( - num_stat_insts_in_timeseries1); - ts2.getHeader().setSourceReportNumStatInsts( - num_stat_insts_in_timeseries2); - } - - { - //The previous timeseries objects are destroyed, but the - //database is still open. We should be able to connect - //to these timeseries objects again using the database - //ID's that we got earlier. - - //Start by getting wrappers around the rows in the - //timeseries table. - std::unique_ptr obj_ref1 = - obj_mgr.findObject("Timeseries", timeseries1_id); - - std::unique_ptr obj_ref2 = - obj_mgr.findObject("Timeseries", timeseries2_id); - - //Now give those object references to ReportTimeseries - //objects, who will give us more friendly read API's - //around the SI values. - db::ReportTimeseries disk_ts1(std::move(obj_ref1)); - db::ReportTimeseries disk_ts2(std::move(obj_ref2)); - - std::vector> timeseries1_si_chunks; - std::vector> timeseries2_si_chunks; - - //Get all data from [time1,time3] (** simulated picoseconds **) - disk_ts1.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time1, sim_picoseconds_time3, timeseries1_si_chunks); - - disk_ts2.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time1, sim_picoseconds_time3, timeseries2_si_chunks); - - EXPECT_EQUAL(timeseries1_si_chunks.size(), 3); - EXPECT_EQUAL(timeseries1_si_chunks[0], ts1_si1_values); - EXPECT_EQUAL(timeseries1_si_chunks[1], ts1_si2_values); - EXPECT_EQUAL(timeseries1_si_chunks[2], ts1_si3_values); - - EXPECT_EQUAL(timeseries2_si_chunks.size(), 3); - EXPECT_EQUAL(timeseries2_si_chunks[0], ts2_si1_values); - EXPECT_EQUAL(timeseries2_si_chunks[1], ts2_si2_values); - EXPECT_EQUAL(timeseries2_si_chunks[2], ts2_si3_values); - - //Get all data from [time2,time3] (** simulated picoseconds **) - timeseries1_si_chunks.clear(); - timeseries1_si_chunks.shrink_to_fit(); - - timeseries2_si_chunks.clear(); - timeseries2_si_chunks.shrink_to_fit(); - - disk_ts1.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time2, sim_picoseconds_time3, timeseries1_si_chunks); - - disk_ts2.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time2, sim_picoseconds_time3, timeseries2_si_chunks); - - EXPECT_EQUAL(timeseries1_si_chunks.size(), 2); - EXPECT_EQUAL(timeseries1_si_chunks[0], ts1_si2_values); - EXPECT_EQUAL(timeseries1_si_chunks[1], ts1_si3_values); - - EXPECT_EQUAL(timeseries2_si_chunks.size(), 2); - EXPECT_EQUAL(timeseries2_si_chunks[0], ts2_si2_values); - EXPECT_EQUAL(timeseries2_si_chunks[1], ts2_si3_values); - - //Get all data from [time2] (** simulated picoseconds **) - timeseries1_si_chunks.clear(); - timeseries1_si_chunks.shrink_to_fit(); - - timeseries2_si_chunks.clear(); - timeseries2_si_chunks.shrink_to_fit(); - - disk_ts1.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time2, sim_picoseconds_time2, timeseries1_si_chunks); - - disk_ts2.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time2, sim_picoseconds_time2, timeseries2_si_chunks); - - EXPECT_EQUAL(timeseries1_si_chunks.size(), 1); - EXPECT_EQUAL(timeseries1_si_chunks[0], ts1_si2_values); - - EXPECT_EQUAL(timeseries2_si_chunks.size(), 1); - EXPECT_EQUAL(timeseries2_si_chunks[0], ts2_si2_values); - - //Try to get any data from **outside the timeseries range entirely** - timeseries1_si_chunks.clear(); - timeseries1_si_chunks.shrink_to_fit(); - - timeseries2_si_chunks.clear(); - timeseries2_si_chunks.shrink_to_fit(); - - disk_ts1.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time3 + 5000, - sim_picoseconds_time3 + 10000, - timeseries1_si_chunks); - - disk_ts2.getStatisticInstValuesBetweenSimulatedPicoseconds( - sim_picoseconds_time3 + 5000, - sim_picoseconds_time3 + 10000, - timeseries2_si_chunks); - - EXPECT_TRUE(timeseries1_si_chunks.empty()); - EXPECT_TRUE(timeseries2_si_chunks.empty()); - - //Run a query to get some SI data between two root clock cycles, - //instead of between two simulated picoseconds. - timeseries1_si_chunks.clear(); - timeseries1_si_chunks.shrink_to_fit(); - - timeseries2_si_chunks.clear(); - timeseries2_si_chunks.shrink_to_fit(); - - //root clock current cycles @ time1 -> @ time2 - // [------- , -------] - disk_ts1.getStatisticInstValuesBetweenRootClockCycles( - root_clk_cur_cycles_time1, root_clk_cur_cycles_time2, timeseries1_si_chunks); - - EXPECT_EQUAL(timeseries1_si_chunks.size(), 2); - EXPECT_EQUAL(timeseries1_si_chunks[0], ts1_si1_values); - EXPECT_EQUAL(timeseries1_si_chunks[1], ts1_si2_values); - - //root clock current cycles @ time2 -> @ time3 - // [------- , -------] ** notice this range - // is different ** - disk_ts2.getStatisticInstValuesBetweenRootClockCycles( - root_clk_cur_cycles_time2, root_clk_cur_cycles_time3, timeseries2_si_chunks); - - EXPECT_EQUAL(timeseries2_si_chunks.size(), 2); - EXPECT_EQUAL(timeseries2_si_chunks[0], ts2_si2_values); - EXPECT_EQUAL(timeseries2_si_chunks[1], ts2_si3_values); - - //Verify that no exception is thrown if we attempt to read - //SI values at time "t" that is larger than int64 max. This - //should return *empty* SI vectors, but it should not throw. - EXPECT_NOTHROW(disk_ts2.getStatisticInstValuesBetweenSimulatedPicoseconds( - std::numeric_limits::max(), - std::numeric_limits::max(), - timeseries2_si_chunks)); - EXPECT_TRUE(timeseries2_si_chunks.empty()); - - EXPECT_NOTHROW(disk_ts2.getStatisticInstValuesBetweenRootClockCycles( - std::numeric_limits::max(), - std::numeric_limits::max(), - timeseries2_si_chunks)); - EXPECT_TRUE(timeseries2_si_chunks.empty()); - - //********************************************************** - //Verify the header data is correct. This is mostly testing - //that the connection between the timeseries table and the - //header table is working. All the individual metadata tests - //are in a different unit test. - EXPECT_EQUAL(disk_ts1.getHeader().getReportName(), report1_name); - EXPECT_EQUAL(disk_ts2.getHeader().getReportName(), report2_name); - } -} - -void testReportCreationFromSimDB() -{ - PRINT_ENTER_TEST - - //Connect to 'sample.db' and create all non-timeseries reports - //from a root-level report node we find in this database. This - //is for smoke testing only, and does not validate the contents - //of the resulting report files. - simdb::ObjectManager obj_mgr(DB_DIR); - EXPECT_TRUE(obj_mgr.connectToExistingDatabase("sample.db")); - - simdb::ObjectQuery query(obj_mgr, "ReportNodeHierarchy"); - query.addConstraints("ParentNodeID", simdb::constraints::equal, 0); - - int report_db_id = -1; - query.writeResultIterationsTo("Id", &report_db_id); - - auto result_iter = query.executeQuery(); - EXPECT_TRUE(result_iter != nullptr); - EXPECT_TRUE(result_iter->getNext()); - EXPECT_TRUE(report_db_id > 0); - - sparta::Scheduler sched; - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.json", "json", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.reduced.json", "json_reduced", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.detail.json", "json_detail", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.js.json", "js_json", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.html", "html", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.txt", "txt", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.py", "python", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.gnu", "gnuplot", &sched)); - - EXPECT_TRUE(sparta::Report::createFormattedReportFromDatabase( - obj_mgr, report_db_id, - "test.stats.mapping.json", "stats_mapping", &sched)); -} - -void testClockHierarchies() -{ - using sparta::Clock; - - sparta::Scheduler sched; - sparta::ClockManager m(&sched); - Clock::Handle root_clk = m.makeRoot(); - Clock::Handle clk_12 = m.makeClock("C12", root_clk, 1, 2); - Clock::Handle clk_23 = m.makeClock("C23", root_clk, 2, 3); - m.normalize(); - - simdb::ObjectManager obj_mgr(DB_DIR); - simdb::Schema si_schema; - sparta::db::buildSimulationDatabaseSchema(si_schema); - - std::unique_ptr db_proxy(new simdb::SQLiteConnProxy); - obj_mgr.createDatabaseFromSchema(si_schema, std::move(db_proxy)); - - struct ClkData { - std::string name; - double period = 0; - double ratio = 0; - uint32_t freq = 0; - }; - - auto verify_clk_data = [&](const ClkData & actual, - Clock::Handle expected) - { - EXPECT_EQUAL(actual.name, - expected->getName()); - - EXPECT_EQUAL(actual.period, - expected->getPeriod()); - - EXPECT_EQUAL(actual.ratio, - static_cast(expected->getRatio())); - - EXPECT_EQUAL(actual.freq, - static_cast(expected->getFrequencyMhz())); - }; - - auto verify_obj_ref = [&](std::unique_ptr & actual, - Clock::Handle expected) - { - ClkData data; - data.name = actual->getPropertyString("Name"); - data.period = actual->getPropertyDouble("Period"); - data.ratio = actual->getPropertyDouble("RatioToParent"); - data.freq = actual->getPropertyUInt32("FreqMHz"); - verify_clk_data(data, expected); - }; - - auto root_clk_id = root_clk->serializeTo(obj_mgr); - auto root_obj_ref = obj_mgr.findObject("ClockHierarchy", root_clk_id); - verify_obj_ref(root_obj_ref, root_clk); - - simdb::ObjectQuery query(obj_mgr, "ClockHierarchy"); - ClkData data; - - query.addConstraints( - "ParentClockID", - simdb::constraints::equal, - root_clk_id); - - query.writeResultIterationsTo( - "Name", &data.name, - "Period", &data.period, - "RatioToParent", &data.ratio, - "FreqMHz", &data.freq); - - auto result_iter = query.executeQuery(); - - EXPECT_TRUE(result_iter->getNext()); - verify_clk_data(data, clk_12); - - EXPECT_TRUE(result_iter->getNext()); - verify_clk_data(data, clk_23); - - EXPECT_FALSE(result_iter->getNext()); -} - -int main() -{ - ScopedDatabaseDir - srand(time(0)); - - testReportHeaders(); - testReportTimeseries(); - testReportCreationFromSimDB(); - testClockHierarchies(); - - REPORT_ERROR; - return ERROR_CODE; -} diff --git a/sparta/test/SimDB/sample.db b/sparta/test/SimDB/sample.db deleted file mode 100644 index 31ba869b3ecdfd0b78ebde747a29d8d8e19effcc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1572864 zcmeFad3YP;x%WSs(Y|K4lQ@a9$+4s)wrtBzNC?>p+4qEy1SfG4la<&BgtgjtxeK%f zN|)1prIfamQVyjphc2|VryF#EwiHUCEu|Z$htu}`-ZPf0$F?-L&U?MTe}3`R%O#ro ze!ipe=()dhKl2%F-@2ikarw?2o7Zi^U>jF&+7TFKplv;U=TG*pTSeJ{Mqy7cHttxxXGi0z)oYgT z+_0nZ{OP`){^szN<1KZ|mdTN672d2i;3)5LaD!o+&RZLdW4@zRJ$?OadQbMK`x7!?g?`KPQUtf=UWCD%r`gZi5>gmJZrKgYHZF6|)I7{8qrE++!`c`(HK6|eB zwECah&mY>}k3U^sfA8VGO0OHX3U8!R81sE-f8tGNj`V2z-W+z1wbad+A%}alT{_!+ zuD`phHMG6>ABHW0e(80t;F&Z1!$kv)!w%;f?L$xJL38-P7)#yq<#J@TwnM$AkDct< zdG<(me-9n3b*KCKcXgjU-_y6f;ugaiahEFw{$IR@()l{S73X{R4J$78ZRuD4_<_dV-F++1oH~21r+9`mhc|jGblTc5QZ9efmi8Kh@V)UObEst~&8xQ|VvLeJ)Gg%$ag{;jsN4 z+9|`b3VX}kQSHRPvZosduK0Po!l@mt4(+eMjy{W5_V%3XK6m)|#qHuh4PAo%u&vCU z7JmF;QqC3SL8r|IU%WV@wq?p!(A zr|xL^K;{2!Xc2a@d=qmQs~uu(xUwBp&q>M_+(c=b-hAk~jmL1d z_KotAVWD`WMkX;2*wENRyA)_FN~>pCWUiGyo`%@O!4FFD54?R{@xR1R=g{lSohTl) zQT&Hmp$?C_VfBq?&uJ(5sGHT(Z+p+F?$aph!~ep&di#10^`7kQzbI^JZ5jLtG_E`e zDSoV2gwK?nYmjTO#e zj6d{A8@Y$gVZYf@w`h?eeEqQVi{3@?4=A1n#aAyAceZqrm1i>dd+@_ovOAdHhUT`R z{cjH6X0p_^wi=>MWe2fbZ)l+6UW!xm$nDHMTz7!s&vu zMV&{6y{x(v_0L}6ELveC*JX%!v`<^PGo_573RQ8-u&KUc9P_{o9nn&eSBqouUs~zX zN@B6LZdPbcGYneCp%F@ZKgvr+@oXy{qf!oYe@7{$L#Nuhz70LyN4KvN{#5ig+0b*e z|J0ej{<8X7{^8ytn_;aCd(}Tm-}RU|dJgvWTz|gj^x>Xi|5SwcF>K?!)7X~)GMmGb z40JVO2rpB|#FX1}xPN2!>F#4a=la&2UPa}gyRWCx>$0FyDT;Z>it{}#$9Il((N#IF z&DG2E|MyqbmlQ4?flEi=(h<0H1TGzcOGn_+5x8^&E**hON8r*CxO4 zIs%uDz@;N_=?Gjp0+)`!r6X|Z2wXY>myW=tBXH>m{Qu?%*vBhVZG(5>ddDeS4ch8{ zwOgW%)5a>xMu{7|8F>7-F-o(ccd~n)$?2Q0)FoKvFCf9 zZ+RZ~JnDG}eSuGUK8)VLJ)S!~1D>=e>bVtNg6lmeJUyO+p1q#!o{gThp5>m)JRRs8 zG<#-yrg`c-V?7R!$^9?)-`#(9|K9x@_b=T)wQjJ^u)Jt_$nqx3cFPp=@6DezC(WD8 zGfgj<9yUcxJ53G7SB;-Hri`16bCo|TUsLW@jwu1fAb(T-l$?}z$jyd78J;k_+0bKH zfbPYU;)kwdpII^-vl+G?>Fqn)-GBJ_LEN%&zUScie%$%!j|SeOoia)j@?X!)^deS&AXTits6UbG8J0N6*e^Q zU@Ek%Z{3d2gVhh{&NbzuUp03dQr7CFtedhGVXan!tN+Z|1&7a^>uFg~{9CW<*n(~N z>$l+ETHJPX@cfyxqx}7ko zrk-QgKMCtCUiavU=VIq-@knIn{mRD&KlUZq2*b5Ei!j9GnaL)LG%e$^rG^=_T~Q7NZig# zT-v#ctwo0h8^K!OIN4ZMK zzm%=g(b2httEtDY?T&&TM)szGniOkd`+!L(w)I}u4%4sLEwC$!f5rRdEr8A z*=I9&*7U$yY#re&hF6w(vlk#?W%Y!nw)qIcyQws4sU(zD6R$sjB+O#ENvI=pXZaDl zymBxq$ecMo#H_9yqt=eu^P3U1yK#I z*13p7$*pvd)Uk!?Fn#VEq`-w#Pnp&_n=!Ib5gO*sVvO81wRI+AWT6U7ncK)1xpQ*s z493X9%wIouI%DKeUF$T)$ii$tX>J2!~eFq4m&JBcxJ(9;SaS@(PvW^ec035=0jU9IC8BMURObM83C$SscfV-Z$m z#t>&|`^+&2uQFroTw|;EAVi#>kMtY{Dq$P%#+KPOgG>Ej_nbS3i^Gohp2qL?_fEv` zw)d>Y@8G-XmDYEU$M2SVSK;@{=@tLo^osAU75Lr!&Xf2( z@10}uyXhT^@!R|MUi_Z>_A&T9=gv<2o_$9Te$TptUYYqedZqELhw*#HTj`bQZ=qMF zy_sHVcr(2+b%0)(QlM8R7wDDxJiStvqgN*7=#`0CdSya}UKyXESH`8g@Oy0P5PpwI z(JP)Lz2Z(Bz;9Q=gWt|Lz2b=N$8USgjo-E?y<&~fE0zeoVh+ zf4}GbxW2#5bKJAdv(z&K*Y1CEKj;3Y`_t~c`+&RCZFK$I^)Xk}wZ+x!{G0PB=ewNe zohzJE9WOZ^a}*rM9PJL5{U`R%*l)A%vj^;E+plb&v%Sf-&*rm@vHr&T8EgE~^?&uo z#HH*1;^q7(m+m{)T)O@r;W{r}|5xtyjC6RHuK%kypwuHKxWG%-|BNFhIK@lX|BNFh zxXDY`|CKK}FJ1rli=E<2*Z-HU|I63%E?xhZAB{`b|3laRN6(zGj&-3aFvDtamKF@H z4kun3Z#Bqx$vWMEe=M>Yl)j67{XM4+p2O-eo)H7u|L~d9NBS%cc5Ls!_VakO*zm39 z@iuHl%RxEaefrE`ifMusFV4k_r+N>cE52%+U@5(N@?`H|tl}}_)u|+}>B-|v>WDVw zZ13S>uG5Iu>TFg8k2X2cH%7s~W>~Fq-{IpuN6w$@IcJy6l5L{ZN>77ZXfR9GgH~%_ z@2RsV@iy={n!W`WyHA~LJafA7;Px%cx9wPd@YqQ#exJnpZnv|D^K+&OYa2=UB(jaJBz_N5*lzW3OYm!{-=d z|C{~i_Q&m?u)ocI-oD)KvOQz_pzV6wQk%{CL+ksjC#{`U+479#!j84ns)8@)!8@;l|*$|saNl=I3CBGJ-R##XmLFq%GFjiDpD#Axu`k*h2)fJXXtMq|T7%M6)6~Tx} z@ArkVy24Uvk^U_h#;S_EzxojqTYS>{ykV@c$gA~|r1#DYW5Gqfi~`l=hl8ijpE`uU z-I=2YkDff=cf6-hde8JQ)>!0@5i#d_`g_my9FgAb3S)gm9!~+iXAevFP72e~Nt0DB zyMq9?f=Ks{!;^C)c^U#g)_?NQ!Q;LCebT$8hOsUpFSN?iM@#d^f|Gsc7a&A>*Q7A5 zhlpcgqx7!nVXS^XcMHwH-eG=#D4A-6(I2cH=42xDPF9asn$9Qf^ZVOoS}7Iqj93v89%K0Stc)RBdN!IAH% z3)AYth>=^QJLZOI5dr7XR`sIwVEORHCK3+j+b#>!D#G<@-0A+F8~e3efDU5+S`Mmz zn9?mIL$7HuEz(;r3)6x^wd{jMv60ztnHi>ahNV@sr8h4O)55}D#FVVQ_J*{R<}A%t zr+fPs?7*Wi@xZqZX<%WP))?3s?NVW3n3fpW8EsO2VVG7M*cm}7w=hhL4eX3oDZ4OC z>kjOU7AdnZOv?_s83#+pD!njFD-i6Ab}6+mOp6fgj5aB`FidL_?2Mq4SQw_Y3U)@T z6kix7Bg4*Ukz(F3`JAMw0YdDhO!^L4ih8&GSt4gHgD!G`8i#-VfC>wR@M3kj%o+Jq483dlF9#|@AWp}R}U3eOaxRIusLMah< zu=IFOApv)MR7=1*$dP#`5r;b_tHr4{;c)8-MAGM}G<}S`Q&_y4<gF}w zp8cwm+}(1Z`6$vj_c<@x-_gTV5bkoh zruhg{p=)2qVWxu6?Yg@85L2OZZ$~#%L0IqD(|j#cA+)>WAX7oO`(;=20j5I7&W`;| z1)&SJqxl-9Li={Q{@*1%C3&9k-0kW1w7cJMf5&~N`?!0dTXucJ^)A<8SBLWr=eM12 zbM`p>j(<45>bTRf&oSTrXI$}T?fdPsZGW{rY|GoW+vZwdu|8srT6bEfT7GYNz!JBt zw@foXZ~h#v_BWcx<7)rordv%bO%shjHGal;t8uHbN%@QNkWx@~E31|H$`tt}Jpcc3 z`E&An~sDWH0waCz+XM&PORM{jW~h`>|Hxbdap=;CFueO3g1N?ukq!8SPp|0Lt)7lUQ! za&++SmkANrBpJOZgGGIZxNM;Q*SH9rkc zM};ofvJ zFzm=+KWuy$Rskd1q<@rk#9U!m1Q{(c=^tuOY-||L0PoT&{ZFYY<_yCW$Y_g6Z>Wv2 zFuXubov|@tm;`w;F4F&gm>y-`C%rT=40j;UMnB-c4xc=OX50cofuxr_Ve$r(t+K4W zEWOkohAoi0sqav^^EABnO23&GhHH@5>8D9Ac7j8HPV$=>&y>EdAUchGSvr1Q$s^>kPxS z@N|T-EdA6UhI!%XbV)zy48z0lbcEt8J>w6<#qe}GrKdZ?@G(3cp+rkR_J?6*cse2J zN1b7~8J><%sHGqJ!!R^Foet>-ond$yo{mtirSJR0ur)lLcIiJm!*Dh{9pT0V>3jY# z%neVcP5N$U82*N*BX-1}@`qt@csfDpJDp*;9G;HY6aTh945P!-X_cPr4DaLWi1Wd> z{Nby)IxW&SJHvaKI<4Y-@C|=>4^yXgk@WS>@NTY-I3GOW5AWjYbV>iw8Q#g&5$A)) z{ox&4olfa%o#E|V9dSPRsz1Dqs}qvG(iz^$)e+}|$Nb?fT%8W-%bnrPTpe*fc+?-> z#MNn+{=GB2k*g!l2aou}8@M`c(w92J>$y7OeDJV8d=*zGD1EUrypF3Q&Ib?q!&h>3 zTBR>^hSzd+#QESse|Qa7r$zdFD7+fcoP|YkI{2I~yb6h&<;z9VXG7tYOcil5c)%B4 z!BpvzJ`)NrXR3%(!TrAQ6-<>*>C>U`GNy_+5!~ksU(QqsNuLUZmoinvY2cH-@Dip< zhxCb1_%f!7I0<~*7hcR%X_r103NK=+h*Q8v>H2@O^bHBD|G#-oc~*Hm?ibznyHoD% zZlCL~t|wjZaGiE7a*c8R)Oo-2X6G*FT=W2b?Dz;)|F3e?+262#*M67%I{QMqWP9B9 zcHHY9u)b#fiZyB7W1VLC1G)e)%Q}$$&zV1MK5t%awwazaeb{u-w9!FxHS~fyaOXxs7+gZ5zRVq5;un;nsZ>Wldsg# z(G>~tm4s^49*A`CmD=07BJF%7p&|tXkv6_kTWeP&$X61oQcEDx%2x_5?25GTm4wQ) zAP`x|S8AQ#6WT|E!<+Z>=0xTpWu;C^Q+uQdVf4N#tx%OpN?ASf1|nXh(I;0ojUM_i zJ2DsHc;;k8h^w}oCDxr;vm&#QP;EJD^O~7$k(mg> z(;q6Tk5X$In<98Hl73Ug?mj;|uj=x;HW%NxBP&i>3u5(>S9GHKoBD zY2b|BH8mKS${Ag#BU8MQDV)(eCkG>wIim};quv{-=Zqez3r6ZVqYL$7k~cDmGkV9w zU}Pd^bfHE}@J1$ZMsFVS{l&9Tec0cKUzx%X%g}dJM0%-r+U01sNE{F4v&L27-cD~PjEu#@zs3`)l?u*^~Bd_Gz}4Y@e~+Vq0Ttu>RKidE5=K1$}^*Ef1m( zu+=i%{EGPt=C_%5m?xTkXZoJ$Gp4tgE|_+kLU``qKa4*!e%1H^W7ODdTyG2+8;nln z73B%#-Ab>rL{a1)$@j`<l3n4b zv;mfgl0o692q!aOi6~hWjtapNQ8FhS72%`?ED#}n zC8A_Z083Om$3!y&SRzUmWwl-sV2LRC6~GdPL84$ZSRzUW1+c`hm?2mqO4bChgyBOf zEKwIF&jDVb{32eXC8A_6KuZ`tP)18c$xVQkFn~TNKAy2eOVmZlLkt^C!zH3*7HGc& zxS$M|h>}qNE+GaKafwM$as*ZJs36An$&t*Fe=rW)N*Sua(lI;q3NRR0NxUe+*WH+ z`^~|~t+gh#-4u-6QfpH1VlZ-Ztx2sn1|v7sn$&W`w8%vy;r`6g|F{?GBR68Ze`xy< zQ&AteLEJP%Rn$c;U=z@W;;1FAqCRq7Od299CP(_Q31~$zsf?|d9O)aetw>i)ik!o? zCd$(=haz857r7oA)V~u8Qe{X-(9=Gqt75}Sv%3sp9(Kf^Sg7rb`t=7%fiI(43K54n#vf5H- z{<--+to*MvyG*|}-DirL_L{uLSBzgari~Yj2XLi7*=Se(ro5Q%F{-VFBp>#K>3S1g#z@ zTsn{Q#>iRej*k9MjQo`D=)xuQ7;lUmmG0>1|HQ~g>5eX3D!aWga#FgZx4MEc@=v;> z3zx`FZ%lJgI)@X$M2rj+I?NT1yLf4ApA{qTw5)1^ZE}pP5<2{{Vd#pq2u#Gt5Y5H_ z2K94Y1SVo+c>qkvQYppijFF`QFk$$c3QR=FtN@rWyj}^Ih>|m*Qu6xfz(kZx2*eGq z3&2E_YzBY{!|Nh25hW)v_nDhbD475d6MAVNCZgmDKulB%EfvAC`UzqpO0Iz}>Z)`TOQ{+$ z5hXW47Y|iZP(MLTM9Eixn5Y)1)=v-JTvz-Okh*K}7S%u8ue#eAOS_z||pQBD$WdBhCkp`J-2Hbvi&yMAvb3#QES+fAmVO z4iOX4wOk!>K6u0*UBlHOVj{Ylt0T?_5BsC5xH?2kL|1Zk#QESMe{=;`hlq*ja;}az zA3W%fUcuENVj{W>(VP`I;rd$ooG*Gg5;-e!L`+1NGF8OM-~nHB2~&lLiRfia6>%!K z-xpoXR3Ty_x`?SFP6YR9_y6Bl>Hhz3x$bnGbuDw%IbU;r#hG;;ake@qI$m}>;D|Za zIi}d3vws@T{99vp*nVbvpY5Ej(`L1P+xl+nVQahP@0M>`1}q0Hv(5iyehjPr$IL;~ zUrk>(4VbPsU1=I`e9`zxJn?U>af z{h^rV7Z}09DR_-9rr8Bf;=WK!a|?`M;S{{u7t_oFCvk5mrg;TMuy6|Q@x?T&z)9R4 zifK-P5iC@qUA~xR6gY`HLov-KFoK0Tw8IzEYyu~7dnl&41V*q>g|_)(nn~a!ZVkmW zkH82PYS0#6OtT1_#Lc0Y<`5XcLIv97i)jXdlejSy)BFJ=Sg1c6d@;=)a1z%CW8@B~ z(^YMb5&KbBd1GV@sE<}XY2DNqc><6TLQkqhM#QM@AHy(nY9u3KWC74xW+bj%9HXia zGD0V~)X^wwhO7z3$OO>gslJP)WR;AFQT@jvfld+{BCGr{s{S-nR~JxQA}iZtRPwR# zq7yAPMOMsQ*DzyY+e!ddO zh?tMBBvhsafmkzN31mcU9$!hQPJuwIiLV4QBIZT7{){J9DBrx;T%_sGctl3T<{)&~ zx=E3Yh|R_}tR+`msTtm+pEoBq3n?pgQkvRhGZ9AbtI{G}sic(EBX1zqh%}n_bkkHa zA~plzAR{VWp5u1u2kP#Kt zN2xW9O|dD+!jipS){J0mG9syjS2|?s_(FY|?v2$WNq40XWJIivGrCYy8oaSdoY6r> z#3ph^7wX6qZ)^f*bdV9T@to0x+EMR~jpK|CG9osXGrCYOCV69HIHQA%hwds}i~FE^nR|}g=6cojL)VvF?{&poy{^r!cGqO*-<@A` z4mhvI-2i`dJnTp~wmKT@FQN}{lYI;B26)-_pe<(GYMYLy0e%7B__xhE!}2@J6P6U7 z1JGzOnO`(NZvL1#Yrf9B#oTHhV|v~6wCO?9y{6ku*O|7MuE2KzykY!Llr zH0U_#1`DUH^nfo;=7ys}y`VVR8IFo@`bziv;$&<%D%1;#la=A92&b`hpD#|PhND8v zM4W64M@2ZD!A!)-&~Q|UnTV5x;iw3wHJFJwnHi1>F%xmJFB}!Y_JEm)lab-55Hk@c z>%vhHEDxB8IGGrZ3NaILvMn4H!R~;Wh?9Zgs1P#|C(FW75v&fFi8z@TjtVgoak4B} z?XLEz6m1Tei8z^;)p|)_CgS8|z)Z-4+`(wfM4YS(?#3S$GsH~9$+Uo(kl(8@6LE4Q zU5`#7xA=D}b3O-%XX7h?6y-fd#stH1J&wak2q4und4rU_mL=f&` zkyn?TwDzWk-L787OvD{Er4Ta_x7UbDMGZB|-N+D(7wia-V%)6SZflPiP#ObCV`oVU8pq)%tY*btw~@eV*RxyftiT)Aqlr=j{e8JP#-&o z?f#+dWz0nEdT~=3GZ8zBP0d4DL(D|%jF?o$OvFxO6PStOu9Pzqu~Q?qm6(axNo;GP zykX2l>;yKbdoi4uh+QXc8Db`4z1X6@-x172?D&XHMP?#)44c%WI65;CJBsaKCW^Z# zFcYyJY@~)ywKbFB?WE)LVn>ihjV!&iV_mVs2tA<_S}KNRZKb0vu|r6uK36rBQmV>K z#JZ6}4VY>vDl-wg7Ll|ZQY}*TGKYh)gNW5zjuP8UhrF=^NYY!50y9C^|GVk=e@}Ss z@*MQEx?gvH!#&{cb~m~H?t0Yq7T0c9v-4HwW6q>=uXBduHOGUFv}3blru{eehwQi8 zx7zD$zqQ?G3)|M(CR?Ag-fz9Zy3Xpc{1V^xchR!UGRgc)JpVsy-eaC-`jhEl(??8i z#&-a$H!U&EGu4}9!QO{J!x_&V2vq}x3+j4r zg8Y{5=;#Y2$Ytq{E?h`Y@+Qb<>5h)RP=ef*?&!h=^aO8${FUzLASM#zs&q#eE}X}C z6XdIOM+Y&HAUCBux^Tff#+x8Nr8_!^i3GVQ-O+^$Ww$p$K1z3V5EBVS|ws4PQHW+$!nt%6LGR2B>tK}OvK4# zfQ*n|6N!m9`G~>S%ZZ6NSp^Ui*pVVJ5ho)6VnTkcoS2AH`3Er}zororajN_JS(ud2jE92LGcVj@n3AH;BPi8z@JmX6p6|EWJtj)SE`#6)~QS4XJWASUA1aCL~7 zi0|X-2(=r;MEq*54t>W&d@ol=sN(pJiTEC_4t>W&d^cA|sOR{OiTEzA4iOXaom?HE zvV)k2@8IeXF%jR+)e&kuh>7?%t_~3s@vU4Pq1uC(h;QNQ5HS(o%+(R7@0u8ue#JnE0H;OY=D5ns;L5$A(P{P8QeIz&vw zmvME(`QTxH{Bo`i5fkyHTpe*fc*q}L!qp*SB7PZHN1P8H^v4%-b%>aVFG4hDJx;i( zL;9R{|Ns5k`u{~d|9^`6*X~E%1@}?+74EUFpST`yy~%Y1-vA&xzwdmfv)8%6@h`{M z@Ew3xJN))P+8?tg@C<+k+wbwzzo_j>^Z=f%nF@6)P{6~zP%0IEn|4yv(2eHckB(C?5%3X$k8h&Vauc6;? z74HB4nRI{I^8UO8`3!V_lx3y<>qr8CZ11tI1epwUf2xL-OlX;ZJ=&5Wmx1mNFIDaS zB*R*RH zF(W~C0ypVal_+M05}KW0L<{HV0bfFM6P(2Tp@e287{NlZy2h8#yaXq4Unrqj2}ZC` zs;>4WG$+AH+#5=0MuHJ66skSGgythSiMvAy%|5S+xVp@e227{Nk;+Tu%S{(+OYIh4@s10z@{Pn&!R z%{_1uH--|Ld0+$!#c6{tp?L>R;`(5Mtb^VRCN`_C@+QbN=*?g|*Q}eGAj3d7MR~)j zgi|EQ6dc1WHN8fhB0>HDI0Ykd?cxMk0^k%n!J}20HNgZq1Ufv`cd?X=TC=)2LFRxe zNcALCn^pbtH-MKW@US#3(;7U(TNr-&Wd@7PNY#)t}e29&GJQw5JFGsh8Al_ z8N_$R{6q(Gfm7(n4CXHDO0*-M96-kZ9#A z0jEf`@RfuLv^bDh$X5bRkyyZ25~@&FATgh>1e_uf;42B0C=^Ke`AWbk5tFH ziDteMaEin{zLHRpf`LR6UkNxx!pm0@s!~fJF_*6doFXxYuOw8a1%bqDz7lYX#4Ns& zP@MvS#7w>taEe4D!u4l7u|oOgC1xN^f5s!6A~7AI!xmMFI7MO_w$U5gcD9tDI7Iu&fG*j zq6nv`6s1<5*$WbNh^w}!CDxr;vl5e#P;F6*aEio41QAYANqv-B)7X@lfGk?g*U6d@ zOpHe)N_h21p}tJ_CdMI2cNq~lMPe*xbfKm+coSndqjycE>;J2zKT4i&cs}9Dd9L^D z@+|Sp^4Q$JcYn|QdH1{AVRQo4pcC*j*N0u_TuWV6^a0+DK0q732k;5Uy^iY~oerb@ zd-iwRPuRQcCfk47-eWsyTV$JH{h{^k)>F9uzs)+%^19_|d{S}&Kg(Z~^YRY4 z$?ylmmkkNS9>a9(^#9_=pCo^SWeVkP9V(!e+C)jRHCPVOj}~S-=^=lTTnSYjtapPNis1U6~X)frbv>V;iwQy zkt74dQ4x#}V2UJJ8IB6U6iG5K92LRz0H#Qijp3*eOpzqx!ch?n4`7NUSs0EA!4yd{ zEgTi`@08F6(O&AGNB*|M08%%>Kl4K%izXb50#5~;7kR$^Ed_W8+!4yey3ItP>?WPJ- zB*_@izyfwq8u+e;B$)sjSO_S=6iF)m!**GNDUwv*Y26^ zh=;OW9)u~9uEC+RqYiy%L((}ov=C5&DUyy+hbEXJX~!lyo0h4Lt1?WHv>^~M1qF_d zGgO5sl2*O+p}V6;fhm#}qywg?tPYqWX+{uka8Z_(46*j6hTX?r22&(WHKh;T>pOq{DV2{1+C`dX6!QzXvT zngp04ai-QJz!ZtowI%_kNSs0v?%o{zk9(m$aT448L)*(>io^+VQyEN=xDK0|hq8uX zibSuNR0dNdj$_mOp`>z{B5`cQwh~N{IErmels62fNc3QXx);M?io_9d%MeVFIE*do z`yByOBo2+(RD>xK-Poia#nEAk#I@KCn4-9g0!)!Oh>g_7(pwilSk_TLPS^i?q(>#s ze|X;MIpAq=|IPh%to~o?_PYMd_3y4XyLP$eIsf4NvNPe_VK4E+-zWcA;=um#7d{Vhl*{n>FUz5KizYX62 z(1C9N_%Xf#;3B>OV2bo>?6CGTD@8sC{U7D^3U~ce|0hLW2>qXmL8JD6Qe=_P|Iy2u z5loR|q9vS4JE4v*TvAW>rpQIn3Rm@{X~7ivDc#Y9OX&u0ikz12=y;q)ikz12=)xuR z6mN?BmG09B_3zy83yeV>2x}yV|NRf}y9bLFo zp5RT9lhPd>*hGr_lkVukCGt3LiX4>g=)fkDz$TQJhp>qxm33ef%5PO{B8ggF8#a-o5)W)b`ArF%NK$17 zHle&IViQTK>cA%CKbEkGB-L|Z6V>`4!xvqEO(dz<1DnuG12&PQ>JMz9T4?E_v25`L z*hG@t0bNyAX-}6@HEbeDu7NHRs-$SxM3Vdiu!(As>NP*Gi6q$xooqZ| z-J;EeO(gdtkkjuTflVZ@K_;i)-PQtZBDs&LGXk4PUd_}|v5Dkfu8vTr@kJBKJzO2a zCX&0kIzq(;Hj&)L)gf#mxs$6S)NWuC$sJrB!X}d2xjI4>2R4!1#?>KgBDs~TBh+(X z6Ui-H9l|D(o4GndWd}Bq+{D!(Y$Calt0UBSU=zs=TphwDlIyuTLbV4rk-UnlL)b)e z9al%F`@klWS8{a-n@Fx@>Wsi9l53bcDmIZ^&D9a-gD3pSRa_myCXy?;I^ulrxIejq zt3%jCayeH=oDaV0PhP>*A#5VKjH@Hg2aoxamvePGfK4Qqa&^S{;8A~a30H@(iR5Km z9dSN*#GhQu)gf#mxrnPH&Ib?E_5U{MtCHs_&xbu>&o<9g_g~zPxj*Q>$-UA&!S#&m zW3DrDcdBVE?oIYxcDLYWoa)1KCHRCVV{sVYE`BGZ9}OVtz=ph%I)*r*e!I@!a)6j=;h zPF5XNbh3xMDRLLMSk+1D?o5%f=+y}vrM+~mKSjm@x8+uqC}xIInz3L+3vH$YzLe%G zIEnj1Da}?ef`zhmjW4CS3Qpp_P)aiuj9{T?UF}P0o`RFOHjqy+a}k`x^}!UG z2)!9hY+GIBO_6`lo52X6NRf2_Kq0iQN&rQw3tNw2ma4B2ph$Hh5dZ}vaqZ$%2*Cg- zbb?FeqO8)a38p%bOoylXE|!v2fFjk7B&s0QlT?5r)rMG_sjJ7T^=4&zDu`&R%GIOA zinC%~sugKemG#n=FG{r_^ptLBv38U#_gpbQwGg>Gxw*@_QVS4I4xsvOsaitixjc}X zk5t@9%t~F_l?w2cgzB>-kn;1D08pfSd?ld*Ee@ob`APsNQuFvqLKW%?q?-6j04P#k zzLHRhLV?s=z7hb6)EvH&P>tFHso8ub02HZNd?leG1p}#>d?f%BsYbq%P?cH&sTq7F z02Ha|d?leWEeNEh@s$8jq#F22LUjrRQd9X#04P#Z5UxMti51E>FEtry`ZFE@6sdZI z4qIL+0u-q_Y$H;;(mLw!UjDo}sYys#sgu&wo|=d-yswHQmr6=mJ@N)p6OcyWSfw2* zrKtc#YCOULP*e<80gBW(L;;|v7^PO9*$Yx*5m#+_ORPJy==y(`^c%_Zg6CVFPkZk4 zocCPeakzivexLiKyVEVZzU_La>yRtp{G0Ph=UvXD&Nipy__pJ2$1z8{!(ji8{T};q z`%1gf_PFg`cnaVl+j5)NX0g6#ebV|#YtGtd-GTf6CtKdIyl8pS@+HeXmK!XaER)T@ zGe2NXm^YdmO}{aH(G)iAFikc7!T32m2XM1-mhxNWOG;GPrA(7wMMoehZkUNAg} z?*rUyn24RN{l}jsAA@BB;~jdp{Zo4=P38s56Z+A@3@1J6Pm_~j=}>zpP1c2_Bg}8o zBmOk`7nTmShtlL=SUSS&COzy=lbd1b5H67>6T{LG<~HCGY4S8I9l|BjZ>9Sr?9q zV0nN`q{+l^R0x+ylWpOs2zCd!M4Ai?M}=^SG+7poiePnsOQgxXa8wAFNRwURsECum z$9-usE*urYCDLS7I4a^4@KIlyObbVaaEUb86po5G0esk(Cd0x}AzUI&7KNiCe*8b^ zOOsjQs1PoZCVRqB5kLIz_od0Ga8wAFNRvIm0(iBLuxM$3OQgxDtkz2cE|DhJ0$jp4 zh#ZWDOQgxB;2!^BF+;dSnhXkX3F93aE|DhZ0bHQ`QeML)(quA#OBml;hD)T$QvjDR z0zMdtOQgw33>!?tCDLQCh4xE83rbYOJq>9O0zfs00VP}_?Z#%{66L$8;u2}s;J^Y{ zP#XBIhO~2VU?HG{OQapxJZzUWTq12B966=>q>;Ep+BP_{5KzJ;(pGE+E>XVA)Zv%j zK0R$296CPg(04YZ&4WV=0VP}_Z5nlG!X?s1Y@)MiaABidpxE;7IU>DnNm@Z5a0v?h zU*HmHSucI){%5>vZ2{$=dd=S=y=7L~fOOyzmDPbuq$LF5#unqElKoW&82nS5#cfgN zPxtmOD8nUEx7U@Qa99;LbycgLQN@zOQg=%ltQ>fsvjw9be>Rhc^ zgiEBZuQiKsiPTv&3mxK7KPJVGzxwgfK8nC4QfF#S0xpp{U276>iPWiTNkh|xfJ>xK z)|v!dB6XtHB;XRM>uOB`E|Kc3H3_&x>Ugb5z$H@0kfbjAj`qjBP@g)A?S8aQ2mknG zxJ0T)+*F23q>f;dx;U+kI)qE44vR@cxP*59|11Bu_y5OjTWoWz|80E&*Zkd9pS8~N zFE9XKw0zw14olKuE1 zFn-#Y1oz*p{6+be@^XiLtY1MpNipX+b2VY2W_8< zQG;!t3^^aPeJaKcYx~T~kRifd%~cX+wq?i^ft#opq@HKulK~r>GUSzL8Nbqj7|fax z%#d}Wm7A)O!nO5uZ-(p?Eo@a!nikBEvC zOLuf|6B)8px}yu%(DmL7SuEYr!A+#eSm}-~Tsu$lrpa3Ajt*`jO@>N$bm5wLf;UZ; zN_TW{6KOJ1x}yu%%HzCgvQoOEgPTZ`fzlmaxJDl1O_PPv9Ua_6nv9d~=)$$J+nXlq zq&qsei8L7|-O+_>Vy8E)StgxN6S#>qIi_kejCd__J z4qbv4xrsD6pxGGURdvEBauaFtIJm#e_=i%8)tM#_18&0jYn7WwlRE)7VSJ$yH<2bQ zLKXOh(Yc8Vi8Q$ba1+>(A~%ty+7E8R_(C~1 zk*10dZo>G2#!aND%!8XS{(6X;NK<_WH(`8Ua}qb=|dO*+=TI$C2k_!jhDbp z7=Iyh6X|Oy6c@ZNm$-@aLA(TRqFO^__*x6NiSz-aft%1v12>W0k5F(E)j~^Ew5<4o zn@C@SRJy#Z(%&zoq5w<3Zc6V%3SBK!NkI{oo^MWHjYx14)gskG3~nO57qL2RBe58R zn@I0Ll1|%bGjS8?-3a6~zDIBq>0QX=G`>}CBE6HTGlH8)?_lbv+(ddiS4XJR;3m@B zxH`m5q_=W)go+JrBE5yHL)=7qGgn8b-QXtDo47i}O{6z+b%ZJoZX&&bt3%vGdOcT1 zsOR7&(pPbHh?_{SJT@PUd`1Jsy(=g^eV0n zaTDp4TpgkAgPTaN;OY=JkzUT!8Np4YuVCt^+(ddAS4W%=p75tH=jsqQkzUHx5$A)) z{plrK9pWa^mvME(`QWSm^kS|KaTDo9Tpe*fc#N+9w@W{fJm2=*?@4$zdm7y@;JJS{ zyEovOf4{;r|8B%H|J-=y-v{u_zr}d=|9A21|D$;J{~Pv4?RVJs+ZWmu+jnelu^qAb zZ7%Dtt@l|Y_y)iR%ky~p|E-oSxaNNu*ZVQ^dh>MC3#KoaZZ~Z+O*a0{_<%8PT#vQ> z-z$$P?@*2?&GJ9xC*+UG8TkyJ^S4-Tlr4r=3{M$8Yq;BRyWt49e-C!EhM!=DnjY}O zrZ%k|BKG)3Yf+({lOc;#Qxe)a88T5YlcOtLsLzm@LOZ9py{w&+AyZYhsjQuoA>)L0 zPBCk!os%J-RF*W<&Y7Gc9|Yc^m{iU}WXSM{+e$1%hAd7KgVDQL3iEogQh< zka57BtW_gbcX~LOA={vy-Bpf~=uQuLGvpX>0a{f6UnSa^A+yk{6F5rC=vse<%mVJP zttwH>3}rO4z=#%FM+bZv%`0#c_lGi?RbT`QrRW-8Mso_B#C@TRW)v8~LLs`^m(hFz zCvk5mquB&Tuuz8f_%fPH;3V!2Wi*q(2o{RaE?-9T2%N;7p^Rn`7{P)i-Qmk<4uO-n zJ(ST50wY*3q}zNM%^z?Qw}vvBJzxY2c65s`qqzf4;^t6BGY5=d!HjP5WxDXO#6A3s zkLAWtrW3)(IKhGy-QdfFxGF8{gP9Hl>&;+dE9xq5rX5LoGZ?WDnKlH1g%H|NB^DwR z#8!HT<7#9fGOb9You4o`KZQZj1I>gLP>BvA#a zo`h<%%Ac8!SemJ;$Ex*aWqT%oXsXK9qs5A|VqV6NG^)yaY0DR7dgOmQZ;v4`iBe8-^m#-vLpCy6JT)q-mh|Cq5HEIuJrty`)LS!2F zNJ$iMCi0cQ zLS!Z&Tz|$BE0k|uW<1jLXFOsdGUE_BY{{g^LS)8b8$s8V)<%c-=jYM&{|@OtCEWf0 zc2Bn_;C|ix6?fjf+db3uit8~~!L<+1|NE=+E6xGuHO^+o-yC0cyczU=m;LYdFWTQ? zzkqN53)!dI{>S!H+hewW0|9W{w$3)s=C=OP`cvy;*88n*ww|=Ew7M<7w0y*Jvt@;4 zqWNd$`^*>2>&#dCSc|N6Y@EEm25Zs z3{U>+H!L_omtI@$fknct^UGMv2aufddQMT;i!mH!Tr80 zxfPBIK@VB-Cma=VBDl|&C8xqsA?P7X-h`tfP6MCxWyz&*R0w*=k}u(?h?Bs_eOYoS z92J5dvgAoPD&iFIQD2ta2}gyXhb;LKj*2(|eAt&IXTnh-=pjp9grg#U{6FZ+k}Kh; z5cH5GAHq=)Km70aWyz6nR0w*=k`KY^b+wPM=v@FkWXX}N)=L8PkR^Kp=)p7y4vYpp zWXB)|aKW&cA?P9NK>(l!Q$d3svTpnvH~p7ivun^p)`cyA9!$A1=ppOGi+~xcy5%qF>B=nF~21gbGO3*`A#%4ed<-0rxJ!B1oL*G8? z(04XurNNrDDZy)J!EdtOCMSy z8U=dD+>CTU50%vcJ!Eb|5bjDbwUs(9YImRt^pLq&Qwl*3nHy_LA?P7l1U+Q>St;evL#D5$6oMWy=W0qJ=pl1`O(_IDWX{%_o;rahx!SnyG#`FLGgy;Vc z;OT!0!2W;F_IBF|Tf1$h&1U_Z^=sD8S>I=Ui}hyfaqAXqmvttd1^ArhyOxJ7AF!k> zhb=*i&HMxN$I%1WXs$QC3jY5srfW=X#(x@rXne2n2IC6jWOM-@#y9_6qj(jA{9XAY zazx$^0^qNPuNXdLxEc2XOw_*@pesjC0_|F}vue9uJ<|l!6bR(VW1wBjO7*wr$bz8h zqZ2L~C*Qmr`4cpKbkmwA=g767>0|oskZUP6eRAYkXm$CODp*7LBLL^k$&r~^sgu&w zo+C>`&8$izm#SLXB7`@PBdbHrZrwDMgvgNr0trzuTqPlL5dK(B1>*ccXZ)`+3n4epVA#2Bt({6lZTEsAW|?r$jnhBHOKwY%@JqPC{g_#j7A8OwVZ~MD`&54HCli z><|f&J%HChLYRK8k`USbwUQ9oYcK#v2-8nXBt&)}UIGbW`iV$FWUr=BTm}ESL_%cu z;w6v})w(3ZS6Vq7CKt-1qqSeg;cuctg`%3O4Ue+>`tW6 zB}A1JjfBYVKqN?rYLRLo1__bfj#!;ul30vELS(lgNvD^znMjE2Rs?do;Uh?h>=tBl zy5TAbk=@MH89_p1H!*co5+b{it0UBDkPz7oTpc1Ivg^4zLd6CNk-dtmLnK6Y9al%F z-5?>dS8{cTgvhSt>IhXFBt&)%SBFT5>}sx#P|ra^WLI%@h=jJSN$UB=ZBsy#@E?B!e?A|bL%xjI7K2MLi~!qp)XB6}HAX9Nk6 zUCh)`Nr>zsu8ue#JmJrFadkjKX!rj=D0#kz_5Wj7|NjTp|KH}`<8F5S!S$#s=Gy6+ z;(W#VX=m8E#yQFH6UTjy8y#0UrrUpO|AhT^`x^T^+rMnzwB3vQ|2wV!w0_U}PU{J4 z&}zcd{@!CbYFT8F%-;v?f6Cl$wwS(SdavoIX{pI-{F(7H#@mgnaHan%<#FYMN=7-O z>{KpOW+`U*W%(&Q_3v)^cKM*}!!`ex4Ozo3!%XRS(u1YN|6q=61}vU=zG{hvH&jRI z6rt&pBM(wj5}H0aaw!#)Dl~m^WMXPcLenQlPN$|MG<|Ypg=$K|lP+>FOtmEejmVK_ zs!0iGM2=h)+~w$hrwjEta##ML?PX|0j$BsRrZO}lM@|apL2=X}G$KdFsVu1sjmVKP z0{&1;Dn}!7_QdKk}N6rFWm8vN!8j&L>fg4+^Myhu8a4<(cLOr{y93|1N9`feM zLg0$DssN!8IdTuZI)S6~kgoOT$UWeO+o}@9%ur5q4~%G`mvq3F)64@WaepYMc?U+Y zP@Ast6Vj9|f~ z?(*d{)4)mG8Omv%fe|ct)E&N@W*In%+e5idoRP;kXQbdzxA}4*B;q!B{ZqMZYbe*j zRT2E@7GJKNs}kHC%C&J-1b4d0mkV-LS~rGrty~qsn{M#sTDU4eBXSE7tT%&+y{W6b zxdlklo52W;$jwI}(1`LCh~iASE|d#kE1eV7*II@*MXvPa{79s8LO*fs;+zk`KqGX5 zOXZ@h(yR&QnvqP0r@B%pSw$mq^N>UpqfOS^K7d?lg!ED7Xh@Rfi@etmKWg8AdjTe+8}KTg2kI&cZk=NJz2&ob65uAwO!G_Ths_c5PICj+{y%R@o3@x{8DBDf(fB4j`>$Sk zS^2aQQLa>`$iI+3Bi|@rC66)u%J31xO@=G*+`p$w9RQvG=*(*_MK-5&D1e|)bI}jV zYc_?GiGEOC^C|k#Bl$(Fyn!q$dhG3|DKTvdLmE0g{33Rcc3ToYo@@+DN0{qCPvptVuylx?$dj94=?F6&=!rZz8)|j6M3>M92Ic_ z_^>Ze=7pm|^hBQQ3P(l!_}ekdLnN}3@C+RF+=o3-i83s6XujgPvoumH|T-#%X*ET z$Xl=l^n^K4Mo;9;coFo387#s`dLnNcIhaOI?7%k+^vHn z3jrm1B6kZmgPthg{O5u8Wlm z6x%f_J(0TsfuJWS@P9#1; zYRw{gB6md1LVtbKk4f?4FVGX({r^AU-~a!FzA$fSg*4#w>mA)TRvrp=0BOgVZOtB z!W=TYOi!EcGu>p`YHBe4+4%3qJB$a6i;Py~7s?lvtg=IyFaNjvto#7J17M>(&hR`s z0T&II<5_?|sN4YPDv&imH_~F(zw=+PNWnmXi~_omtkl-70@(wml35eX3 zVmEm6d#7*RP;w5ks<{yaMM1BW_;`;a55;u|Gj+ekqRO^onzeNDtM1C96 zz)k3-ft$#0MJTw5YN4YQUvLxoEl8!S%_<8kDujJ|owHw?-eg#*DxQYC7u8vT} z!A<0^;OY=JkzdBu5$ZX(iTveU9pWbPOSw8iWd}ErU&7TPZX$mfS4XJv;3o2mxjMv6 zi>YcxLe%_hF^Z%M0FFPJ_#Blw;)M2&%)c$Yw z^Y$fnkL^dckK&qtrOj^rx%C6q8?2XG$6B7ocmJKUthPAK&ze7AzS+FSJjL`!(<7#c zX}77-q!^z!e%<&<yca-k`&lq+Xrb~a3 z9vkJFzokIc9P_rNsWdx}*8V8=yr}6@pjwZ4TPFogp90wcG<_^hqm5bVkr@TD1~p|- z)2Bd&p{6Wq`V`1dz_ao95Z@D8AhS_Z66lEnIgy%@Ku;9Nt5i%H>JbZ%%qWnRsVNEc zM1lNHO-Z093S@|CN&-DmfNiQR3G_sPoKsCopeG9Ct>7_7|Knb$FObLb4{a}_Cko`X z$~KkJ69w{9pc9It4$%_@vQK46W%NXW>=6iwVp2IhQ6L{AZY$9f1#&)3lsAl?D3Hxj z_hL9bQ6MiC~D3IX*JyG06fu1Okzd$d? z(p2i7a_ETynTz#|v}0Wbauz3aLQ8X7S;y#TOM$EfT3A(gu9T|M69w`Y=&e*uQR#^S z`3cP-?#oBCbT!ShD}9QpuU8Fk zja=y~%ta!7iu4oLE-uVLFz5-L;8M9Lt2AqZh1p1^!&6MYf6|ln5G+lZT!6P7UL%41m#8L6UuGMDrJ)VGx<|;zr03v8lE+L zz;MB^#Ne*B>Oa&Q=p4{oiEN4J-TN=pSoDGhG+V;SL@#JS^CgUEp~|8cG@uz1P9}Om z1DZ2oL<@Bmy`TZjns74l42=QJn=qnGiuX!eAY31(tI z^CygGVXgx+F`yX~P9~U%0nMQ>qJ^0b%*23ZQ8<}kCI-l(pm(o(UXNfV2FR$OchATK zGciDJg{33Ra$qI~$jPvDI>1Z}kbPn42y+~mi2?F5EFEGd2FSdybj11K5&r-=7?ut( z69eR7SUTc-@UVY?{0vKnn27|P6iM72FS{AREU`vAoIde5vPLteFJ1; zI4Zg@kJz0V*bQkEj~4K=;INtoW- zCdL?lMvXDXsIM_3F{WsYG3`xE&l}VGn_hn3@4Y*-bGF?3O!!B4?(>}Q-Pt?m`QGQ8 z&m9^UiHc%)Iv$P0X<;NPWG3P?EfN)V8hAJor+tyAkeP_nut-$YN#LPKe3Vdy%tSmS zR8gmZ2P5&IP=(AyJRnq2Cx8bcalcT7%tYKLR8c?v_ebJhp$eIaxJRg>e)#W;#N9#_ zG81tZg0b3d^buBXyTMGvok+UWED6j++<`za6Rv*lU^r$XZbuB*gh4R_%tYLV05B7- zH|fko+=`#U47A?N>&!&lf+JuiTyJP$CgL4<5zK@O1j0~e;^e(U2h*8}llR~V&r9J8 znrK70^W@zK05hQmG?|H$CvX_dMC)m4%*4sN`Uh6%g66=*&Xafc53B?Lw*HZofF?6>@;DBInP@%berDq2t^GsaJnYcf&Xc$F53K|= znTeA(4?8rOiIX?s5U-~GMZQ*nMw3QkCQjanKrj;w{12FklgG@`2iC)eVJ1!LInK*f)T2pGjZ}@TPb8FPF^QU zX{;a5?Re*`lLy*LAv1Aue_JVJCQe@4RtlMkll$6A0W+cR|NoNl{{Mgbzu>R=Z}Bhn zJAL2wz2BGg-GMy-bA0FcPV-s3zwmz3`$_Ky@oj)Nde?g|@cRDvaqmDm0UUcLfH##nIjI;mV{%QNW z?bq77?PG1fuzl3_R@-XZL_7)LG3#;b3UmU#gKq-7+0twAbbM*xet+QqdK0t+=%l$j z%y<4P)~Gj@pg}+W(F77KBQ8l173SL8zoyrCG5A4TDe#+(d#_L8zqIrJ1n=&4W-0+(d#tLa3xz zrdTXNhapq~H<6$LLHFC_QdMnJWJZGSWQ`#0-0=x|7VN5ZeP_V4G`NX49SawozbED< z;5-NgCY&`pPM-vBV#HC0S$CM)6{mgT+WtuIUw_t=?l|2P zmvamwmD}yf;W!-@SG$apCUwW@wM<7>ZnHbXak?B3A$S8lH-gyZyHrlW(K zh|^`6j;_>>ap5>!m+9!>CgSv3rlTwMVr)20w`DpyxQRGDmg(q9jTjS-(`A{C4sIe& zZ)G~VQWr*t<8)W1ql24>(^Hv_uGE52;W%BD>FD4l;`CCcqbu`&FdV0wG94YhvR!Bql24>??#}}45QBT-f8h`kg&wyb4qR^z6(KvX7~m-5#Nbp(=otzvqHz=PjqP$1@eLRN+=T1pCN~jZkC(trxW1%v6Y+Hn#Xayh zo7_bFGQ0$C!l*|wcyj=_iTGNiftxT(12++0gHUi2MxnzMUvLxg)kx(Xr@>-dGgap% z;;WFd!Z=0eCgLj*32wq@aIFx7n~1MKtVt(HEymy`;>(d_(n(rRZX$ju0wulhA>2g# z5@bqx;Tks)UnbNU!cD|47V2o+M0}}KN2$}`CgK-Kb;wP`mq>M#iVbcezF4Y5ZX&)& zs-x6ya1-%`QXO&=@e8FoN)-n;5nmwHAvY19FV#`%Ik<^#mT73WC-~FHRzu$isz5!so@B6;TeRpEbe~kBa?<3wfdRKZUdVc76!gIp2 z-ZR$yGxvwxZ+5T7djAhyPr2UW+UA<+{JZm$&a88%bE)$zr_=EZ$19E}9W`(O`yFc> z7dXb+|8D=b{R8$J?F;QL+gFXenCuTV`82I$j#M(;xQ#W+hmN zF&DYdYR))X1smasA#DF7Scc70TH8Mf7G!w}+dm1GWO)kPKM59Sc?#P<3D#|S3Llw~ zU`>~&ke*1e;$!wUd6Pj;BxnXeG`PI+~Uv1N20K z7DhY2A@oFo9z{LWpeGVEB-*<;JUx-1?EpQ|I7Nk?NYG`VqvJlS`C*sP6A4<3HG;GQ zy$Sk@8%#o*)u5$sw0~BD1_O;O!;@>KYV<^cE(0AF!xW94NYGVa@2g>?W?lDoC+I1( ztJ~mRQmyNraDsLMcc+E|q$d(|5(iBJ`#VD_J&~Z3z@A(~iAH8mVjiLoNTQX#(#}Yt z2Z`9lY%VkZ>Yl`0sfyxTcSI6%q$<5v^(1CXRTS5{J(B2_s?58xCvkyPMe(fLB8gd2 z70?rj^Q9_^W8E4_%#^Bto=BW0RZ;xv=13wYRRKMbh)PwIKGdd2A|h1*J&`z9s-pCx zHbfFLq$;2%5?xXir7yKEk_byxKu;vjk*X;DskM>B*-{nI6N%{vHk-lJKGo`QVj7an zW-!tdiL($0dZKkTqA^ph>Pbw+QO-NYYkq?pBr79{DM;kJW1hHTL1Hq3K~IyV~Nw{N}wkaqvc9U1zHeGoF-SAH(%fXkM;k7{r|58tHGVv|34w{_rSOC z9e{m-d(j109yll9$CChF^nb*k@!#ve*1yF+$3M#Vy6+>txBAxlrg(qieadA2|2y#f z|EbQOI6v;Z$GO2d*6~xvBaSyXmSf%jyY>&;kK0$;gLv-W2W+?77TZQyzi$1Y^@#N% ztJCsLOU-i3(u2S7u&e$9y?}X1-Gf-&C(L&KD-{)8prmd>B$?;}C3P1fh*oMUx4``Nd(c#{03qoshbi>CWwio?n(sF%IpSWBB|RFNhXMiBz+0G_NMc}5Mm-pbAqnD zAQQwyk`6_rqs|AXqDi_JkVYk&ZeaJQq#Uzld~5OeASoL^|qx@N6_m*CNs( zF_ENi5$UM&!86e$J&Z_)#6*(TMWmz72Tw(lbTT3x5)(;!7mkYh4OeF5Y5fBsZJ6ni}#GQB%#Dp9C!BAo%amUcXbYdcLJC5+Y6uO`ZHk3OP zw;=$;gc{H!CKAVS7{o;DX==no;@19w6}F%`aIrISOaH)1K$DnA+>FD6et(^qNZiyv z@-gk+d?+!IxUqj^C7?-6B#z-Qh>6xy?k6S^NBf69JnYcf&cu=ap_PCpF_Abt?9e18 z5{GbzS5yDOUaLT(F{2R^i5n0IVuFGH0Wp!d-Yk7!4Qv=!}=}jL)K%~rRW5F!&0#vwsc!;9iJPx>*xRVrf3G( z1^S!$&VR)eEsmvV4cG;eq%P`B(GUog6l1h7mZCinDxnLMqA3t6DduQFEJceTRKhbf zQZxoaCB-21#!|EiLM1#yBSmu{R8mY*Pb@{NAXEZ3k)lBmDk(;3PAo;cAXEZ3k)lZu zDk)~EJC>ql5GsM2NYN+=l@!A?E0&^d5GsM2NYO9|l@!x7GnS-<5GsM2NYYIRl@#L? zizVqZgi7EhlC&Z0dmDeyRr3^?k)%giBS<@Ue3H%uyK3Fv7;r8PZX!wF!sX`w5pxqs z8k%J$DP41tv^D&_MjE-fu+Xw95soEkclhp^rfJ+nk|t=WLAb_EBgvO^X-PVzC58!S%}&xS?KTJ+X2oG@SCS@*tNJ6o5&cs-c3le&|1T&ANdHKjA0q!%+C9o$5cUd(iKrH)JpC+WaUM+Y~Nq}MVX zU8x=8!by5B)6v0ABTh~a;Ic;a1+U^C8H~IzCWDYAsHRq zMDi*G8qF~3Jnx;B+>V4L2A@-M6Ui$PG-!rza1+UGI5r&vd_$Ws8r($k3cL(%!u|DT zQE>Z{Tk#^e3HOT{H<8?eAHhwyUmA&JU5ZN9LFjClEO_SH{oS)6YiH( zZX&sH;PqB+BDn#tgPXvKG`NZ6dTnHn`=wTHBDoH)@;km>(z%J`W%wD~g!{z-ZX&rB zuYsFze^KKml55)KCX%Z$za}&vBNLgW=qH`0; zixCNK!f0@<5QCdYE=8I~r~k_&}88aI)=P^zQUX>b$C1yUVy6Uq5f9i?J}n@ILbb-+#N`~P3>2z@z}2^|b| zhwQ;`2JZ{*56%yE1b!HJD)5%T6@khA-}yi8f0uug{~X^heb4#c=DX52)%yqU)86-Z zH+#E0zxJHMxBhMSOm+X+{V8|OeaJoE^)J_}u8Qkk*Dlv3u5(=>=l?lh0|)SaF#ktE z{m*xfcl=oU-oHy7qni8wowhIAO15LRMK-VX8`k$*Z?rD9`mp+czvU*&5{tj%b@iUV zji08aScK6E@=P455rtMzibWS6NzcUQ2Tz;GN5BW>cEo3;SbpUxYz3uQeB~)@1*KSq z8L%mgOm=CsHiT@)Xh&DVAw@3h9XytG7Ib z^hAo4U7kXEBE`CoIo#wL13i(VB>>UDOZL_yw+y}r8-3P6^hAm_p{*>^6DgX9wz5c1 zr06Z+o8@N+dLl*R(N+@Zi4wa^nOS|tz?jigq3B1KQ69&6GQDY~F8<_)4JQnWnUSq!EpQuH+~M+WGL z6pf5_enaSq6rGBCs6kJpXiKzrad>(nMe_lAqH&4}J&~fLXr znayCNCsGp;2zsJ*O`0C&GPQlnMlUlBTd%miPTsmp@NtsY4k+u48)S2Fg`o2-YlDw8iQ!AwirjN73bm^ zsnd~$s%)CJbbe|yLJymUHfo1LPoz#mF6jy5v+K`YqVNBIx+8Qd^rq02p~ayo!G8sR z6#NW&0PhT5A6ylj6+A8QTkHh*c;LRkJFy#JTVPJW=l{C@LH{xTGQZdN9Xt>4Hs503 zX?P0YL*ARbmwNr4?|D9eXa8N~Io4AIxd1<-{be25F&Gz~$6|>_z(KOu!I!l7gj@Ra;=`Tb&O3g&aCry(f(qYFZO{XE! zQK}|7K504)kq)1fk=C7t;I~)mCK!pdZZ#yCU?kGI*APT2l@p9aS~nY#OfV8@-E9b> zmD&kLBCXpENhTPHwC*L|S(of@r0Ff{{q;mP3*WMk1|y4nee1 zLBU9*b<-iq1S65wU56lAsi9yb(z@-CWP*`M>%PN0dI%$t){TcG6O2S!cOHUhbv}3@ zn%1p{BomB8TK67;Xmvh#KAP6eha?k>M4IkHGVcswB+|4Xl6mKX9xxJV`Vf(hIv<>h zrs+{cI&;BDq-jh8SI;v(YrYiAaZxM4IkIq@&IU&qUL7 zDk2>+5@{L~k&ZeaJQYpTuZVQWNTlgdL^|qx@MJVSN~%LfA{|0BxB}BEuiAqrBIzI! zK`)pmf{{oEgevM}@OUKc7pjnvNc)5;>QwMpB<&Tdkda7xges~rcr=oB3suNSq+LQ4 zbsBg$l6DGJ$Vj9eLKSrqcqo##3suNSq-{bKbqaVelC}y}$Vj9uLKSracp#GQ5UP-o zNZl(`Q9u6oM^g6)Rme!B?iQ-3AO8CysS`pKG7_n~5R5f#qmQt9Lk&hEbtjUrrfr-A zMj~|w0>Mal`ZU&FhyjZ*C}x0>NZp12FcO~QIwO%fj-SB@wBDxcj6~{I9GQg3 zTUr>2)Gc@sjD!co!B9pbb@R}{bVed|6OQn_6rP}oGL$=0HzELxgc{IfBvQw47>q>g zX=;o_>S+JK3Qf=)xY(IG(m${g&}1Z1hjDn&@2@iwsYCrEU$5PP4`n1$H}sFJ1T-0m z)b%(FMxynU`x%MU!TzBS3_EnTGj(16&`Lm)kw_gFc4#sZsr@*_tEqqau2rDXe$g0- z)U^l%Bf-G`fRRY;GfN*>=^DyN===Y_sqgBzNJQeT_o+~`l+<$d{#$9&rcQ151T|aPr*!2$A9@lK=Kb&84zR&qK=T*+L9slQe z+VL*O2FF?UpV*(oHvwK|A8q@Q?Sr;E@Fakc^{dwVt=C&Gw02m&Y{^&-V*kIrJN3q~{r#T|?EzP1jH3tle=;-*T$M3PTRJ~OYk>Zb zN$9ZspA4M>`aiPVCA}FM1g-!XpO&Vj7^TIr4DAB;4~tS4^=4=igi4B8S{TdFG6WyV+9fV5YCNeY(LM6pG^~5r?4?-ny6B(KY zp^{>r=ETyp5JDwz6KNU;p^{>tx?^eD2%!?Vi8RfFP)RXSvtnsl3850Wi8M`wP)RXT zGh=Dm3ZW9Xi8LLCP)RXUu~?ekL#PC9B28;zKI5r|Dl#KY-?GMZe%P}7-0^9;8SJX{ zyfWZs8r(#hUWNeG)>Mjla#JGX<8lrUL%b>+|n#Cem9FXf(s9^SpOjdNUG?W*Bl4>B|u`Xohca6X{JjHXR@ISF{PE z!A+z$;$?6Xo-a3xg4>_ofEU3{cs{Ff6Y2H%5!{65g^{?4^g2Xv2i6P2a}(*yaGc^V zDBMJPEnWsU;dw#jCemvLUT@_l(yQ@0xCxv{gPTaN(nj`pUTEbe(kt;QxCze-IyaGC zfuF%mcs@J8O{ACOHE1B8c+=S;-DmRh7n4!2k zezD0-q?h6)a1%z2kii=Qz)hqtLK?UUvovrM=_LpSH(?YyT=4}rkzR~c?zuHs_HCx> z+(dd2QdStJpa^&T=-l)|M1q?z8eA*H;3m=+BG#nGq!wdv6X^v=GU+j`CpVFvk3dP6 zd|0C z+kV?@TZi=}Ysq@RI>#EqJ%7@2o#isiOiQ5S_c&Fvf1Mc?Y;>@^`&xT0&2~_Rr5Wv@ zRb zBr+_-@)U9s8J1;v3OR`ktF=6ZoJ59|T%JNsBEve5IosqK11FK81pvFy_+$;b2pb=+ zL7YT}wxF#nauOMug|@QDNo43Ha0ekjLvRuq8jZG+z)582LfT3KCy}8~88K;Kx==WY z3~fwXN#Gaiv#k)iA9V%{K5 zB14O#oyA~IB10e3a%6y$$k4cG=Qo6t$k3UnhZ>wjW*!b{@8a;BM5YJF!AUeuQQ;&q zb8!%T9Pb&;54(hu$jm|78bR8D-pp)--e3~iEQT$+SNG4#bR!k*E5nm(rfQr-<^rUk zuVR=2!@BR>%q&D=mBTPnvl@H5Gv^~#ySfeDCDpL*31?;^34IJh0df+V^ALE@B(T39 zq;e9O7&5W?Vkptb?8!tCeLxbebdz>QG7%(Vzp}XuIEl=;QWd4Ev?G$4Ayoks0!|`xj#NeIGHr`w&X%fxlgLb$swmy2t&z+$sR}rW%vn+urR%ggl9?)1 z0Vk1}B2`hkPn#l{$x;UjjK{>we%+~37_0UpOw|AMaXyFTE$-L=>?%K3HY2c1Wp z7vXyVzlrYvJm%mNY^Jzn}M#6AhYAU^Rjdp5}hGW%gEAkNOZKPWn}3&L^?_x1v`CfJFr?m`67N=*ejk=1R8 zBopjJR`(%-Xr-!xoyh7&M3M=1BC9(QL9|j=!A@j#Db-8 z=}kmB>U?l2nx$_M>C6Q?k)>G?>8SI;bI~lFi%5s;M3z=Xq@&IU&qlMOq&j3LvLUIC zIv+d}%?719WGAu#sg61yJQdCQr8;CMvOcMfIv+e4&3dIeWGAv7L|-d#6Ut3CzF#8i zMk3e=QyVfL>_pZjR8c2`$0J#%P=)M7)*)0;r-H{KS-Vh$>_pZkR8ft=qmists6uuk zYZ0ob)4;=#Y==;V>_p~Xp^7>QJQT^?BUB+fk-1x_qD}!1MlvUaDr6@zcL`P03E+W9 z=1!ps*@?^@LKXGne}5!%yHJJfMCLZ3iu&QdFOoSfR3ST&xfQ`CcR1=zHrR>GEl4uC z!`ZSN>_p~f1cIIL_VWkBu@jk_5W{6yHD-XF$lQnkuoK>6Iy;d$hM&O>wBEYw>_p}$ zj!Z)2krs9$a|ADfo$!K07|Kp$4i6nnXD2d;aD?Zja0bog;c{o@1_W?FLf7?G}&)T=!qqg7LPTAgO zyTaCK{iXH8*0);MS*Ke5XnD@^Cd(E}XUD%fzNX&oD?f#EGzsYW_(F{a&aR;g0_^$Z z=nv5I@qTgON@%34>dDaqu-E)+Lxx~z&nHJaz+SU?;)(@1Iso*1OoE5+`Q&H_*b*~V zY9_Pib8e1afU5?^NobB-63x*Gusd%YtF=d#&B?L+b0x+&T5XVAJR?UVz?B%Yw59WN zEd7T~LmTS_%Cj+YbOK-_Ol10Vm-Oan0J!R7d|Fz(VuTjQac?)SIIT5GpBV zXkjczOCVGN8d8?A_$ehMr3IWgi4A*>W*b;6NE}&BeFCHLM6o{&5C7d6@*G) zBeFCJLM6p0&5UJf8-z+=BeHZ5LM6p4#bQ}{3!xI&h%Bv#`HZI;rpSyeeaRZr`GIUi zmTm<*XIpDjgN?}2t8j7oB{3V3rHNT)lF~INODn_QYow8zC8cEzA{@)o;_%%wO+zI* zXL^>#2W-TM;iy4ppOdBg0UI%5lvaJF&&<*WVdwIQacbR}HZ4ncw8Svstl8OX5d=13 z#8HQ-HB-B?`;f&o{E^;>{;Vn8*}aHl2{(*X>dWMCb`O%cs%4xssXMz{GP+VzI>Xs( zB%_0k$nKJiuGEnU;p|Sy=wKtVS4&1$YR9;6c86qiuo2m-B%>?!Vr)3OT{1e@i0qY; z(Ulr8CY;?S869jy_6o`9N?jNo&Tf^A4mKjYMKZcl3r2;rnEZ z$>?AsvKu9%D|5a-oZTQ99c)B)Jpzqp7Y=rj<%^MfDKf4+)f{pNgQez{stMDV(2=A$p*of>(L~z&Dso~j( z>A>r)Y((}Fybd-3C(>XevdgrQJ>FBTY((~Ayb3nL zdrD^`vP~NVtLY1u-s3BuC;~)58bdHt+9UtFyEghd6 z9RxZ)KJ-t9c6@U54QQXVIxM~8lk0Dv_=azvlsj`Y2k7{y0nLt2jxGQlpVpDJj!%w7 zAFU*x0unU)GR4ju3p@u_0-7D49LxHkQ`S2^Io58Dyi;>UL!XwBW6kEsNMdZBFDOV zqaOGl;1M}iSi|&z{m%HU^)0fyv~!x(k(`!e-Cbm)4m={qI*U)CFVy_3>A%#Vo#~se zz4?}dM-K37Byy~{@)Y6`Io4Zw3h{^>E3Z6-ctnnMSDr#VBF73WPaz(WWBoNxX~iRQ ztibq zT_|`&j!venB;XM_+MTwNfJfx$huTU49+87(#v^ib)FV8LJ-6O`$Ki3g**Kz|-w-?^*FEG=1CPjEfJ555 zI6NMan}y@&51gWcN94}OL9}uVS3?GS*6U{EW+H8kAnibJ?mUFvU=rFahAj=F{j+j0 zq@sgmcyi5Dc-CvXa#5t9wPKi};SsqABC*h67^!)gz1_KU5vyI@%_(u{nvvQ-+7r&r zK$6))6Y+>#7XlBOoQdWVE>P}GuZiZu$ixDS;qTDM?8%*jXe?QoM=K4bosrzxNW_wr zAo1#++;pjm(pcIN$xV~00FTI>B~?)xOxq*5sZtf-5xFT+6{XR%Es~ooRRJE6nk(C=IAhk=!_`3h;>BnNk&{5w#(b8!J@- z9+5jks-iTc)0Nf0y&G&gYy-=M~O#9lvp$a=hJf zrDKZycX-a<5wNe z;1oOjc~*V2etb`9t)DzfJ(S!xA3S{Q(6zfz8E?6H<{i5aADVjf$kbhzZ&NU_89=lg-voo`oN+AK%B+)=!>AAFUtnC!4LGJj*>=Kfdcn z1_}n3QM7*Y^a5!8n5CihlczaA>&GaxSqodrC|W;xng`xP7@S-)RcrlpqZI0up)p50Q>iO94pa=|4m| z1SImh|1f`-hX4|J-GE3k0Z8O^2O@}8>L~z;ylz1xnE)j6x(5+ND-{)hL|!)`l1u;+ zdEJEwqLrEoKq9Z(5J@HgiM;MZ1kp-W1t5{vjff-@fJ9z*B7$h8t^$zA>sCaP2|yyR zdl5mjQdt2=U{7*G_M;HNhSb^yzWQ@ z(dvBgd^E3H5=kZii99`tWZoG9NaRNe=N%1@$cLml>U?l2nh#2K<^qt&2c$aceDGW} z@0aQjkjVR_I_iAzY&7qc>JX5~d!#z*eDF*(@0RKikjT5FI_iAzR5b6D>JX5~JES`5 zeDGv6Z#ShfqbG1RjdyZWpQ$ zkjUL8R8gmZ2P3)TLKOlMxm$%Q>ICpWBzKEYg@8ovW}%Au@xMQkyGf`*Kq7aeP(}Ul z-xtXp6RHr9$Q?zn$sLY*n+-rBcLYf$cQ{)JNaPM95P*cQpCK3ykjNcE3>RY6m;pc{ zcLM?dNcgVT0g2r8_<7KBu?9%+{{M=OXFEc_480cmZ0PY&IrL^c17K|^5}JrUz;A-D z1|JFD9o!t868KZ#RG<*pi~Rr&|7+O$e-HNlPsQHKHvF1C-(k7fM@@&^-lEs z!Sj-*<~e}x01UW)=l-laj(q^w5~iw-5v?7w@`ALAMr?#LIdLy0Z`jD;MwDNI^FilEgLL z1^No~hz#E`_2OL}F3?h-%WaahYGQ$I0yu^5`GIW~jlayQo&p^Md(%H{6Q?N9Jg_%y zp15K`fqvnrX>hX_($Yp*-d&)5U<=JyshNyM%BANP=p4ApV4S4k6b1SQcJqy6wPwn) zIR%;puG|<$tL>DFXB21~xN>8bwsd}hMgcg5NoZqzLBT2V^bNo%WVuUv^Rx?GB{DuO zO-r#&i(`432W%@Ar7r5t(=rH^6zjAwmZyObDgmd+(>4f|6#KLwmZymjDgmd+(>e&1 z6bscG%hN~*m4H*^X&;12ijC@t3`;oHDB&BOk z{#u0b_Zn&BW=UzON8wn0AJX{lnWmuv+|G{v6tz>lI6!|rh(Um#hAI`6qj1HV4zY2jyGmJXV zd#B}BBEe{eAx@EBfuKP%d;_P*FUPUz_@F3-Ky&3g0t2PLaP5KLe-mJ>A49c>jMDzDzIlYN!-C z9GVky1-}%0Z}7U{g+XiJYk^AOXrKpc|6lUo??2|h#6JPg{QD4|`M274o-g42z4sg5 zXTA4%|I2&CyU9DxJIQPHyzY6ylk!~cnc@C}`xEYWx;MM0xPI>XsOuirO4nrPFL2-g zR_7+?6vr_VE(t-&$Rv8_NwipwzTbb+m*In+hps%t>3jiZ~Xx1 zfA#=+t&^=m%dae-xAa+Vuq?JXjJ^XfQlMo)pVhx_f(LEj2(FIzKNr>Fjf>{*kEPz?~0WA#0EDH1!g9g(vivo=U z&r3lPnlwbYvp~}Tus{uHVipCu17H@dr>S8U1)2d4tN;nkfs3657Jd$_1T-;=0?YlN zQ`Rwy0&6-)-ln;wp_oO1wVWd>0Zq)Jz^V-vqV<&fF^dB0G>6_Y?9kcH0&6mdRsx!s zMS+!g=+MM03aq!hnmpRIKQd-fV7ivlaJVfw(n>0vO70_*W2BXwXF1=e5e zc=3Ch-dX>p2JLWf!7K`_#PSqk76sN}c?vO$0xPmSg_uQw^;n)l%%Z@`EKeb3QD9v* zPie(03arrb6k-+y)@XSOF^d8#wmgNHMS+D}oaAM zXne8;U4)G;=pf9ZK*!Kl7BPzgeMDPX#4HN58MrZ$pCK@d0zF7uNx&=$G$?H)0kbI3 z!i<I~Sg>!KbEgr*FlEM2!ubWYr zfi$$POw$hZ7P=66gGp$!7`8N$_RlJWk&4cip-MAV!z>EtAO$TK!xRm(D4dN*EPxnB zYF=h1dmQWd4~v^`RoAXNcoQ5Y{( zQ5sO&B872M6<`*HGo>m@BWi1;FjlGp%%X6HR7Gh>ZH^SiNL7GY6i%0_D2=I2k-})H z%IuB2|KHm2bVuj~-2d+m%?bV;_y1Yk|DTWh|Ig$8e{Z12|4%&YFOR+d=ljth^%Z@) zd>45C?ESnq?%nO}^8D5FoG0nI!gH?sH||sT?!PN>$NxLmldgBV*1OKaO8-;Ndz~9W z{Qu1HsN;=}6^`-t@7W)--)3KB57~ZVd(8G$+f}w1)?ZmaWqp_RcI!2`)1PSdTmEAC zspTtp-rob3xaD@sHcOAC%QCv-?;YR6$!hWAzx9{jfeTWJcO?(>dr%w3BIDJTMt3BQb@s96m{<*$pl|f)Xj$= zS}CRAD~h`NkYs|dDC+h@5Umtb@D)Yfe@HUHR}^&vB8XPjd%#x|bq6BJ1Yc3qEr=jm zS@{89QPe$%Boll^Q8yuiXvI&1uPEv+M3M=eC>ZtR< z3(;aws?!U;q8O0ssPn<|(V}0fGY@=4(I?eWY<0({qeZV&r>Emn^NJp+jyfNliWc2c zow?vEiY}>+Iv+e2Ejpz-g(~DL3bzSW)QRBHNa46pg?vTfR-uYI4Llqv+#*yVUs1SO zsG?2+4@C+$3025f6mAr%s8hg$k-{;d3i*n{QK5=D0Xz^X91*IJuP7WAs;D3T`+5Jr zq2t3Hp_lRPe>a9!hE5Ou9N+r)?%)-{nSs9sz6|>Rrof`W82|VDAM?M_zs*10_gCMi zeKp@<-=)4O-rstk@}|7Iyvw}fJU{S!#B-ZxC3gIO*L|P+7WaI&*Y&ci&vmV<$N6{X zi+BRS9%q;1FOFv&@4<8b!uH?TPhs8v8vAtHpKQbB#yrM5BFZ(AR+-fCTG4OxC@ zdC+pgvdl8J<2yLP{(q5De{&;XGw2^`cPMBFm2_8OF%MQQ*R7F~ZYm^+TY5^mrw{}y zm+R(8Nw*Y|#LIh1x}y*TE0^o0NJ%#olEjTYCEZU5f|bj4L!_kJ2}$Dmo|5h+1i{MX zx-L@E&4eWJvYwLeB?Q4rty&u?=~hCLxTd>ACxM2x;Wegi>sTEw(L|u3ZIZNVVu}6% zM1|77I(YN3-M8$$ZdFf-9)WG>AGS$UlxP>&hBi-Jv7khUaMU!o+5Ko~oGkAy(lD^! zZ>-cz)`*HCy#f~$jFU8?qDZH}#=UW@);?J_r$~FiMH=I1wSjW+j3SK!7irAWmd-EI z8i1%U32iJTC`3h(PGO5QcS&!N27wDg#;2ueDMo2=tVp|noyDTmMZHCu1fh~*mKMf} zv9W&gHTDSL_M)0?SoJW zL`9LNL8zouqdBo6Erd`BL`CtCTuG@&-Lc{gawQNI#p~rtN>!Q_D;|_9fv6~6Cs$G` z)67`$fLsYgMRC7eNvTe;Sn*o95{Qc8K7^akcxr`;%qZ?f+8WdOfkZ`d4?+j6u{4N^ z;%*${y7NejG=m%ZGtMqvgOp_^DP41lyAa0TYow8zC8ebvg=58?NaIy*nx+vI#j6nx zqGH5wji@N@Kop3I5u>!~Gks?9D#T&G^N4Y3-I+G6xE%>bYh3G>pEbL9C4xXyj5z8r zwPtEpaT~I@yg$+#(VsP?yLbg6S;7q?mHIL{T-=HzF4-F=P3kUgk&Ld?l+JK*vt)D- z6~)UXqbqe}Lb$j|GCGKg;zr5nO6?dIE^d&F4x*yCUNX8;FUE$8>m;Lts3=}08C|In zW5UI?lF>m_6xT>bSL(v(aB;O{bPyHARg%$_S}-bHTqzkHL`89hWOQZz4~C1&C8L9= zC|)WVU77R!;o>Ed(Lq!cmm$z-hEeBvufG5P#lh?U2ZHAX?SYpA_XX|cf9L&g@7uuuO!EBH^SI|8&nnO9?(euC zaNq4-?LO1>W7lKY|G&mH&iTL2k6<^zTIYDjA01CPjyo=MbURMB|K9!$d9n}or^?8xym~p zh?MAIBr0qhmFQX|D$3Q~aet&l{~}T0V>wE6ED{yvss~}G}(Z~R=2=pTg!{HSrIu~r*9~3iySCr^kfL8=|>Uc$oW(0Ua>)pNf zSdP&p`VQa~fgLS)MTs^8ctrrP!ce@TL~}7{FdeTb(M9mQ6cnLJL6kd7bPxav)PN>l zQKD4{z;onHnspkASCm-T zIkFPa#4AcH;b0(IPnml_J93jtEZQ7;{jfu4J4-Ck99juz;uR&9<)K3puPCwj@@lea z=4oWSqQv5RqaOGl;1wm7VZ-!+{nGfY2Y-u2iNzUs1y6H$b>I~x7GrF93D}xOTK}a6 z?QCzsD@rWO@)Y6~B^G6Q3h{~(%dB4rih*y*rXjy3K4*M|~<_v*Xl;*da1iYfu+inu@iqbrzq=D(8 z**vT;qcGy$1dr7`>+8ks$%@rcH{mU*<&b=nyzjYA^VwFHS*_ms|*swmy39g)&l zsS5Cl(iu_}r3V@a#L=f zM}^CDSNtd#CxyDp^i-y!E4R(TaG9>kbaXU{%JfpEqbs+|{%~1$Qzqw*CQ+GY3QeLB ze@XQ=**mRFAGO3V!85*0+k|C+z*7Tvrw!DiOmj3H13aa@=?&DPO#1`WBJlBMirZhN zy#ZKnWZ17Mc`o#wJ4+B zw})DkX#>W?5I@vJEy}F^KrI6QPem=tto%SN{2y(i7G>6apcY2GlEJkUs70BE0H}po z8c>Te9Rg4bqtIqSY^kL{Ey{EdyeTqh<26%t)S^sh!5ayK6dkoF(_;X&FpAV_DNu_t zEr&@*Nv)+oEy^?;CLN{qL@mm+8$bq(&P4mP3_&f*bQ?eh1eriB$}}1hogt`2nZ`q+ zqoEdMS`U$qQci(dl<7Z2Iz%nXG$A4#rJw?}DAR<9bckA%brWL#E)PL1%DM}YWCFD) z>o!CXtt|BbwJ7U8M3M>AqO2PcLA2T)e>Gaxorok8s6|<~B7$hOLH$qeyPqZpcZAHP-h5g zQT7UTG}NN(k?N@P!3)u{TdG6UqU@6DsPn<|(XvyjL)4<|km@KF8>mIuF4ZAwQMO5S z)cN34v}~2?%mr#uwn%l<`QW)|xkIW$)S`5+R7af;o{g67k?Ih&DBUg9QRjnaqNNj3 z9ikScyQDhmeDGAXbf;8@s72`xsg61yJQ*$BF4ZAwQMwJ$lGQooUK^-I={OQ4t8+vx zO1BDC)XCuSNa+@#3Q>#F%|aD*DtIhXx=E-))S`5wP(_^x9*vZa2~~(%l#U8j)M?=1 zNa=`B1*iq@|JQeXz9aOF(ECDngf0n94E`keZ15eyUBL*x{qIHW05}r3BoOrf%>QwJ z!hgO0LciVj9pC@+o%HSX#k~JS7a;B3<~`T*Ydi(uEuJl&3GQFGKkR;^dpXwqUvoX| zy2-W7<#vA4S#jRzT;TLOzVCR*am2CI;kN%A`~Tl<-(?Tm{%ZRqp8B`lcAoW*))%ZN zt-Gx=@C|^UvSjhR|MT&@|Id$fkKgioq@tS#OW>c)R{PKgi)eltPu8gD-a!zoT#h>; z72P^W60hp1=*~e9tXz)UBNg2^ND{B?sp!5z5UgB|+aeX+Hb@e$=&9(gK@hB5j$0!Y z-84uNxAatW&mag^YSQLNMYjx+#LIgsx?>OoD-~%|q@o)JN#e$yitZN#!Ad>a5UJ>P zL6W$>r=q(BL9kMd)F~Ff*@F_MQbA!-6}{D*L0WZ6xj7L`UtDNsny{! zO#*vr#!0Itmgx_GQYihYCQ4DJ2Vl$jwKh?TGVK6c&gO|L7L@4#fKr$QH@h1xjgjTu zWf}s$J;qASWDTV#(+hC%z&J@mDav#LY|Gyq)uF+MH5fV8e{ERL0F2e74Bl)9+5 zOcNkfQmW6wSeceUs05UvOd}vvQYz4bSoyGA2`EMRkX%WrLcOu_4RR%*6y@vXN=hZ_ ziIoq^m4H%|uahe&)o4zvd_b-Ql%l*}uB23??pXO+xe`!{@;d1s}d97r0pcLgblF^mgF)mzQEg2msMR}EEbfsR54VPC+Mh8k!ULhG>sS#ts z<>ivmfl`z&m5i>`h0)>iC6du+pVqSfzZlvT>IqqbUkg4Iygj%mI6Cm{z$XH=z?%ct z1=a=T1Sb0b?f;SgbN)yCY5(oolK_70d&>6~-+JE!@Bex~?7ho-sW;^LAJ3zn6WISh z*8L;&0o48fVLT1+gV_JS&v~iyY^TTZbI0fLOu&TWsAH>Ro@1>2ANKFsKW~53Ubo+4 z-)_Ia_7B_3ww&#tZI;a@+W%jnX+b9|xXon$zjBdwJQk_YzMzw3o`~*Gg@#3#Ie9%UPMn$5cT)ZGADzq#T6+URALX#paRQnMrw6BYUu5EH?E>R>oxqC$g$z4(J-28f9YtqF*U z;5MC@sL*qO7-+qT*NKS=?FER5;MNvmqCz(TVj>9sU??$Bp@$eWm`+SoXcl;03SH0y z8_Jy(8U-){YCw~isL&CBm}ossjhLv=0B~T1Eocs0?5wcZb6_Q)Nla8&;s>3wPE1r- z$2sy^%^?jXCMvAo99aoy5)&0xYG4hmr`%6WR9Kri^r~Tp&URK6Owwo*t;R4#5Sg~UW9vfn5bOXRtkxU%7V61NK91bx0OO-qSA|$6{go1h>6NP zM3R{3|6~oi2%8_4mJR1$9-6Rkbl_-+9 zDRbCA?(uPz2#!Ywj<*mKm2=fYEyP4+1`d%`=pS`}n5cBANiD=gC5%HPCi;_FiHXWN zLyk3xiOShH*2TO*#6)E}4rpgFn3$+cQ;!S~6P2@YMEiY*5EGTDLk=~FiOLil(%!}4 ziHXW&949f+e~Jn*QJI8;Z0i`Uj0|oXT{olBi8QvW%+e0@Rwg3!29wZcF>Glb?VnYd zfK+y~j8vMb8Zl8Bj}*39j8ZgWqB0JVT=X!C)V$2z?#h{nHCuF2n@M}Zm9a=NTXZ5Z zQ8@#F2TjhTzdh8ts&GxTG6tDkj4}Eh8ks$n(-F<(E7NGDxwMn_|5tGRzw!Nl&ftr| zT5w;mC-9HJmjao<{y@zCFaKx!IsXp-Oy3`UpY^@lx6^mF_fH`HPkJ|d!=7JyK7p10 zt)5QzuiYPYzss(`;KXHD@`DW)j=QKR|?|H|6Id(Z_IL@&D6RZCp zw?ByI|Gm|I)V{-hi9KdN!}fpp{=d)I9<>#0Cu~>P!q$IUKWD95ueUD6_W}IK@`&ZF zmJOD!j^7WvA0S-q?|1~CFxu{qUQm^u3x9CqK;^c2Y`99-g+I7S(iz=VdM(q@mD}kt z;VRvh>FDSMRq3%zM^|p6M~AC)S*D}ou^Lr+E7Q@H+vic?D&3Xo=yjnL@XD#6Q1!o9vxdrH@)-nBW;-rENmD zKlu2--D!iEsL~uw#{iFOpYaAUQKkI>F%kSoGsW$%(%yiW2!2o_CaN?pASQwjjzmmU z=~7Tuf)5T)OjKz|DE^>AOjKz(Kt==~REdcyT}A)vt;9r?#sS0xPNYFhRA~!9Oavcn zB_^sg10W`X59-83l@0*JMDT+H#6*>)AH+oP0gafbqTaVhOjKzDKuiShZxRz#R(}u^ z!S}1gM3t2v#6;kcCNWWE%?B}I)GHZWOF>LjX$U|}n5BW3sL~;Tm@o=$7Q~iX3Sy#4 z2Z1|JQL0W%ROu{mBVm-H6BAW>3=k7Wky zO^8TGZH)gN&ogq8gOy%mFb`4M=sA%8oCmsQRTkBqpjpsg6?PK}=M=QXLW#RgY9hsrDcys&1(c ziHWL9s-x6>5EE6WRENYw)gjawLQGWcLLH5msM@4D>U{7*v}%>=keH}iq&n(+@O-q| zA=M! zq&n(+@NBelyHtn7MCCTAjyfMa6RjMV>X4YI+$z;k=YyxBm0P4bBql01OLf%w;K^v^ zCaDgIiOP+LmaNVx_u3#PD#ws0S)C&>Q8_A9Q741PBb6gU6%Z5p{{JUCLSMxG|HIh- zAHx3s$AWJUZVbi(e-FGGcsOuRU`1f6|5yH3{2%Zi^Dlg?;{L7s3G@KgxF@>4=lZDYZhQyebmy;~kKq{rYn=r)7^w5@#>zMZYBi5%JsP;Qq#SJB=M@A znrSNKJPTlEjTYRoy@cf|UZbAyU=-gCuc%PgS=M zf?%aQt&3E3_aI5Utf#7*2SKn>oYqFFx_6KyuIaAQI+)F1YO`u}xJuVxHiKLJ8qHAoV#RG!7L>J3Q6R$@`=qTcHDawVnuER0nT%9Vgn zRIigODHUizta?DM1caixU#_H7q25^aTDcMsit0YOl2VC!V%5EJB_I^lJ#ra6s?w}j^=i2i5Q^#!xspznp3?TVf?*DT77Jml$OqYI9A<+G|qdbX&ORN-H31?6eEVC2AzFQbpxV+ zP>dL*RiEiItLqVGw5X-looUmm>yTiys6~XLdKrR%P>iTPOs$#PRb7iL-p`HnM)YS* z>8`FpBultqq*7ldhpVfRWV(z9grd4iGP+VzI>XhKlF@-sR98qwSL(=waCNz4bRZPf zOC_T#wPT#V|Nps;&`Y6I=s;)|zW?tJ!EfRT0QUvo89Wx;9PABF3fclc4ZMU-!0mx$ zcoM+>_}`B{zykCEzU(XES%5QrcJC|Rn)e28uh-%Eil^>5bp6xy zRo7FllIsrFPS+CGWS1S^0`M8;w2!$8#Og8vP9xC_@np0{ZzIxSgQ!M3g9eeQj&hlzK~$r$L4(LV5e=dmt&BuPxm-IQ zkJM;tBr1HQMvXQ`qM}^3fGTPq<^XksKPgeq#ZEfN*Q?f|N&(ZEPl2vyW*StKfo)d5sdqj{02 z5UQxru1HiAn**q#M&lw;AyiSLRgtKuAO8CyHJTQQ3ZaS`tqPXFjXuJvy#Z8FqiMO+ zED2CWjSdD-MW`Q47!Ina(XwE>{-BrvsG>%*0;nRiNrx(GbRmEWT5s!hsG>&00jMIh zp#`d_(O&?n2mw(T3RTqTDh3UvLlrgJ2cDM#AT-&Ba%YX!0T_WA(1a>#^ay||T2E7h zDr&R@99Y2*ngbU*Yb^g9SP5uC6*U(9L8q)k6*X3Nj=Ws+OhciH8Y?%Y{X{p>AJMQu@ADTFF& z3)@N|R8hOItrS8PwFPaZ5UQxnZ!3jRMXgtq(h617=CzeVsG`=>Rtlkt+T6BM2vyYP zw3R}rqBgs&6halXZltU*y~Y4l)Gk0Ipo+#PYtTj5T#IU90svLiX0@9|sG@d$yIF)P zYBRMgbc=`mm<)4>u~|xZ2q`9vwK|0#($etA|>kirO?BI(Hyz z0IH~+r6#pN6}722blyNxD^yXNGUQkjs;EuIu`cEff+}j0a6mhY!B9o5Q#~>ORn#Wp zi1zyqfhuYfh8$`@6}9m=q`iy7Llw1gI1Z?yaf%95Q9Ba{+2}D^?H$}kx^6~oEYjG( zGD|zqTRQ`xH<*Mri(yMUY5%O+7^JelWu($fB~+pB|9`C`^x04$v@+9B!Sl?;gY>is}WO)hS`*+l` zz;as0YeV(}4EsWNy}uz6dejIBQ0~DehwHRmXcd_xP3o@GfT0m(9H`uTcZTbOHe}TAik9 ziD80ge4QQ&ivgjB2X0jxghicBX*vdYSaV7Z!lF(u1i~Wpp=OHPU#HgrVG(+tMp)G8 zVnA4g?i-1)sME5b#)R%0p0KFXolyKfg|Mj8e}JS2-KP>3by|%6*INmTI$Z<^3!F%U zu&C1`fUpSN*GgE_=?p+vgznP`i#qK92#e7B1_+BfeEl2KTO(7>beaGi#pwh zNqcEM35z8A4dpM+LIC)QnB%!6!oA~hlE8vAk|T7cgHuPb-z@Hghkya)lsTA2#dN`szbt}?vd&!^&Esn z-7VE2VNrKUb(G2u!lLe!>X5LgJES^FjR#>-w@Y z_d!_HJES@!ENb@(b%qcYwR?m*8evhpTdJeZ2QNfxC!{(gENXX2b=3Lb`DpD`a*YwR)?kq{}FsC zm4d-)}(cZb(YnM{r?4g-(Qbqe8*oqzKata`QNxcnih0{ zLcbf*1?rHm`MZ+oj=p(Qh)-X~tt9!fq=puN>X!I_rW_3@vj|PJGqsB?Q=k?J$95e~kZtc`( zgI^Qvqj$h2T|25)9R$%zCuwJ-Pqz+|#H)Mybmt%lRw~nuNS|&TB#Bq`^y$7q z5UkXt?U6p+Hb@e$?CH~8gCJO`O4}lRx@nLkUeQz6J%b=vsYzQSb=@*Z61Vi!b;lqG zRw~lwNL@D!lElk<>bhSL1S|DuQ>3oj1xez@p1ST91i?x*+7PMhWb-QUe;6Boq`}(ai(h{b=@dP64!Ls=@ZOmFtszaI$WnkFq^@MQ`G4WfKw>l zsU}WQrvo^ES?bGe;uLk70pJvZ#1#wbM-U90!X$XOO0&GXei+GodyG%9nT%R<>ACeo zNMZ#sPSS9S`VENXOl=&i)thB=>enNhRoOUNtvDCYs2@ZctFl?z()so45DJ{aB(zaG zS|GlQ&Z{3l?iOk8lHU4$#M1#7pO&VjRG!7L`n5>KwqjB0qTc#GxspU-o$N(EXFtM8U80jH>6BUe(YP;achORfZ*qP|nEq*S7wSp90b5^#$84!M$2jpoGa zSIL!tQ`EQ1m6VFq9jjj{R{~B^-zHa5s?w}j{R+7faEkg?xspYL?C zN_C3G>X*xvfK${rA>4e%Q!7+tMtvjF%x65}6!i@V9khbdz$xnMaf~+_BdxCvz5|+Z zc6}XEmYJk<&8c68F#cX6jod6LEp7U6tiBd$ocB!AP>If&USET7;1nZn}Gzt)v?u8ixrYqx8KYo2SY^Y_lrJ2TD$&Y4b&<8zL(V~=Alz5(E6d(M8)KFe;i zeZf|>?X%6Z{>%D`wScDp&azr9pR<%K*I2qc{(@5;^e@^+CxaD;QESam0Cs@-XkD<< zVIHl_ZynD@`{-XpI_v=T(ZPsxl-aH0nP?x~j7Wzapgx)yk&ZIAbvzaAqo)z+5TWR! zixKH4GaC?!KKdDv4iSnzS{Q8BHvRi3)&~ehAI%IlYnvwmq3EN1k*FxP2M9$Ujf_Nv z2t^;Qi$q1SJU}S=XksKPL@4@bTO=xq-2p<;M*|~KAwtnd%OX)xtPT*0KAIPa3K5Dv z+7*e4Vsn5{^wGFTRESXY(W*#P)G6S>NFPm$M1=@NA8m?6MV$a1i1g90NK}YW^wFY7 zRMe0E{gFPJ6^RNFiay#CiHiE+zc129qasluLeWQif;DcVkFaWKfKc?&s9b861caiG zt_29isD2D#IE12)HU*pW2gM8^6n!)(KqyA7)e(w5Iu9TOt+(Xnf*>_3*z)C<9q3BzN!-Gy)M=1I(?jQLg%_j{-DEgN6kE{eV5sJQxa2NnZmm6n#tjhh8-7(Amzu#r;Do0ZoLWZ_%(r6QSr^h(oxV{y%%y0Ut+k_4k^t>=hf^ zYhxjNF0yQdZS9__jc}1IaIeA`YzvjMBwNzSx|40W;LOPdura20kkbpKKuAIwAqfeD zgand6AeE2;frOBRSfFB2hENRihFEY2#c+io792t` zJkJmd4xt#HYlsDhPz=v8#DYU8hA#w`OU~NHAcSJL9G-*_3i-{_G{Uq$O%VYAAr!+G z80Eqt6vMNPa^Vn);aMUV%n%b_6G9CkgkreNC=-NG49_&m1R)f|rP-J=p9_To4+_IG zj50w8#qe~aOb|jbJk2N*gis7mHOd4b6vJVoOb|jbJO!9Uw=W6YJ(GsdhxBr=oKxT{ zLMVny)JcjEisADh34#}7u1pBUaIu<65kfIM8ImA`LS|BiPz;}|OG}4P3{QfzGR!CW zyP>ewf%sr-;_yUB5NjbN;0ofD-igDUIwc)Kf$jeb-AAlg|6k&H6>R>SV9DqV_g~## zc7M!$J7iq=D`=pgfr17K8YpO>paG%=#+<6WL}d4q1Q0iIHa3WxI5Pnrg}8}qkETsag+m77CeDlp zRg3 z(pnrhaV7*0(#(qM;wH`nfsg(3G(ljpRv=HgX1R7*l5d9Pmw=ceukkf2ggkeAEqrwJxTuU^5Ls#%fWFI!-r_g zQBRY9t9r^|=;)0TtdCWZ$o%hAP64DX{XM~s^o-b-7K`g`#8 z^5H$S<>0u9;Q`um)Zc?A%7+uQ<>0u9;W%wM>hHl<%7^=D%fWFI!@FtAQGX8}FCXrs zEvE|NCWd3Q<*2_0k74`&$IgRH=P~g*ZhzA zAMoGrzsY~pzt6wRza3%?milY_7x+v3WBd;G&+KdLkJxXsUuGX+KM3a!oMI81U_00= z*wySJHp0$i&t?6-fBXL6`z6FBJnj3U?-RcF`fm4K=R4%<^R@XdgB1!F`R4hi`6l>0 z-oJZ)=Y84xJ@41OpY=ZI9roS?Cm8PWcESk;tGpL`E4TYHvG)}ocpVA;^7A&;^CzGYImQz)xF8x z=&p5N;6BeCbThEl;TJ{E7d=_@SvU>xzM|7b$)ZHjj-tzpRuo+X>mR~J=fEC_e|Ej< zddc-o*c;*Fa7Mx{5GS$EwbQlLwZ^r`CAem|IF|?ZNq7TxO8AcR%g#?a-|xJ`c^yDk z_$z4O|D*<3w9UoX?cfO*Gk;7Si?+&?Fy=xQT`5yMrmm2w3QTR0sd<>XT&Ct?YO_qu z!PI3kRXS!0i<)F=2BtR2)O1X3l&NW$+8|R?F|}T%!kAhoQo+B37KIkFXst}uh6EO^ zk*OL?t(K{3Os$fsDom}EsY*<(kf{i!mdjLmsFX#Q%G3pzxWokX9JTkT1*T5pTOkIkpBAL1bQ!bfm#FSH} zmSM^vQ%f;r7b$PCx82*uBAZNY^aa{)0mRiZu?2`jGI2Q&2W4V25C>#p zt#>)QUnbT7F(?zOf!HS#tAN-m6Dxt(BNHot7!V21RL@RN7n_iYR?k*8E)y+4^vlFm zK#7>!54@8$ttOKG`B-~{% z{)(|3GSTU7Wp~I#2M|%2*a1YlOhkcblZkd9T4kaQh!&Z+(%r;fB@e* zmjfYnZcv@gKuCQX5SIZVb!~7Q)}EpPR_fV+=q>7GrH&1V9w4NC4Tx?aq;3s}T|h{^ z8W1~ykUBLWx`2@SG$2}vu4JVy4T!6Nka{#Ant_lyG$6JEA@yfKYy(2-&Vbkogw&h4 zBCdn31MF&1I3Nd1iT148P@fQSJhbz?wuxLR1L z7XxAk5K<=wL=*_A4+Ekd2&oGLq74YC2LqxN2&n^e);W_-#7g}a5c{18R_eZh7z9G< zy@1#Ugw%Nfu@?xb?*d{E5K`9#!~hUd&jmz}vxAj7E+D#rkoqkkb^#%ETR`jtLh7}E z=mJ9Ow1DUYLh7>|3nAk9I4gBoK)lPbpOtznAPxf|byz@L4TRKR0dWWjsk;K=AP`b- z1;ha$=7{(J5qI>kQeOo`KM+z^1;lP3q@D_hJ|Luy3WyjGQa=SmFA!2UWnX4LWxt-4 zdMO}|+26%VofHs9fspzrAdUbbbx}YJ0U`BJKqP^XIw&9z5K{jH#GpOSO5GC>`+$&o zCm{9$A$3kb>;XdRn}8SqLh71;NB|-AOtzJ_TWmM8QpW_uN!u}2>X(2x0ff{o0dXA= zQm+KWwLnOn5)jt_A@xZ>90x+`l7KjD8)T&(35ctKkUAtF4gn$cM?f3|Lh6oyH~@sy z8v(K3Hs}ku7@OU`p1F&;(+6FV`1>a2qz^hG@%N4J8@eI!_i6YIosjtZ2KWtKkofx) z{Duxl{Cz!h(hJ>>&2C%I96juXzDN9dr|%AR4bWp}e% z*v0H@cC7EOzL$MZfpvbT5Ba)$>wR^;sXnjwE$@ro$Gso&-sm0lws@C&dGEPio98vp zvz|vGj^J9)ZqF9aV$W>PSXf)|vim9bC){_sk-N*i-d*RO>h{7O124i(gC8oov1ky! zeanmZqH`gx;5FE5@KM(p*R}99+TvR5nhkpm{?+-i^C{;ioOe1Ayr1jg#hmK&INo$T z@Av|oLwEw-m@SS)jxtBk{f27IET(5CiHP=EHR;nLt}^uQ#lkOCNLa=i4?td-9P0} zfSB-O4*7`*KjIKeO!zK`e8hz3IOHWJJj)>uG2z=BauXAt;ZPAVA;}>ZG2vYtauO2` zbI3tV0L<;g1c25?OaN#ZVnUodOuS#lT}`}S#2q5tFZv~Ska+(U?f~)r&$<1?`!8~X z#QQ(s_7U&D!0jd8|30^ec>jCc0P+4cT!MK27#AnrKg#tJ?%3c#QQ(xI*Ipx!gUbu|CrlBy#GTkO1%FP z*G{~Difbd@KgqQc@1NjWi1)AIt|Hz)$~6=3C%Ns!`-i!0#QPC$EAjq0+?AyJ&bPQL zi1***wh-_Cj=P+A|Ciim;{9K6ml5y3!Zi`^zszkS-v1f5k$C?WZUgcDP276o{TsP; z#QUeXwZ!`;xHZK4*Kn(e_m6U`i1+KbmBjm#xD}-Pj=ylriTB^;E+yXo6L$&m{_9*L z@&0ePWyJfhaZ8E!f6X-z@BfNhLcD($x0rbU4sH?g{_WgC;{98=i;4Gd#-(H3Cd&k;=l<~Q;*>3$g2*duqRd#$_LUE+2YyVGpI+p1z2*7=oHOu#*Y&Ve=}OlU@cNH&{sq>}KjD19c{8jEYH?oboaf{m z|8)Gq@wDS1$DObMr^B(vQRO%fR8jc*@6rI5Bu)vBxsyv0=Y+@H!6k{4!edr(N#d;V zm=#=-I4wM8IhQ2P3y*2!lEjJOF%4XjI5RwE5tk%R4UehklEk^;&|6%RI5`~p4wod( z4u?L*C5h9+p$EAnaeg@T0GA|A5QpBwC5bb{p?7mh;uLY{43{L%5r^*QlEg{k(0yEz zI7=Mbz$J;(#G!Ruk~mKsTFWJg6U8CGoH$b)0%(a-#UX$;HCL=&Clmr`Q?teDgy3Je zWNN-xlkhy3OwAZ;5+31_sX1d!!oyrLHEXO%c!*1;=8ZK8AL5d!nPW}D2e~9e@^u71 zz@fv$g!ggiYGT5BIdq7aa5;w#5)&@t&;epX6NmN_6E<*YkeINRL;Hvct2nfmn6R8f zdq@d^e{yJmm;g(T62yehaVSnqc$7o^#DvdqXg4w8V;t%uCVZ4bF=E1l9O@+|e1t@JSABCnh}1p>4#3PjF}}G2!DJx{{dC&Y>%a z39TI3LQH7k(B;Ggz|Viyq;VD;-lV!}5#bTKgjUe0=A z!k0L75itSY+6BafFK}o+G2!zZsv{=ABG+1CLNAAEhzYRzwVId!m{$=K0NP4o0zexf zCIGYoF<}CScv6D*PaLWsCcv`TdBg-*{W_PJ@Es1#Atu1fc_A_3YaA*kCcs;J0Wsk# z9GXo`_%erP5ffmMYZ);i!J(PN1X$QyN=yLEXAlzr+UdjufOZ-&0iY$?AKqdP5$zAp z-#A3HKRhpTh-iO!Uf>YX{_s4nSpQ!S>;G>GMMG=B`kxm3TksdbCxZ_KkHSfSO~Lx$ zguuT8uLQmo7!F()=nHHK@PT6gzx}WIKkt9P|9by`e>KGF7yDi849$WNU(RYhJU33QG=C^~DKg0D8*RNdPaDCEs99H(P zb=9~+u&)2d&aXS~avpKEIafNTI|Gil96xk?)NzMnucO&f)3)ALX)95gzT?FZzS2-=JnRwOoZ8P|lun-&Kc#2DcA6VAi(ie}mN9I;kM+C~;@HFDlE8x+A&YSsVp(zE zLA8**1AX1G#P~g8k%6_!BKdM0pIBNSSW7HYD<_U$EDZ&gD6^=Zi~|`y zG7ji0EoS?aS;(^D_)g$xQ*q41fu5xiwoS=Fk`>2*7MJ)Q)5scbhWlluzQ^>k!nvXY zZPE7eI7YNIm?|sTFswd;n*3x3{eab9kVR57@aNMZkn2AGPOC#PJl^i5taWrgkiRWpJu#<3L zY}o|Q(>h_t<0#qEkmt+F^7wKbE?Zjf`7*IQt&ljJ7P5FmnML(v99LUf?0H0)g)Afv zvjvV1syJric-zv5=YvWPl8`tQx46XptX4=Im0LE!{j6R{9HLtqa=)l7k1xlux~28* z7m4L*g~YMEki|EZSyWHPLA|BL?r$oykcGrCzQFOAien}Y_AQOLA5(IWgv4>c#U({A zXoTbz!u_(bMK7p?>>C`9qk~IBMZZ^;#+TzT;nMn|-xEvIiipFAA&Xa(SyWHP@x-OY zMXxBckVVAd#lZ2rien~@GcJu3J+I^-iHJjxi%VQT)`&P6M-w=q#6)MTcYL)} zqH27?m*Xg95R*JS(Ta(qm`g*hzo{xc8HY2M7Q6nYth6j9j&25y->Nuf;xOmZi0ijX z4w9HS0=l@w`HDu&b8#?q*+l0nDlvEBZge~jjfMhM!xO$7$4HmfyU4>6t&}*18nPIx zs`F$VObt?wRn}RS6312phf~Ed69-v?lujiFNlF}lU0mXLO(SIq4#h5;;&@FhWmiwU zS(Fq9XP1VYssRdLj-#|o>z(8QidIq_#SK}^Qq_Ah4(Bc{cFt1PTb2|@cLPVUien}Y z^Dd1zitaf$s+jieK&!2Pmu_BU0Mb|!kpv;mdJ^cxkdE~5L6s9afo?o#33j-2r^E<(dWg*wzpF<3Ug~~ zYU|>Ya2$Hs1l!wM5xd88VsUoWfP^o{(dni2cJhEkD-f?`avU>neoG_#_+zUp9gHyGqDJbYEgT zj)*S}*;E4&z8pg1K}Z{UAfgo#V&`!dJ*p~ChA4Uv($=G_vMeM-)nksUR2(xQz8-|M zU8Uq82ss{N@4dw(X%VZ0%F62a1c<`-mW@*j2=YNdJ`}~M#vXh*#O8y140-IKl@Fr# zaTZ5ZwVe!s{2(85L|NOEeB*Fre`#p1ilG$8_?JfZDxX(L2bS(d$cg>`N5Je2y%_px z=z|6OzhM6>CprrDf5-TO{a>*E<9#iOwodWrGMEM&=uL11`@c3GF4+I#0K8!T7wmuO zvoOA3{}=54g8g5x|9cY!`@dlSO9#Tn|1<4>smH?hzcX|-6M8Q6xzO$KqwrVIKtTfq z4HPs`&_F>01q~E5P|!d@0|gBfG*Hk$K?4?R0Co=tw|;R+=!|A1%vAB1nX+9<67%v_^11rs^DjolPxp)dqX?pn?A=4dD3y-}&NjwapFvJ@n_$TcI~#FTkIN zejIus^la$q&=aA@LXU2dxUTAh`dZ;8cF*G&=#~}v)9sFzXkFb~EuY)fKe;E9J@Y}(sf?o-KA^7Rw zCxRadzAt!x@Xp}P!Rv#^gD7|)H~{+%bOqaj+hE7R^}&_FWx<8Px?m(YCs-Do8Y~V@ z2!?{*pd;|lz+VDy1zr#QGVs&Di-GUL&IDhFeF;AocqH(#z=s0w3EUgFJ#b^-MBr%P za9}VH5A*~&0xf|n15JUof#rdQKz#ttdxqT$O9N8^=LW_F0s(ix=Ks6@ZU39FtHCS& zANybMKkI+m|AhZB|D*ng{SW#-;6LNP8}>N7!GEoP$bZPc*T37p%OCYO`?vTv!u|)B z_!s*Z_^bT${ImVj{U!d1{;_@*c0~9$`&agl>~Gm$voEthWWUdTn|+G?3i}22)9feM zkFf7!?`Q91Z)UG&kHeY72iO5N2KyzpvD@Gb=Jo7Kb{V^ntz#qX9JY*|$`-Q|;CyB; z>+t>4_ZQz=zSn)f^!?QLBJ8U8E#KFDkNZC7d&KuK--mqf@!jjY-FKt!gzu>Duy4>8 z7gsG;HTU($T3cGXy1NpCjZs+AynY}$5N#evz@7aqi7p5~mmUnacJ+o856?=(`sTL9 z`lA(d+oNr<_UK%3Dr#=lOy(J5O2iT^-LoA+37W!r8K{`YW3w5{LaKbGQsG!z+dyAS zZ`)wFvn!s6b@aFND1L#Ex#TaLhoTc$1_e4$i!HZ0R?;F-X$2CMIU-Svri?d(L}i|l zs1zon=s0soL}-zSSb;<&M@Me-rEh=8TVcR z*Xj4Jf$Ow;FM{jTd*{J5d~Y#ar`+p@>-qPxa4orKFI>;NryZ`v_pFEOj=3AxCv9nmNPd0UF+b=-gPlteRs`-tM{(SaP{2ffvfw@gK#anb0=I~cW#5L^UmdPb=-+% zwBL#AW4p5yuFRe1!1(*$cN~W6zwYRO>p$;kg6lu-sE6y{@8IG3w>zf5^{;n~gX{m? zVT0>mZU^l#f4==HxW0Wm$jJQZcBl{Y$J;?O%pY!_4cE7B=ivJL+wij9H*W(SGQYcR z6I_3L+of=QQ=ZJ<5o)muS(%&%_+8JS<*itF>sTR}6-FK%4~ z*H>yK^$beJFB0@`CZ{>@9_`n{VkgzI;21|2fb-E4>Jvp4O5>vwJf9GP$5 zbU9p~xe3r=zI795kNM_Jpgrd4o3M=ExCz(ishhB7p1iRau3x|LO1OUQ#s;`PaU<5@ zS8qHQu3xzk`#isV8gOJDKOKYXmresZ%wwmq_P%%;YwruEv5cQTjqCHd(^xZ~J&i4; zM{nqd>t}9&cF27C2CTzR-B1D7M{bw_*H7MnaeVlOBDg+u3ZwIhQ&@W+KZUjTu~S&a zkDkKydGHk0%mb&+gX>35dExru>!BSoAG#jv@PpT10@n{*zW}c9zaHcGzUxci`rhj? zI`27&wfF9mSbJwqVi|`|;`-cw5^LtZlM%Sydom2ydrm?-WbQtJb$Hi_?Qp&G#0I$D zabgi%Z$D89*V|5DbZ$L?wRg)2ti7AB!!q7<9j?!f*I~_^zHS9vZ@8`ouBWcU?eO~R zuntdNI{?=c*LK16x@$3x*Iv64uGd_P(K&uC*50vevG$H$i)B1=Ew0bdHCQvrYg*un zu7P)%dDk_#9Ui_0>+tGp{BS*V4P2Om$1#ovj<>>f|8b1Y;Blyi0zZ8qJcQrn%aXVqHYKqRHzn7@bz^cFTsI^! zj_Z?Ua9x+g=&Vg*?X5vrd#e$aaTUV#S&6XbS0LQFmLndnm!e|0UV^ZsjTp>jSc6Nk z0vmANOR#v0aUB*-!Z=Q{`Iyi{q19le-w-Shd?PT&f1SUQ{gm%jU!!-s=kxCGxrL$? zt`9oDt)$sX{mhtz=>ByCyS@P0Yb{!CH2C&gTYDD3eu zThN3pINx|%Cv2D79qmu~nc{=-M6{>5AF}G|i8jaKYuno%-`LeN&@Fb+Vd;Tz>VYtL zuTsy1TjJrKXiIzm_TLT1dc%p%Xt*;LPlS6~+B&;>qjV*SRbuQr9pT}2JRLm9& z@V2_V%(AMDqyl=n+WKXd)r#MAMMTRhs--eKt;S=X@x{oZ({*X}O78~#iX^~28E)u~we8BnD8k@cW%DvM zks4AHOmqQ%!$5z3v^NpPL>M9iTBH5iJjLh4+{FF=g7(*$(4^p=z$yP5eka@N+wXne z^SYU0`!>M{4*r3$Ur%@sgtJBQXips0$xD2Uy@8OI*cXYNIi40WNl92afz)Ydp}i94Rp)gNDmriQ)D-ss*OSJ3hKHpux3VJ3=B z0XR&op{r9`<(ZdHKfO?A#oCCvoIZCBp%hIy-;7_WE+bi9Rx{QKGf=d|91^v(NYq+^ zL~V{pOh;4BGlN8Jo{^{(rlDxDIV5Umk*Ki(iJBadn2M%MHiJY>o{^{#!YF#KIV7rS zk*Kx;iRv7Yn1cQPW0-}Ef0%s}&gyIP4tc)i{+4@U(L`5^v&Hcd`$ueVGjGEKORsv- z>{c%Rre~E=n~x%b6{tnDGUpMsh){v)fnX8pRk6Jcc*C1+2gO{FFe%C5d!6EBiDttKGP1zu*bY*0PfXQT=v_^gd&Ay$P)SI-7jd#jSm)dN48na3Ri-QB7& zJ<#Ajj;q2X4K4zZK2;Uv#wG`0@>Kx}P2J@;WFDG3%g5|4LEF5B;})^}jVYN&OH5(z zv2A7I%W*>Ua?nVfcG~pDdiO{BV=CuO>K@67lcP_i;8MvN4qR}lWRIB^T`C4&Eb+ab zW9;i8esjRD&g%g};IZ z3L5y&(g2#W%*z~De<0U!l|ff3{b);esIEdYmU@}PO^4MERr-v?&ztmkX7-RY7|(nG z4Q7sAtSiyh20X@Ckoy4KsH2s>mMV8EJ)1P(6F>1Qkb4P?ze{r-e~;vQrLRrO1G_xQ zECxjv51<3;;WvFwsR6Tc94;|u+K1ji&FYOcT()B9P0uS2z%NBri$DdA-N<2T!kiyX z8x_2@m!Lxnq28va%vCz zi>Otb>e&{dNUas9RcmF=BWl&cLUgFc3e;4Q)mjPks8wb0`5Q#7$`ooBWBZ@Ac^H4O zr^Ef;q7y}oE9~rc^x8jb`#jwKulYm#I^24DvmInqJ~LH&S{wkvbe zmLyz`4y?s(H>O?2U@#|@lTPLxNx*C_I+>SJ0#)26wVLH#Vl(2`Sb-QX*IJS|kBITY zW$3_aD-csT=+yPse`~V2=VV?=#FXwidR7^+Cd99@08PXURlDox`Q-Oz9a?m$k2zF=2GmYF`kY2^-O*0XXP!}kA*x0hC1sFZ7b|6uUl%cDkY5)!WsqMNJ!Ozz=kPPguZyHI$ghj1GRUuw zsxp`XA-0M>2dz7ho>Tm8D4c@yoZ@RPy9U=BZ*7EkHZbQ@$n-mFC~A_vzPi{+dRAFf zbp)kBeq9`;LH@!Q0=}@yC`*I-D|{go<3D=cDaL=RvHgDz^9mEP2iFAF`R`%x^ZmyA zhPT!;-+ip;8rS!n&pW3%X4rdeJ@C-~%|GOBCHTMS$EPFtzl`J{j&OIPt6SiUTD)89 z|Dw++H4sz!zhveSEp`hHKy^f-VVk`RMXmw`IC`|+FIk4OTT>|o$P9Tm0m1jxi4HVF ztxeCfsC-{)0rXaTHqV#LOQ|iT=Zl_Iwmv!#za6AEJu_A(7oB&=>VzHWz&0xoQ~ABr z_17bo&FdxeQX;1GdeO7Wh(!^<)e6LF<)ZVBSgp{G4qRykVk)1Py8e2^vU$8@UP{E2 z9xr-U8L>9RUttAeHFD8;N32F@MF+N6ftbqQrLMmov25NhnU@l=YQ@lro>fMy1@V_# zfmpR%blwrG7Op}EHd}$1%Gaf?zaFt{o-Ub}5;3Kxi=I_RtQql_S%FxUTy)+Ms}i=O z15H*SrV3P5*I$oVwlGzhml846xkH9Wi`x*t$qK|OiX*u z%NC+4^HL(F3{jC-#3FLhc}FZFT!9X(w*oO$fU3Iwdc?AYr^>vPh$+KU z=~-pOwqW~z4AahprUnlNUh=+OwAgjEGwOK8vCjT0`$78z+ed9Hm>1vy6Mwt=y-f3* zX7xH{OCAK@v0^sc6S0gv%>c*jH0EzfA?+!MOJ8vte`D2 z#V8gz*xKi1wiJuImyTFj*!mQL$XffGTLgp0FJ83$tTjt#Emo^P$#uaeR`1!Bb3FT}&ZuXXKIhbgk}?F7dZeQE?9!Jhe|>kO zMV%ziE`3fTcy{SCFe$#!cbxA4Z1tX9`YKrA*`+T{5?JHeHOQ{>+Zg26d3Fu*>pZ&# z`E{OMgZw(ru0ejC-^L)n&a-QfU+>v9m;u4FOP_<*vrEq@em4}JU3yNj)ytmU*zwjK z@XqQzyY%^`-&sSRUHbfWo?UuYSyZ)W*C4;nvulvQ@P*KLb`9#U@P#ljJt+>ldeM|9 zAv}~G2ZQ0EhB6jw*n_6ELwM+VO?W7MM&jpCethJqf)PinYF3q%E@q8`L%Y$|HuzTR zgF|KJvyWYk28SBS*=#G>h3)@w%x)%hIy61_aInF@ioMSFq_4=k)RT07spwxt7r6$V zpK|=sQDLir$ISnsCCCK|Wp^VMtQ7l*;G}yP@p~>==*@j_#Ku5xS7Pp_Xgtx?+flJN z38 zY^|1#j{ayzOG4UQ-ejFewr#R%N>(yI+hhiU*$g6PGpID1L6x*!26GZEL=OBtXhvhR zHeX_^mEUZ!H8+coR1sS+iF?FGKzevCsVJ%Ko&XW-R*FzLdLmRxB3uU|*sK&GGI}CJ zBoVF!5g02)5Jpb~K@#B_wD7Q%67ZuZ0WV2#94)-sN(so_5$XLj{W4cb5*$Md4_PU} z=zormqJ;;olwkC~Mn}-X16E2f`X8eqv~a(b5{&-0D2WyhS}DQke~S=WxX(%nM*my% zuA#(UAJehySc5uVKpu*uEG5$Cuqo#eFex|E;71v{MNuJA7TLzZwAIKq6sFB1+h~|J zk8A^C+B{NY;={drz`0hQ-Hj*ZiTzy)?=JYbs64G{HnGpUx_5xKJnYp}l!twxG|uES zn^@kVT?z1p&d%<1mE;4*uA(^b@6wbeGl_*A?2Xe3hkdDv(y#}XrnoelSl$6_|8va! zOlW>E5%{S8RsS@0tM4}Ncf3B&QuozG-*WxQRqXh;F*mr=fX;Yg?J^wSxcuDnq0EPE|vSApE-m$vg8#ONS-YZN%H&m zq6uWlNxViRADLe&`M%^NG~UPTy)xOBbskIlyeXT#+=t2yUpiyU-jo*2lzFqv>a9 z&1A)iOMVp1SDH``r{`iVN8(PH3?BnXSTh+;w%xN%@s21_RxWvBxJqhIcYqWj%cLON z?^)|(PKwl=;_VL!rFYnc@9)Hmuzkiuh`6z|kG>NH5KkXc;vZ)HM9LQ{e_1^yan_W#r$Vmo~Yyx;Tu+Ox>LvFHKUW3EZg za>o(-?Y7_B9L!c)^B+w|JRI}5YXG)7R#;RX>$u+8UBRjLv7a=fTLR|9@&&+T zju}j0@9kcKIZ$)kYDY%`cdRO00YlZe241~8dr z4ii-@p{mk)OtOU%(x_L&L>WX#!<5G4(|}2t8B7F~czMA@5SUK^CNs@pqS_@_RcSpY z**3|gQLl)Ja(i4FrZgsx04AkoFyU3=Zl38s6!aF4LUGy3*9-Ck~;h zUZ$t4M?Ha0SMCTK+tM*h<$oM;DC}k0dfG;+fY?HsCGSadY)A8s?tyq`G`^`b8Wxkn z8OKU0DGZCCcf|U8qV3_A5C;Ypem_a ziV5?0IQ5)Vv#vPIpu?S5-(37}#oX+fOBMSl@}l#7%!1`;+(<33tFLW&ufoAA7Yd3A zw+*&+M-`r?>_yIN>yEYUiqG8;izQ@V6wV-?@&P^Aib}l9YJRmEK5!MyO?1mvEc|Nj zgb>~gQkzySfiql`LF;4=N#c(TqVvH0*PwBH65Z6YOL>#t%`GQo)SWd4sq7C*>;FH` zgvJHC1Bd-TU|(mK_%?VypJfIiQ{*UW%ef91I*`XE&tJZ4t1lY@UaTT zP+N|MJ8#5t=Pfba`Rbg$XJEU37nPz0FSB>o-W=ys_)#$4d9dD}17Mbz1jdN@&Rb%? z^En&v&jK)uO#w3s7Ca9o{C5BteEgW1H^zt!&s$=|^Eq4b-v(e7ngV7N%y=H`_-6nZ ze7=|&V2l{@yd{=ApR+0dEdZw86fmP;%kyB&e-nVgM~ImL#)vh~TVl@hIotD31279r z0W%5)Jr5TBHvkxXN|+g744L%S*z`FY^-lpXb*6wB4XYl^`X>Pxd=Qu!V2s%H*$?-m zbv3CrHqO`#Dx+I9ryv?v*rW1w0H($iFr#4F!!E|m*I>S-8h|k|f3MoXcx2s(Qzh9p zFs4zklu52*;J<>tgR zOzAh}D}YJF3?{WI@$!O6t-yR4FcHjQqS}R6RcSpY*)}1jQLhNBa{FN#rZgsxOZNYt znb787N8pS8XZ)pXmG7kY9?u^=4)+yBJ6(@EpLNc1)Y@;b-Ov1abnN>4mjd7H&1SyY z!YEkM0$9?&g3c80T5F;+6^vNYf+d!;kh3NIO8}`2dx=kj9AiwC6=_1vnBm90JGi{Fr#2e3t&nA z41mE$teOGFh$SspVo3`*Thc!TVAh%fW)v)G0W9gC05JF@RWraCv7`k{ENLNUOZvwE z%xY7>jDjUCfF=DS00tkJY6ci1mb74ri(JUrNB%gV80x-+X0K<=hHO+%L{R05zQj@?Kv8Q=U3~D}Si~0os<`PrDjDktc zgH8QB0D}*cH1R3pjTqIuB~~?`vswLp0A`sfU`Ao>J6|I(-^2EQpy)y-GzRSdz5Zv| zpMd?p4DA00JagRJi@pj;g};IZ3K}SAprC>Oat)yI2sZ4Et&gc!!_kZ-4Q?h(nWeOb z?T@g?vdzoH*Ty{#qv2cPM7U~(p5#`@EUBZZn3!h>_6}zLfL5Yb81}dp*wqX$M%I)ImaHiks&neqVfE6lvHfpze~f|u3V#I+6f{uKKtTfq4HPu+ zpQ(Y5F$3}{dRLy3n)*Gfb3p?wfZLnFV?yuqkW78nIBiv|geQ`A-oKvSh zyp5VucWfIqr|!TuYEIo@j?|pGV;reDbq6?7bLx(6q~_Ee$wtkoJFbzMQ-4q+HEZgK zM&4kn6&oNgwj*B1iC1h%d?Y7cu`ThHIq{0EiLc6uS8Pvwbxyoui{fi?;uYHzUz-!J z*sA!toOs1{B^dppiETp=zfM9>&h4JLt9|p~@whweigYiVPr@0^U@9K#*#{s+E_V~uGo`G(0r^m4LKsfaP>>eLZJriz; zhkK$e@d4O(BpmA%cRJ~e#S`J4mbT8W-Y8v3QYBre81?y>CN#DQi?3xJh=c4SR}m*$ zP(e~DQuUlyp%`EjS&zr`+RF8|^u|&uS(Eu0j&`^7#iQ-Bw2$|7waJgq<0~pEq@7G` zjyBZmWtLTKBo)!q)z&XFtycV|DK(SnWO|s~AU0Ti3n}Sb1NOZ!EHx1&BGMcwQ0g8w? z^Y7JYr(KWstfTZrKy+9G}d| zm}u1P2<953vEAF){vXdIn9$Z>f8ZJaFZ_$x%X|-cpY+b~%y-{b^gxlvb-r`IA_;=BAH-2AJD+(SeQUVwL~maW7UL= zC{DKJVbnTNt`4JNN~2K%Xt>OvQJax1FKE;Xw(|fDr#UoK>%uZM)}xVaO&E=0MM#xP z!f2S%XcPk)4l`)fWMs<=8a0A#GN55MhlXk~Sf<8$G_oxOqfxAgMs==Vs&fGin;A5! zGqU9cjcUO*3D987p`qFfB~xQP8rk+ip+-Zw4GIlY`rVj_!sxIWG^#SP zsj(i7Y&)LNC{{#6x#0;7QyPtNfW`qcXhbryw#^BR zVnsBRyPVK4rO_A*XbhS`L&(UM7c>OHHU`kxXATY3{wA3k>(R)zw+W46MKqLKo6s<& z(Fg$=d(EK1XJpF@8oXc&0vdbFp`i-m%G6koMz#Ph8pVoeDC4(in9^tj0F41NXjEim z%L^J6g3XWZ|A1`<6S^YU6Zla*qi#}~Jf+S==_aM&QoBiMIMQFd*=Sx84q(^2NojH!!A(lTnzNvSZPu~vNqhwR z*!mNimxQL^^onYOZc?R7l%^a>K#fb3no+m5nVM7Q5~b$Ut!<{})U9o%=G3iirsmYG zZKmebxkRZs^=q4{SyOA9mFo;?c*V7X3Ks(nkN9pYTnsckQfHTPG1$tEdy^sDea<=A zq3(ug@=Cv9$^{6-g$G8pp^HzTsg5j~IzEA#Qx~5=%~^Pd@jAuKfLK4``7e5hjd^;M zBc_>Xd@@Mb!-b|D8r;K$GD~UjREowW1F(n7a?KttG?|F30Z3bGI z^uxDHzkv%)#f)y?LYHSwQ~Z*aryoqRgbiHi%8kKluN~d7y~bUQYa6BFr=hN-540KT z(jM4GQ-jnXPqmhiy!boyq!q4T)wv)vwM>tNr=n>|FKl@b)tuHwlSxW|>}}e#^A`>U zv4yeye-5*U3H~eC7+C5*$zJDs&iftjBo7C>|7~@>*ZFS8Z|twyD{OP&A2qAtTj z0<#u%oz@U0z5F97ir&N%@u(AQ3(?g{e2nn9aWtq|2}^Ciig~F#=2{u_U0XK4xj`wE z-gA1^bgRA?kSnzUxmvB%c|;EU>h*veK5W=Daw_k-Rt7zC*?jEu>KEIL($h}Qnnvy- zKyJDf$kk}2&LeW*gI@s1;UkGnBd7AjYh}lWRa&X@h#dI#YXG_P ztw2uY;n&KbM=qP6pI-eUa!PMMJ!=}dYCx{U3gjxaQs)slSO-uA$em{ea;nt;S{d}n zWm^+KuYM6Z<;nnh)--aJfLyT^$VIeL=MgzrGY|pfCR>4=YUO}d20e1w)(_CDUqnv1 zih!Oqjhq0;oofYhf>!E0A_waWctCEF709Vp8E9qDBbRNh0loS~-9%QP#Yx`B94djuEb7=->N$@QT)Xs@ zF{uE-w*0t|tN?sV>aP!|olx|ZNV}p;ID6?!lSJ0Id=0YeoJI!ub?#q-{5mJFL4KVp z*dV{oA#9Le=Tb7ruX7F?eFP`@ukPxlJnrL z4a_+eGX2&Xav;;!Sm#EjXO)FjJChCa>s-nP`3v6&_{OSsnWFC2MEnZh2otlL(%|b7 z2tJzYWe%)Akn1u&gRW`%R|GPa>ez@%lXJYx;ikiyLq@1ui}+ELO3o}2CD}I_XEl92 z#mq&Q^)j?Ac_Dl=$1ccy)u&O%HGMtR=Z_fEU22D(rO1;k2fLv(=S80*9!5f4AK9bb zkhCQ)0A&{spaZ#@4bni*pascWXhN!d-1JbU4TL4AIyoDZ;Mk2Erq)OE6*RAhMebbe z{ay_A|17AvY2T769$hPh-u};eehGCW6dR7}5EE+FGQe$=_6f@tl#6Tn5P5dbDfu<@tiMPalZ-tCgRZ z<`+oc34VB3xkCYR#&7SpaDFqoIh3oY{!W=NzFO16bC)H5YgR*Jq}F81u= zr(4@LK&`^FPbHXIH(5TD3XUt_0K;TY*}& zT<^T22Hxi@0JTL{pr-OZt0mB*md*Dp^HQRw^gYwFrr)(KfZ9SUP^*#)&pT@1dA=M_ zyVwfURGw$G1bWo6`JH86O4O8oXL{B&YMTMIdMi+?lnc*0YT$Lg3{bns3e;3yXSD=+ z)Ux@UWnN0uls;#A)--BOfZ75pP>aZg=N&chIBx>f=39Z9%Hyn-K#y8Bf3wU>iJH>i zOwXD|Z6lyoX9a44TzKA518?&NK&{pa)KuPPwFG+9viX{2UP{!IzGiyXG-~SswHhl> z1R_{5p@aL4LhQ*-eU9n3){sY;K7XA@nVwY^R_#$X$glG#8{{v1 zBQzdmgZe9cBTNr#I>Dp7hTu`AN5Y^-*-*yP(5nMYPp$@!@_LO&nLaD=qbNT<@_0Do zPnMrO`>0jSTJ$IH0Dtl-JOQ6S3v+1@H1%L8NfTTM$r}8DMMbIOc&66ziQUa&+B-68|E&LX+@Gk-RO?zbP z0d#x|p|>Q!5s zT8JNo=C0m&BHr8w$E*$Xb|vP@&xrS%#Ycq2VK^Et5#5)-M`?+-;M6q|_lW%&>0x3~ zQU{y_5%9TImWjZRo(Q}o0s;{_tQ4VQ^hBtTM0gj7Af8iYRYR&8JrSxT5e|b0_*8Do z8dBxxiBKtta5ad~Zlws3(GwveiEs!+XtPp;(f=YH1QA-T6k+tgNC!ZK7Ar*<{V&q~ zp@HO8KBi;Yu?BU}f!u4Qx@J;$(*(U+7@kY?ceM^Ax?;WYa7#x=f3%||A)S+Fy7S61 zWlgFOssDqs^-Nboupp0!1$k6jkVjQ&W$|Dv+3aPum2aaM(7;hk(x7HjEYZ>(-mrEF z99;`zoU?t$%_fyN(4V~0%QVkvrVy~Ddw^GpxOi0)PbFlAQ7Sai zpS;3Ap%t1!D`*Q%F-nET+mc(n%$8#D+~pD59UR*X2e1Mvtpo8vz!3jK_pnq-|BmG4 z&|pfkHyEiD+}3dZxNTCDBR?(`w0lQ#Gi^Z_b6fy!BNPS6k4pvhbtNx@HaR8xYc5G7 z0*y-$n+8QuX$GmRSXZ*iKv@-{R8SUp?$3@BlqE4pW%V9PZt^l6k&dh_Uy`dg){H+z zZE&`BS0cV97Eb*N@92;9;IG+PnuwZTD!&I!NN)5pEwL7=3M87vszlqDrl!fly|F|% zb??G3v|914=pSg4`cb*}jkce)Zt1M0igiP9_XZe~=!3iEJkM@y8x8KJ&sprXbN57> zn{C@At^a?63DpPp2R_Yy(f2<-!Q1V5zx!wI$wiyM>VMK{cU)xOV|#*mgTk}_AN>ew zN*;h_Fdb&zv$oLGJcKka(Ja1YV&9$_OQr8g&x|xi6{j4HH#cs=_c&ENm*MSs$mPzqWeWR`gDPopM@lJiCt^p}}EtBG%`bHgxnaUm? zGr0NKl&k~4tY6E`(XNipM6^AOJ#)FuU1rT+9yMi|xTNCGHfu?hLAEJNstmHtS5jq= zZL*Ro19_(M7|gPFL-V!F<3#SCNTco%2c4vcGZqszFCAi_>7MIiS~c@ZHsZXS~S|2vsbS&$9f7#Qon-9M4NpPlOah;N?v8{T@)dp$n) zbw!^o8t0nnJnXp1{<`g7w$02=2p|sLIVx=eHGFQmDjj~p8E!P!h@J?aTjx^hQ4#FV1DX-XZ=^kKve!Ffe(J#8aZKx`q+ zy#qb1&^gC;H1Fshh<8Tgn>wRmF)1u1;pKa&q%bV_+Y#&U0pp@|Fr0t~Ri34cY$IeX zY34&{!jK1?XnQpdyAjy?qlqqXYmus@nB$IzQ_o2?>x#pqH{6N!&Bgy#%*~#;RIv{t z?~vQaELe`ljno3W`r4LfCe|~Bf?~pLgKgbW#c_<;i=5Zi9c$YapSvLzOQc3GVg~Wl z;)M^i4i$Ns)%lVlylEm*D9CCq=wn5_z zZtYLiNWu}TIo=%I7i}8=yKpnW{0lWMeljby(An6fa_>7dWP|k# z4Vpz1*tu<1?6c${m4DCPAqLvnLQOjxftOnFto%~RcOOa~CQDvnf#lipkjlSvZ}Mug zobqdnx**c$+uJL zm9htq;lN)zwPHxQf z&#D2ge*DfyR1f531k8i^{x#G+gb(8j#kA*X(pS}p{mxrr!1Fm<@J|3RGfV+93MM=c zHvCrs7<|^A8DI<<@zz-JIh*lc0br(?0%kPqcrfI@48Y){^vnQb#FEb%SV{kq=4{J9 z4#0#>0W%85JdU3H63npRlk!Z=da2?jRUcse7dl(mBsDC`fJquAamX~=A=n;+8J6?S z%&^p{#LG+n0^9@;Sg{vKdTjRO00Y6WAa4 zvw#Ucx6TA6s-1yVmDXdDZDU}XCNE-AozqMue7hb6OeUGZq*^6jUNC{(e?J45;1lUg zV4~XhS5;{}CfTm{h97%L^v3bMGfb`=9w36B-lj2<-Jg!~TS=@-6e; z<9Wd2aE~k6>Dupn*735V)_$q&et7Ji{)6#5-m=xi_+2n!@d}n$yh6?v?~5>guQfA% z7e>M26~N;C0RV${WiCvV5&_4GYS^302c4_01V!E)eJC3EMCD9 z=e3Zt#ru5#rqUEJqhRq0VDWwrfQgs`hBspJ@|M`Ve9lJicL5l|3^4pCSiL-$z0UzK zyh&h;*uA_ZhA*G9<@+oEQ(+32Q80aZuzkM+z~Iv$O?=9DBgQXpiS^6pZ2o>5fPrK2 zO!y&2!T#mJ0DcC5nPU)kjL&KGxruaIx|Hs48exc1^|9{c{jK37@|C3<<{}Jr} zE5QDL-1)3?7TEuf+J0kO_n&hHAR0deo9D*X$JDFAXoNSowJ&9s(juDQ!YayDUM9Xa zt~n-_cHaA}vp$)%Ycv%TfetNMB-q|SD~DD>%XTfuWxcG?9eio(ralXnRzay1Ij*tOLJH*JL2)L6jhKn>irFTC-5${CO+U1BW88P60|8_ zv-(v425&=a0vN%FSuI#%Rtq_s)xQQ{8qEMBjDlG$fLZ-300y7fY6ci1X0>36SuNyj zR{s)!S!xQHQ823oFspw7z~Dn!%>ZM>tQIUWtA(7+>Q?}mC8mHG1+!WJv-;-%3_f?& z3@}E_YQYk-TFBX~ei?vSWD1y3FslVHtA7T-;GYo5G`1DjWz!)*B1xw6oA!oDt#{kSlrhpj*vswVN`bPi^J|NW$Fhsk0tj2?3hAzlsZtO+D+5v81V@=+1!CI@JeQQgiB#uB7JF9a>4v zsXNk^np1a}Ej6d^7+Y#i-2t}LoVsHvsX29r)>3ookE^9-O&wIr8*H_bKTT{$ypR*G z*pm22PP}4U;wy9F65YFC2Tu7sT0l_0h&A*XgFi0w+qsa*+TyApD0SAy8Cgq+%yAhs(Z zr*NJ*x!kcSt{SzGz913G?X`}7fir;8;LmV zc{Bh=1N1lVh{cxeYw79hj)ohgTbVJR?nxm~l#HUy{ZEUVf` zDx#;WtzTwZt@uq>NVL47I#tR)+J@k0oe@jv?(RxSvbrnto4S;`R4ISoG6aX}EGQ?+ zBDXN9q0#2e3#q*S);=rk-BUz04lU6)p~^rqll&k~)mjaY-YO_$~^P=F#K{@2w* za8yoBq=wWS6J4;;SHnPmf3!Cd#zbmoHEngo=cN4pXUotAFH^s;9_KgGPGJpr!!c5I zM`Doj{u4Spv>v|1-m2ts<2$z+JvfJYfFDtAC{4o!U8UmtO~a91CifPaH?+>j>@Gpu zydIl9^HHS91#6%Y{)bOiT(R#F4XgNF&P7nc_WRS*hSmaVW9#o5G%;1s8|&R4?Tsj8x|BHsa!Qp{8KAB@|G$MY;VXi?M+r25R|4%TX zt-=1lGyY%r7qOT59`Zivo#C19zOCqiB9H5Q=YGd&`x|zwhos@v0@8VF2ki^N~19W(C9XUMomVxyr5Ad*vA7J_>3A8?}lmtZl=b1G_oze zrBSSiMs==Vs&RnEPBUm!XJpF@8r6dR96+PX92%-sx0xF2(a5&qmKqJ^f?FD<^t&+@ z(C9RSMpZ_(yr5Af*v9}G9p=zbEw# z1&vC<9t1R^=Fm{Bug%n0k4Cn&wKR$q(NHd}rC~~=5dbvW&7cv<$d(s0B7)ryXtbF_ zL$ym)rp9_Svh7esqgW9QN4q>(R)z zEftMoMKqMVQqeG_(eMHqSD8VB&&ZY+G10O8XaWKZ2zH z`ae`TbTzbC+iqmj9OoD9ZRwF$B7nCJh~|MGXG^`Om&(DVg+ z8QMN{2W2SN5530MkJ)GgvmLUerLJ+lWgm zjAS8IrWD+y2SD9=H!00m!>F5-CQqp|QMyTKxYTY^8jkcAZ#uY1_mkYDG`Wo6CZ%D` zSx~_~?b!CALGdLyp?OJY3Qn)6Hs~f*xT&PTKHPoIIoYA^hG_CizhTM+2*ia4Mzx`fPoSxeESfq#ftphnpFqu7c!%*i#msdWSO}KbrQV95IDq->kiaeY0rtF}QCQWtP(5X$l%QvR$18z+ z9vlD=2W^JBwC9!4}7S7~Mt`MQ`GX>Z%j$b?E9N z_-yub<7iN`5|-Mc7V}bh%(XJ;+j7>etPGZ{XwUQtkUk>Ns%rtcpcTl~YNgI2a^P35 z0p#!r?xvAbdDpcv=#k6jW2aZY*k+WTc6!z{a@ByG-wNbvv{L60Iq<<(0dn}9chktJ zJn>o?^vGrN$J47{M6TN0J68$F`K&;$S}S!Pkpq8y1d#Juft<=~ua!ZMTsGgmLFAMk ze0tXOJ0}2g9xITm(n_626ok!$g9l$(5&SeF1s?`8m8T80yTN6O9ei1q4$^d%SG;(tRIj0rKMYK}q5jj{h zFb9xxSb>~s<$zWOJ#yLB574V$L{7PifSxst+=YOg-3sIct<-r$4%Qcx19CPikW;NP z(8{1kF56lIdi9IQDOViOv!;=|0FYy>K#tc+ok!$gEy8Sc_0VA}kW;Nl(8{1kF59{U zdi9IQDOV@Zv!;=oh3$Voa{$}_O9Ko253|p)vwcU4?gN{Dy3GR#`TIlOBXHub&UxG3 z74K_-p#GGrTP6uk^%0%7G7n7_($rLAa^ewuuI_}YWKXV^y-^n~JzUZ|s&wSi^Qzss z^nB?ivJ@>oGTFx*DnSF9*jMVAhY?)6^p!EG0KwjHTsVTy=q;(gKA?6&(N`kviZbEs zr7ukqS>y6G$gXo58RXZwe+}~MoWKV8b*^B8{5prQL4KV}$soVZIc$($?;J4j<+3|1aEC%&Z&^;x7LsYnZCw4H!?k|EUem@Y>;2) zQZ~q6_(s4tR<+9%b+;zsSNKMlnB9~HUyD)r$V4x5VEutym+={NP1C<3kg-(9MX2-$ zoUwkm>9FQtC+gNBeiWsWv-_(ut~~mBikXWp>xF3B5q!$}*af+-`ZVgermv^^%qL^I zlR}p57bDLRI9YwLH0MR1Bl)t8h#|F#`lRj1I8b)+06LJX*&q${3|f#`z1#G1q{=q& z27^!VB2<0k98iK|H*%O-AI(=tHP8NfjO6z=EVW=?05dmZq2{K2OR9Kutq^+qKkJ!M z)Gvhyp6cKzYSy%IH(&Dq!})*b2Rj3u{)gCyeE;zN!`tX-be}Fd?Rv@il5>V*hP~I; z`=4|6AD)sx!jZXvoUKQ@CZFE+7xh;wKQGNM7@2R9InO@JAeAK8FGmND;Dd6{jcL~p z7|cuMg_d~9Q!t;(Ac%L@`YWRp<)2OMu&uR(usAcm# z%e<7RDSglMtZCFX0&26YK&?tHJnyK1=XnF5hR@$Mjhf2ytd>BJS~kD4%u9)y((g>q znnrCspf=MA)GFn|^Nt#Lo!0?srBKyA7esHr^8Y66s4WN7&bI=!3c2vSqXyCOm!j=QN}$&I2x_?&XV;$L2x=o)n;6(j z6Iho>{{I0u10c8}u)=>6djsq6P4gb`yx{!3Z9X%Q7xzDSl;@K?%Jia-;8CW}K(vf` zrAwWj7aU`Py&XKtbzY+f7yE%_$aRHetcKED`{6+$prkw*lQjJi~-H)0T>f( z1hTQg7>rD?4U4o|NsCunq1`nIjOh^23n!uJze$@Wahtm7BTk(AC1C~t6Jl<`OiGRi^?)2&yZw(MLNFv@#vj559{vM3slSKbeo z`DEkSxp%FkX~HKT0Y14?-ve`eGQKrYK3Sw?Xy)jWcsQdp{1vUj* zzR|KP{<-*(<{vj-XkOa%iKaVZFU5{Te;7RzjW;Z+-y3;r__N{0(2lyZ!A}KV1p#gA zxqUEk3zFIbHfqbJ09h20nbCY9pB#fSYsJw_VV&^|`F)Z+qTZf>(r|_JiGp5Ai~I_z zt~p6p{GK^I?AR4uY##*`^tx8Qsi4$O1%;|`3RJk)PZio~r$QT5;UuU)>QwnbD(v=Ch4$L1&`wo24l3;OQ-#{ENRNODcl)VA?N_8C zsBo8`D%5^OD!jRPZf7hov}3ZHy>wuFwMw66r_W6 zK`B`lq@9kApX(Orw9VFO8@AJuAnkM{b#7ZUaBwlHyIj%kplowCn*>z)i}?w_Q2!s? z!?c&|;JMquV3xQW4DCg?HNAb^HWB?8kJFAG9z1s&wj&*Ly#w7w5FHth(~gd2&TR#o zyx#pary3EU^&W_6Ai6RIXkVGkxi<>*)kd^}zJR%(8z<A(iza`$G~lkk_jjW*|YwnZ;@<693ktG zI^^u|yNXuS$W>~c(IK7o$3Y9du8cogIQ#sruPx*of#S~qz4i@Gw1E>(zShK*W_|=iIR6awL~@Gc4Uca!0o&e z)qvY!C8`1Ar1EFrl)V?s*Ehz=`TOLw>#n%zbb7e#G3o9F^@K{K*%?q_zn>~Nf1I2~ zC9Ohf>--p~aG#$lIDeU{qC#oke46tAM+2=ZTbklO8(+};&gLtdo@lxu_JP>?=x3r^ z8lP;8HO$t3ynbQi#_*ZY+v~m?{Aut&;HbFUu4T?W3X>Hd(fVwuGIIT;^Ot?;-1tCx zd|PpFP>&QSK#_-&efsGSwH)Sl;9SqTl$wW=FJy<7!xfB>8CLN@i0wNCr5_(I+7e=* zduG%V$x473C$T8F0r(VPa#9Wx=I$ReXdXYetAFTWTQB`#giK660GLe3V4^bWRRt5c z-}hU9iC!*C;yqz*_A!IzF>$-thnZetO4ciV7$Fmr-vmsc9Fh0;L}Apc3MO!S?>7LG zN8~VJ?(H#y<}q=*wuhNsf{FFQ9!AK-v4PLMw(=%4iPz0#i&5b!sc8y^AZqJJV$fCm|7sXV+2 zP1BM6D)36jfJ8fPBkU@OkEaV6ptYQaWjW){CrZ!Jk!A95=ncQgjjhxFw{4w!<8)x} zgAB%FvB2h?+QLc}m>Em$EFY|wJB5i!q$X0?w51%Qdzb4|*<9-3{JMR)T%ojkAr0h9 z!3)3Ae=Zjd>{fQO@BzoVu8=kMJMgdMFc5xar1tOH20_r)xCLhks__#O=SG2}?Y5D@ z{o|z(Q5^MY&L`6+(y1c&mHSD61T*!D0L_*j~5I_JC2ARoT4>+WRjS&piwj|5h8t?<|T|+D)AH`wOSf9ff#? zZd(wAPHxMpJx3d~|92cemjOH5YO}KnvS`I!VFpBr-4 z+(}kM^UCtH=eM7|>fB%~aQwluDRwopQQ6cTwZ7@b1I8OyUR{{CzWms#RhDJX_;qP| zY4d8cs83BpBE|@<}~1g{)$w#q}ak$7$GX{GX$a-=_Qj3xkUStp{7O z@#mVq*sL^dkG&`Qp=h-6+J<8NbmS}H?}vLr_t$+e_(^!|b@8LEINu7D8y~h+ZY+PF znGG(#e^=U>2dc6IR=|D#LoIt=FUHrJv)9wa4^_l}SA22c70(|0rvR9^6fiY#;T7=V zp9Ns_T6;3U2s!coc=4Xy_)h{bO;W(r!jA_>{u2O(*)&b$KN{KvJ-dA(Ae#Hkl^Z<1Mn`4!sj+9Vnl>xD^-5ZN-VhAXZg)0UlYkU3%5 z#Hd%5?+e_G{C&VguYo6l33Drw88nZH+oed%^pcIE^*SU*$i(EMfJsCKlZ}jeRlx-6 z*!~`1qF1+*z=X*i%nX{x#4U9&W_k%G)}+B0Arq6204582VahW8Iy>C*ZM5{R8`VJ!PxJ{#_OEcvW9KUe&Y5`&Y1kzd>gIuGYZgRl(zZ z34qbFvdRD>;_<4!c)Y4-kM~6Y=6We$YT)sz;PHMQfYB4L$^awc@v6R%*Q#fa_b&mM zB~rlDz~fcH<9z{uxmFGsMa1P*eDQe|&ra`O05FSXfKh7T^(x@@ehz@SMiLkizgO|a z@l`x~zMlnPu9gC(2ClCHzVBxM7`+;##8Re+IKPT7-ml`>{rz(Q2Fl_|@DMfde-&_m zp9f&BkOW4=16F)-ffdg_@Sg!NmrDUt11DGkFZfRZm_?Gnh`7OuFMhD%*%5vYfLSO7 zObt9?1zh1j0bul!c`{3xh%c=8;tVUEz2QFwU=~OLQv-Kc0e|?90GP`pfe{H)SKNx& zU=AF}DTQLKUxSe{k10N_`~Qoev|sB1@c*A}exZ2<`2UZC|NkTK{~rMV|I^_Y!>hpm z|5)%l!FyjjH2}3m=OKA+Zf}l_2E!;XnA#WFBn_hZrgp{o&S)UNCvPhgiwOKDpv-yd zQ;uE342-ndy6rZ#?w_^0&v$@jM>czjmle(6iy0eR3l=*;rHGtwJbJzz!ev)@2$!u` z{uMCkN+ zajR8d+-lXcTm3ZvX1xqBY7N|K72N8t0x){TRvBPK+-lVqw_5e=R(}P6St|uh4cuxK z-0Cj_FnUo|8DK=*YSkCFTJ`K!{{sNCMhci3xYa7S)n5W&^t!7uz=*ijsxNM}>e;RS zdjMv&6fiY#t5tBTUj|_G(yB7Rh`80NFK)H!*{%LN0A{5WFg0+iRdB1n2*BvoQ)Pe= zajR8d+-lXcTm1z9W`z_mHE^p{aI60ofYA%2$^awcR;#|a)v9N=`fmW3Wm3S@AcS0n z5c0p){r_OY(*gLu`Jee2n4f|98JM4e`5Bm>f!F2?T-<$r1MH)MYoExmmoza4u4am< zISW_91x@a!?Iy9KXReGTW{8B&fNGs=0ZX(bSGp1{$rY_cOL8S`(UM#-TeKus#uhEf z6|hB1a%Cyel3dYRv?O1y7A;yTRI3PDE#=dQ9Vx0OSz<{_yC+#@)9yHZ70G-4YnS)Ei*cBRtw$&+2F#I96NcBK-# zQa#y~O6*GYWLGM&E7g--sl={SPj;meyHY*bl}hYN^<-Bnv8#4ZcGXVoN?3MJ&+65l z?($H+`dhmH-x_#VpyB0)mG#Rb&qq!~7KA?%-V^$EXtM5`b=!dTKMH@o1V7rY^F6TD z;qyBUq*J+pbSYg#Pr8sA-dju;)65MJ;rvdxMlCaq5ujlfbf*93^m(X+!{v9vG}GHt zA-@w&nD+nE-t$lwr+ZB|8!!R&HVS#jc~peb0OQHQTyDpSzLBwPI?+vkRo?S)PttZ@ z)owW79Sa=PE+aJDktjWr=*uTY(tY_NBy&pSMoIdp;at9u80kw5XGYVwp6DR|Q(Js~TP)D4E$G$N zw~H3@pnK&Z>VNiOfKI*8;r-;ud8o0|-M&}f)7u9(+BcdjP02mweev__>a_isIIf4^FF68Mlz{!Luj|<-?&cFYudU>z5ILXJe1a{*h@B> zDQU7hQ~o!)mrbQ!{_Ws-D5|r0jiWAxg;7Jpwymh8wtqZu9*XF6?;>+OUBPc=vFuapbxr~0l@cW**0Q?jBR6R zp_Vv*5RyQoZDjy3jnLtjbrZ<|fp1?8y#dhB>-k8a!Gz_ON9?eF@BL&}D$e{c>n9vAE?C^Juum;bJD1Sg17&7b9e%(F$nv%Am2KtXoyk z*r0}501drHjl{db1mKoO%%kBJe2bY_f<~9ul`0Ns+$V!ZS6R2JpwXp&6wufsgGNVLx2mAgp@td(jeF$KVB%}bBj(X?i><{>EJ4E>T8j}f(P#iP zcFUmAUe>KDXtb-LdO%~B92!iPs`7|=G~6;&VJ4QKVNFnl5i-$;02+78prMv^s|p&b z8VUm%cgdl_q^2s5m`B4cEfr>B2^!X{R2U%>jS!%*Qw9yCtXoykP}EQzpmC=h8qCGn z@`!mf+%C;xCYGRKy)%muGSLVE8h6N`(N@;2DrmH+p@8oH$AilQtq-(}#J?B6zInRo z2aTynFnmww?z(4#UxGiq?tiq7^Fv^&<>v;gmEv}q8+m_KI1lA}b?l6-74TC^0~Y>hL-$dXt=3&ntuA@8<@V!+7I zk6npk2rZt9p6}B?pKBinJv=aY~(Yo{%57N!xR zU)eI8Hq{;`tQC@Gg|336S%+cm;*(}!1}2&`3)iM)*NhRcqJQP1=MOm~&B8QODQOl? zmV$f*6lHh_X0tV8uUCd?rL@UQjzP3UUq6(FFHDp{=a&Q5(n|vUd7AE!8oFFtdj3I3 z0g$#;krU{;te%riL)q4RAvZMMH=?tuVisPSOEt7e_y6AzcqGvB)0Uq2_U6Z%W@9f! zKOeoS@rs7S^$$g!48Je*ow{$;wFTF~L%#ihM?Il$31mI$P1H>*JKZ7^g?I7DbZt^Y z?b>7K_1f%L=d}r2BrMvaCT%f{xm^d|mR(a>3HEHUXZ!@vMFgx`1>}zVf!s#B)>T9f zcy$Gk(<``3Bge4ob{%-++&Fgp^b?!0GVOR#6S+1(uILAH8|+$F5jo)C*8_5Top)*E z7$)AX1CN{=ACI4Yf?Su}JGTyy%lm;`mtE^BA_si^T0m~x59Aou-mU|WoEvv9h@6$d z$BUZp+|7X8F+Y&&v};{O1IUf}fgHo|+jZcPbL08((@)+xE8CA3HIcgskjwdj zT!&rjDk2AQ0ILDHQ9qDlq5Av5tpemm{6MbVu5}fWgP4Jp zfLzuO|2GFt>Hh!r_}1nRH@(!fI(EMPz2NgN3pT?1)Jo*yWm!5GFK3#_ z8@1afL$Sc=C0fyT_Z5BeP=VUT4@PnV)X*no+E(;06hXv(%ide7#2cj6?f}JkLOOmz2 zLK@>G={L=wj&GexO$Onujjw%ctMpqdL?Giw%+bhrQA1glOcs>qC}lzU`85L8SSHI9 z`m;vxn_nX&PB-c1YqOS^3`7H`_MY+@#wSQk<6jYIn2zxl$mk6<*3a~wu@!bgTMJnf zY0vI_Rhi1ekCQY`DCOE ze=>g?jpyn4g2;K3AvyOc1MNu-?b9Be)C=WaowLUe2)4zr(1xtT2?^=gje|C1kvdp8 zXuPOtZF>Q=%l$xYgVA}_Q3D40UO-K+qAQIW!$7kd@N0w{|7^%2QR}j-?|4xYwLO5^ zLO)RJGCHq1YQR3<1E}dGcBN5c*k@J)9yK@a*^ot|X62soq9$s)0ks8wpw?-0UUk%f zdEN!6>GgM|QDc~ARs$Y2H{RKhMWSZqo$;b3YIg%_t$v`^VRT+~)PQxq3s7tE12u+q zW;NhZbK{&1StM#!&KWOiqP7!Ii~E6EyU}^oQ3J;LPC%{M57ZdOnbm+t&5dt1WRa*@ z`DVPRiP{~2T9Y5BsYd5jM-ABK9zZST2Wkx4%xb`+=EgM}vPjgdTr*zOL~RG47WD%) z#pt~1r~%Ww9Z+lZ12yKHfYpFU&5dU^WRa*@d1kz*iCQ z*2Adz8`Q=q=AJ#_8`L6AOKj}v0oLu5|1UxffR?-BcQwDg>1|D+*wW~!#xI0_5WFQ& ztP1@PjPf!kMj5Yq1x6X)0P&0!E2WN?1!7DM9R@~uDcAk%7Wq? zTO=sYG0KAS9HT5K&oRn^@*JZqD9^D)g7O@rEGW-2%7P6zFv|EA?2IyAl6*)kj51!5 zSiO<&G;|0U9X8#v>ufC<`@Aw_Xow%O3d*~PsX<<$|sAoENv?XwZ_RM4t%o6AaYO%V=$XLZo7$#j%3}_Lv7+s+L_5~ zVX_w=v-8RL(Uf+!)_c;1FtrihPjqFI4dJ};~|Kn#J( zuSO*<*^or*tcLCf5C0k%xAe(1!zCL7sf3&xlWfQ$m9R3&cu~{C?*$M4YS3Srk=H!P~(8+mK^v*E_jj=HnKPX%5D0d4E#W|+7INo@fe zwPjO)EDFiYXugn7jzO8V;%KI@&Ul9WK1m)?Z%;sJxI+3wK`*66eg#$6oTMv$&zv51 z?20b7kAMn#T`S*IP->@wLRBb&3Y+{?p{;f*v{4lbpaQ8=<(DCK)=q^^szM%A(5rI$ zW=I{iQ=x;ZFb*no`KdyC?Nn%|DjWk9I{j3k_AAmDsLdz3O z-5t$MuEQ!gIL0X0b1zOoQ;hbQ8J}Dm4J6kl5eoKYi;7jn+jsRUC?(5+wA117$(sc_ zZL@XShV8T@NIM-$O|FRs4lX8jmn+&Glx@ytlYmNpF+Twq>i>g#nD#O_IC&Eo%o2Bl zp}pv~rnj%#CZZqXaoSOOaB?-aBOPPMg(ZR2VxqC zu1o>iS1L2PQlPIkq80Q7%>CRrL0?pW_SJuS^2TUjsC~%Q@~K{gBR!iQB z@nVX8M~$y_-ey7_4~o=e?D9ny0z|+;B$djsuBxO!$!5<$=zTE%Z9XN zt`=H4hoI*YN%EGFZ+m$!W&VynQ)!P%jx)GsDw?l&ztp~`wY8JGVu9RlZP^?fR_6hc z^+wmQ^sdoUm-5Ed_W@67+a~V@B$|$CO|D3oRIB6Gc7Hlw$czp-JN&Mq6*Y2|T4!`f zr+pH%(CfZhiElQsMdC+2~Z(2CJ{Z-VWwCMUQV?qr$LL`ebeGntx=DH7PtAP#id%KPQgj#)>vTT00feRgLU5a z0|(NXq2WS$AfYpJ-p(#vS7T$>D2~02Q|Qse<#z$!S#5DwMX)CqRW;{ZzsE z%TyH=O8e&Hl>a{(XkFRT6#v=yg64NNU)l6T(+#l?#MVbY6W!AIWMiyhw*KSw3nMp% z&xGDy_ub%6g9idf#ocx-Gnt0TijQc0wp1Cp{?hr&zI1MUAU(dVI5?WO3}z>Jev6x;xO zKVXuQ!-To}#|)as&+Y0Tdf3)We;6SXlV1Z&`eiUt8TG1y3Eb~{5-`!rWl6jz%*{S# z&^#t?7yB^NOH9dnr4J)yV)Cni36vx9{+=j|dR4&$ZtuMhFiFZ`!ra?q2F+vQc5M$c zy#y2Mg*}XriOH`3CWmD(X=Bu@3MOzj@0Yb*lZOD4$QJD&yZOavD|{;t*QlrO1wH$D z;`HQ0P@8Gx2zwxk>$f-xpG`gh87EBpf3J4MB-ATPjif3KfLKU!v^dfaALra)axh!W z52y3{htmo2rv&|z9==!lQvw4126N*h;9T@iBnt2#!z`7DSD|S-viAV5^Z<}($8CgN z1@ZB8Ap^9Q)37XO-1$W5IXbdT9uB?XH@UHO`v11Ab8nmu?A?&T_+Tusd8f9pk_BeQ zQaj5B>*Y>iViKu|R5ooX$LQYW`cyWTdN{vsUoKZD?OsR&`BL!0FZEBpDH_}KHu zj&)rjYwUO6U&&!0{K`n}-?a^bpsjHW&JtAPU!0gc032<%jSTJ|FO7)es84e~nLd$D z6~V9EPXZ*EsW<*qo^IC`Xy5P1Pu?F59N2n*RnSo~lg}3&d!~PwqhIHfaAB zk5BFcJKJirvkJ0k#a-oT&u=?Dxz}0qHXk&1YlHUxmgAH6I&1DEE24R2dD`>iXRn&v z6AK)F@N9}*&1_UQbw{mly77SV#+6qW=B+P3wrZ7S*)x7!nqJyGAXSQWnJZLH$CQTC z%ozy=c8`2gk4zz}SZQ&c1?u=7*lhftqmE~E|9@d{QK0o;OE&&o^B0?yrtPu!L_ZXb zHeTCMte=j2CH(zxPw4)-4+cL8kG(E_v=x&lpmO8Gw#tp=?=!Q(<@fJOJM%zQcEAd_ z?;qE)lX@|})||bbCVr?Q{=4Ff1Fv}Y;6DbyJR${54P1ByeE8o7VDwshGQbEq@&0)6 zp56G50x$(BU~1vVgCqZY0E}KrPX-tfPu}%nCH^J#?8|=yfEkwprUuTu0^a=Z!U>CB zDNo|mi@7(+EWrEs(fGIZsaq7iCzOw z0u$y|Br|9p6Sqr|nCT@ON9%P+jF5@R?*JyFGMH>+)T;_6P{;N|fQeq+P687qcQ7+( z9uv3J!I^GMH>&)T;_6kRSNB0TaD$odhOKW?*K}JSJ|5ficrd zFzNDiQVDC<2LY3#GMIER>Qx03$o~5jV4_!~lfZ<@`^yZP$HXn|FB%hT!e5M#`JQ|L zFd3G?q?1vvDwsfS-`@gEhU73|vidTE<}qWi6P@}5}J`C^1jOnwtE8I-}KgHf+4 zm_X*<-yr^f;ORi?Wi3PT+Q!$zSyIKQ}R|Svva{!Ec=>se#9ZO3Gfyb+Y$NM}06OjW( z5pj7HUwmH0v(x)$08Cg07^MbYuL5rGp8_x;Nnk|$Ud0#3SMlumJ_o?mNdZ#>*H;1G z_fG&Ay&9y%Ql^MFzlty3uj1ML{bK+I%Hm1z5H;|B6>xz62u@YcNCG3`0V}?^z=~%d z_|tHz`j`|jHE@Cz@PhvkfH^G*jEEbo_~HjEo*m&&0WgnB0aF7{SOHh~SpY^anJ2T9 ziTJ{bFV3*y*&F^O0CQ3bm>Rgl3i!jH0AMC0fe{H)SKNx&U=AF}DTQLKUxSe{k175@ z_x~3`X}{J3;Qv3{{6g~z@c$nN|Nlqe|33i!|EI$*hF5|A|FPhAg7?05Y5;1BrXYE4 zZf}l_2E!;XnA#WFBn_hZns&w1b72N7C0x)_}RvBPK+-lVq zw_5e=R(}D2StJEa4cuxK-0HstVD!4HGQfzq)v7OUwd&ce{u==14N}0=z^zunt^R8O zMlY=@1B{4Ut@`3ttDfEJzXD(`lLDp&ZnX+-^-BPZUOiO?7!kKx^~J4LJ-gK}0x&I7 zz|_F4R>7_QJOHB?NRC_hy`+gba5Ym*%~`k-E@*N;Z8wP>J#%F& zF+(JD22|^03s|Bhxzd$rNv>!mT9PYii_|~P$r4La+C9kmhbHU3S+@;X|D*8dOYo!ZnpzKA9X`L)KsuEhNSD$@^rQ=^ z;l0IlG0ofn5zg;~Yt%B+7y%k)L1+3uPftN594@~TrkUQJ3i+LI!nFUN^iDxtobEN< zY`_H6+bHBA=TQ+#1B@pJbGaQS`bNgG=|ngERe8_HJxSaBckPC$n`424+T|okO+Gc8 z9w=thMn<6gM81$7NsdGNnUQod57>>8Yhr<3Z9%WDzFoAK2i+?VQU9|K19a+z z4)0%&OhJvE?)JU?NDclr-6$DgPVY%cfE<-#<77 zMRhi>an!}IFluPnwiUJ1_V)&+pomWQPV$<#% z5&xInQ&1{rL;D6Nb1Y;a(N}kId^|l`Na+7j%B*G|4tb8Y_sYSk<Fty}c=DrE}m z))S674Y!g2ZTnl=nW<&45=T4F?iA15Dq1)Py}@s{CX~iV!Kc!)eq$u)`VF^UzoD(4 zS{e%+TcRC`Hd+&HV_MN?pa=dRmaM!b?-53ntjk^$RYPCDxOD0UKyAU6_fAM06^!Oa zPo~FnN$Zh8>34}fsCwQvMC%%I#|7nIc-ksMw(nKm_R0F;fu~*ayd%Gki9TdDRKybJ z9eE>>;v5sD%scYv^T1-A!ef3mjhqM_{w&Kl_6?w@G< ze<9HNP|JAyh32m}-_~@0?3w8E(dCV|G`yq!srtsqb>Wkti*>Km)ddd+3Vs{}XoFLF zHJ|v9y%GS%pCun6CPKHu5deMAr3?VRH_NsWBV=qF!*MM!wGEO$q-|vYFpbdRmvs}# z0D*5`4L1WCdOaTrG?=j5@`xSw4}PFwjmX6)YNF8uXxt`)#>TR4RY7B;8jb-PTjkJT zVsXnO=FxDA!^KQ2u~2ImE=I^iBMNA|Q3j0-W!sEgB7Lz%53|d^aM1 z#w{{vbe46i3L2eiI1FfPl0$W^@ z4mDf{Xl#%}gNd&#kC;cpEw&aju>=ikXe~y_L?Z}jbjhI6Ue>KDXtb;00HDz+hX#|S zsyt#I4Yv$cn29B5SQAuXgbXx7{{v`r$e^K?b*l;*kelkKfJVC<8cb@c@`!mf+|p8E zCYGRK%}Rw4GST>dfQBlAhEmq8Dri7Ls#gIGMGg(-;%s@uJQ{A7W-${>(6HW_#R!>b z{5PP{CWA&>S+}a70oP^!OZWfd!R3M02U^tgrNDDcDV?Xfs3crp5Vk<+X2HClA)o@iiV2bZkac3(}-kN)*!%RnSI>q6N9wX0#+niJ~RB*k-gO7u$@M@Gg`D1+iZ<9#K@9ZK?}uzks1wh(XMNXjWvU*N74P{&N zh1}41--yntidlGRF4b_0?*G3b@JOKLr!774?ahxj&Bk7eem;6t;}s2u>mQ0d8Gc{r zJ9XcxYYVP}hkW}1kGfyq63BYgo2Z*scDh9-3h&~P>Dr`**J_VV>9yIf&TA94NLaK- zP1<4@bGr__ExV?&671Px&-e+TiwIct&4AnyKaksK*SdvkP@ zu`=y=Q4_hF0J+0{Ah*G;brq2V4t_Ntr`LIxMvh_P?K<$tx$*J%=_kl_ z$-Q%{0J(?!K(5QKbrq2VzJ4Vj_n;rhF|55^2Oc>$?p_c%D}#?0HQ%`#0l5eKK(5oS zbrq2VZhr+JchC>y7>3`j1CN{=&ySye^3GY=e!QrO+;TwfO@1KPVb{8f$Uz*yGC=Nt zAILG$0CpXCnb7#F#|UMa=m^a$3za;b>NY6 ziyy#GKS9nKMSvGIkxKw_`~5&pwQF5Pe~Icg{?ma}uSnh+ zGMFal(bO$xubk5B>R#S?cEqb^FG}I!;iB)TmB_`*vUDz9&NPvY+U-->Sm5*$t!TUZ ziavR$K<(lOBRK(Tc*B%BrPt_PvgIvtmW09&!la@k$XX_;ruU4kuoK!^$f8JlcIT_gR33huq;W!7 zw`+%{^eXEMHhYiy6eX_l<78_-iOn6ovK&^m#wn<*KC!}c&}StsTSN_N70TJ*R2n9G zaZx+v)oq{~JwXrBmEFecQR-Xb4F*oIP3xK(fJq1)(?U}5(MlT~x%=x;iT4)PS`DuU z|33vIm*$q3`{;Hhc>mwEW)%9RAmA~DqtK$JbGMH2|4{$$x|ZSiaPu=w&%|Dhz8dXm z>}j}Ie=+iM_~r2O(DJ&`;OJ|o_Md(vp{dgWKrT39kIBdT{$&0(8qd@71(EY6Lvrp@ z2HKMv-laV{r5DP*I%kg`5NwNKp$%Dw6B5#~8wYL3B6YBG(0EbP+V0lWsmJ_4ZG+Ky z)lmZm`Yu3Cuc9lB8pA-d8t`j`8~<#`B2nwItnYYH6SbXy+M|A;)@5{Fb<}`;z7tT> zOYBOc#<0(<20UtR+_NEzM9s=Q<3&x>?f}$I`hi-f(RtNT1LnC0P}A%0N~6Xw&#VSK zYHqxZk$hyd6+G?gwfN>&$Avqvpmr8?s2$tei7m z)I_ZtP0kwi3sHsNh zRYwij=Gy?ZydS7BY%{9?kD43TY{()}vvSRNQ4_VTfZDhps3}J0RYwh&<~IUr$NWHz zIVWH>;8AnqnGIPaYF3^ZFKVK;1yCFF1GP4z^QxoPriO2Y8`L=%HGhNJ7{%PPCwzli zq-lwbJw3qMO!@yJ)BtF?D}Gn=+ne6j6pAg4o@)F;_y@sT0>!G(|Jv=-%}$ImUiAu$ zGQI)g87o#w9WM*Sm>Pc5lser6w!t&X_=fyol<~b$m2Hf&pg6}C3CeSfvYd~2e7 zvPjF)wsN1=I9>0+CyNXs2bC}ev$^B8o4Dvm);&GcCiZD(rXw)fi;vm)Wc+AKJ6r2L zX+xOW2=6DlvdM<<+(uLCSppGYllN*J(_xr`(1@K)79NNpQ2EuU#3dV&Xr0yYz1pMG zAsDyx$u+|z8w06?oEwvD$Rd@nGRb&R)5G5bjBFj~FU`nqGCHq1Y;dFd9zZSV2WrfX zZdL>SeRI3dZO9@~W3r*6*BcYH-MasORbW$~`Q#Xs zSu2ia3hRt#$nTTn5%u;2l!hy$PZac0TI5$yb~ZFpSW}FyS%bOh ztFb+1^F?cqyYA~n^=OLG9@E+BtD=DiS3ii*v47m!;kJFK{tO}7-9T>oO00r|V~m15 z_u>>Z#b}SI@#!m~f#ljGLczXlQL(Cc`>tLErDR!V+c<|hC{{eN%|(_W4YPA>$5S>kRmv=`mh^!9bzMD$}k zPCH5tPQL-$k&e0Efo>y+j*Q1?M~5@h3&19?cYn>PMg(ZR2VxqCu1o>i*P+bxWdeP* z5v`yvVD9I}3HqV}w6BLwPq#(`L+wMZmQVE>%_a4vC((7S3yH6}2dxOP&HP*7>HzrCOs-f);Ci)8bOCQ4^rW&Aw@Isn)0ypv4;B zw767j)N#<_Cf~HURBO~Dpv7w6w767jR1r=pSH%Jo2PUs~-T1k#mIux5n>l zy0WDy{aOXtQ1(&O8TgM)gcKmm$8ob1z2f2id!w*%*T)}_=u zoO~fWv>dKrgv_vtUxC=Z-B9}R;i4@e2D)cPJ&~*gm~j$|f*XLp44CYa!-To}#|)as z&+Y0Tdf3)We;6SXlP3U^yJav@8TG1y3Eb~{FJPjV%aV9cn45jfpm|K(F7{!jmza|E zN*_kZ#N<7I36vx9{+=j|dR4&$ZtuMtFu79>6XxC?GiV+Yw`+Tt=_Qz0FYIB2OiX?W zFu6kplQu@Zs$c?l^WLTHn(hHiB3rbB?B*Avt?;cpT%(@87xe7s7f(;`fZ9weN7w^V zT))Lp_-yh4$T(rz|1W4)Ohdh*)JUq*0EmSoM~fr<@Nv!!CI_>{{BSzIe>j~We@f6l z>EU~&KP4dGZ!kAL0?tMMM4|u>GR#tWcomwaBYP+CO5H%B9k&s76~xEWg$&SIPQ$XC zapx1I=jg~Xc{ucj-{i*D>Hpid&b@Iuuy<(D>20yV=AGKYN*0(IOYJNltd~25iAkg; zQrWbn9HV=e>r>fW>f!vleYsqrw0j{9yfh+;qdv{~WcoxpRRq6sKM9avrr!8d zdAeO&pnYG=Pj8I|4s1QZD(EPg$>)oXJ<~tT(Jy_7_WHI{({BXZ>$ce*Fv_wjyEj35 ze@k(C3sC%9Z4|$=C|YSZaoX?Wr>Ad)c!q9U5QR=|%c?y`8?^t~>Xbn_^cp8&uU=T4h=Gj9-_gmo^Vbm114y3RTlFrQtMlMuLIeBcIeGQ^+b- zT3pWob-V#K8~^91<2UR6|H9y+K`1|3W z(EW8E41N+GdtLl!E2jIPa^u6c%8ljkGqb_v_wPzO^FUR0zzVqUzpG`Z^vc$skcr6$0FwhUm~3R!s|qGi$M&}X6TP~f1SU-GU}n%fCT^*NG1E&hu_g`1 z2$`7tCScMlgUJR)y{cdW`GJ1}FwyJQNnpZc24)7$W8#(=7&E;DlP*sum9TdGI$(02 z3?^NSdR4&$vj4sxFwra0Nnpa{{bdHtW8#+f7mbNE;V(wWd{2H2Fxe}ENhhOTRWO0v zzE1)s_sU_yWc6hR&12%0)E6_o$Y-$(raz|(=& z%UXuw$D3bh`f^ieY)ACnjZZa(8Wz?cjhqa>82VahW8Iy>C*ZM5{R8`VJ!PxJ{#_OE zcvW9KUe&Y5`)9C!&&%xJ)f#xbDtNqq3c%=DS!I9`@px5VJYLna$NL-rb4&`D8hE@a zc)WiC!03rrWq=X!cvWA>Yt^&I`^NxGP70VBc)Tijynh71jLHF{h`79pFFvp0+3Ec> z05c*3j8X%yR{^*84*{60Brqa=ui}g2t9bT&KLx-%ECoyrTweuz-)8|By&9y%Ql^MF zzlty3uj1ML{UiVbW$`3NKLKUVQ=f9|8fIXm&DL$Vso|HkyJzabvLlQfSmVZSI!AR8mMJ+p{XQ`D~z}rRK>ULk;>UPg=^%t}i zGa;D;yuAi)bvwA#e+$6qX=o*YQAONp)fcx~_3T#v4FD6A0Y72N6<02sZrsthn9Znf%*TdjI_ ztN#LknUDge25z+qZuRE?7`=L`3@{>Ywd#vot$KE=KMTMdmjb2+ZnX+-^=AMWy+EoA zFd}ZX>Wf>gdUmV-9DpfG0aJqzauq_zpV$5WV8hb^_`ms|`5Bm>f%zGjpMm)qn4f{y z<_uiiJrjj}RB-JRIrfq!=D^iVF*Rr5O1PlO{j}XAcJ$1ZvBV6K&>2vzlPzG0mgGuT zq9wVam1s$>q%B&KD`tz9dCHDx;}ZbE0x%l>dCHDVppmsyHbf=sh;dgC3dBHvMZI?mFmf^RAN`EC%aOK zU8$bzN+ovH?#Zs&iCqcH?&(>*+S6Sg%2$6$_y1c1?+P@$+_18KdF1)XiO7QRN5XqT z-wsXIeY0*Gu>MEk&zImw+ck3)Y<2kjP6O#wZXjJs7txb0q=xqv)5SD%14KB#6RuIq zOk)ISm<65bADy0oN;q78CrmTFJr(jh;e=`b|JpkPb#c1abh7~yP;aA4 z9L(i*oah@F%cc|E^jGCQANM3}_XpYyGndB#2er#dlA3&KI6Y9zrj3k1`H6fXJ(3)U z_A?{tWFD{^9mwC887XE-X2%45AW?b%N`53t&m{WtiIH?)z6i;j61h>5K596ZFC<3# zQp1_iG_EH)$bZom&n$`sdbI_;y83p}Vjgs_JVgD^J`B*Q7dpIGj?6%fo$mI%`kvlC zxY54RTxm+~DgQH(&i0Mv(*vvQkB?HgR=(dVPf8*gcNNBvXvjgjlZCqozOUa6}K9u5@zI0(=NXY^`5@gaL90E|CN#?M6P zRyYEn54w~A!1rd^He!T~ZDXWCOU$f+BoJv^830Tpbogc6L^43&+gBs?fQDYrM*sA#sHmZ>@ps`904JH=1JYpUVw>VtP#1ad& zhT&p_Of*7(#!4A9Hk5U%3K|>KNFAV|*Qk+rH<$q2@`!mf+=6d06HCzO^14z50gV+h zXmpizs|p%jY9s(?ESE!riMlP1m`B4c;uejDHQ*K_WV{>U{{b|X$)M3$)~zaNK&=k4omq^KiN?PH8duAp(N@;2 zDrmrU*&plve>}K6(E32jNc?;8>zk*We$beT1jF})?yh?__$Bz$>;6aUnArpt8$6~3 zZRPyZqkSVrBm%H?@E^%yZY+6>i$|cAWPd~iRVrJGdayU-VESt0a_#WUMi}>kE!r^~ z1B4#8a2y4Gn7us~X9X`tUoUcc6RC}Wa@pkJnGW)jT(G?) zn2ybeYC)RRN{M3np$gh4QM4cz+l-dvC{eT|7u$@MNwaXW6y%tenOO(3*_yG}E5o!>+T%8cxf)x zNTcrmzaj8Qpyj76J@M_$k2lT6UW$G`dR5~U4TtL=iaZ&9U+6n^->Pd1u7ihs`vH&o zZhcE2>rro_Zd%#t7MUo#i$|twlNwp6JvO7)X1_YGP1qt~(H=Evi($;|I`Fpan#xMB zXNx`KCx9*@VAVGQay$J%Zlhi6Dk2BG`U*f!ui!3?9K){Lb>NY6w6XoEy)NpMLVrS=oNPsEOQlfZXkVAlG5nx{AnksF5Xr+--g!$3z3zb>NY6iwVF_ zKS9nK8Gsixk-HX<+v*2$?RKrJh+MlGSq#X%(GTR9$N{?!JaTUF1NiAD$XTNZ@S-Mi z*8p-`{6J2%Yh6X;R5fxnAa|=D$T3j{b{%-+++q#z(@&7IMjYTpP2{cuAv5U7`E`&4E+8|Gz!HwfVzMFEy=> zov(i{`25R)jqr!+{Q!9j6?ZxEb|8}<>w`=EC92!_PX|)HB6(}bV49#uQypiooYCv* zUfy|j#H(j7O5x(+qVK4c$i>UDbS_@bG!a$1eWo`SIK4zG+U~xhPaY~zyZFIKPJkLw zrqmg|M(>g>Z;7)c6n+pU6(vFT;(Mbi+bCZ_agH<+l;`MQL3xe@7L@0xU_p6~5Ehi@ zC?!F8jvN-0=P6>r1{{cDd<%BE7%xfI4hw0Fm!#h`gF0@TO3mB{Z*6?-TU({yS|I`% zKVptX#)}%tvShNLJVz-D%FnM6u*Nc3rqG`?g5UfaA#u7%H(%=^Pq zN?hZ|$<}-ln>%`CIdZesI0Kc{Csue4`mDrdi>N`ZV$Ip$%snvKi;LPRuWkd~=m~m| zuIx5mk5bB_IwIM%H+hBBFb<}`?-VUhgRdl6M zV;E>w1AdKg`J4?u+OXpJZf&- zvmuK_&B{IFMNQOh2h>u2pw?-0UUk%fdAc+-vX#TQ15DHK~8J=OSy@DGBw1d3Im|AA31Ix)(4)hjT{_y&k)tXL^^yetr7 zYGfZU$_20uo>9g(9?yqde}wDC3b3WR!&(rdzK)+OnBrz$ovv zG0OO+$f9UGUU@%U=97(Q=iarFrU{>X5Aexj`W~3$lku&I^2s7COWVqB;FEI>e6q+O za!?6lFq=DWyNQdAWZlz4ZDJSX5E_NaUVO~XC*wy`+Syv~NgKk{MtDEbl}$E;=Qf&B z&k~3Ln|wFKl8?X?ghuRavhY9*fy%E&B`(>JMC+_Z?g9@#3*(kPxn{UzV<44~b7PVX zS)>wHCK)ekdiXoR!+#j`mu6%)8J$-hHn`D!C!luJ57d|&-K+-u`{s6^+mJ<~#$-cB zuQw)Ycj*5ARe?=`mT$D|ihnMCr1{6q7n+wgeWK}(*h{e^(H}<7MB@#M>i0(88vbm! zF|?!ZZ17WoS3yAAIva`Q#XsSu2ia3hRt#$nTTn5%u;2 zl!hy$PZac0TI5$yb<0=9pMhjILQna})U^cyRTD2p#*!tsQRLhw9G|qTL$ZvL$9SA}bZ~Iy6t*KBbG-xIMi3nt zkJFCcl$kjRHhI1KYfd#HK8_k-8}e7G)j zYuzKk=K`-(B^IEDjcUEK*Mk`>8`74!T4?DUf}Tqx$y-Lg?d83c`8)bdr9CP+&fuD< zXujh8Qu}^XTRVGQERfr+Et_M*>O3H_-sl>Z-ZgsaQr@`wKHyW@w%H|sMAI>?$rTBc zYIWS&?oa0nnb9FASqDHP#>x>TRw4Vem^tv+sXyNShyN*Xri<0L*0a`5fO$%qY zzlvIvoc|M`gi@OSMKF11%Q%rp2XNqsBms zH~6N-rCOtMa8kJ-7MM6NdA;k#&vmstkj@MZ7t#X>otg7?cIn!_v1@WUxJ1djom!$A za67U@HQ;t$iE6;@uoBgPaZ))7r|g%3`TE8>olXyzJtp0~pq@~PG#ddG zTK!bP`Qzj?Drpr;TjwmO(Bh{G&R?dgs8HHBKTP@mqk-0yElu&CjW1|^XY-X!Pc+>S z`#@}c^fS>djZZel8fNQ1UcWGMWB5$y?RDP`{xoX8BkDDrT!Pe1*kmc!f*oa|+MaW8!wP4>P^Ql&n|!FhV9K?*vSs9Fh0;L}Apc z3MO!S?;U`NB8LfcZ;u%?kBQs0JH$DQ+MgK&i z01qxcW8KAYChGjY9&L>LG(UE2HaOe%c$&Ibk z|F>{399j1 zCT3RyN84>9gZsxzBceF!)0|JHPoz^t@GJL|010O5jX#yA+qDJS_iTQ4RWxv5>j73l zN6Ab+Uv%u5{$Y-O=|i;FnNzbX!S=dswg-%|tjg|9(B7wtvo`|8ztu+ZJBy-~b`z)l zPM)4!0r3powjc_f+?G{)jy7oj=Z?=V2Rqwpv$G1aXvJOSY0qyyJ-f_V^EMwecWZ<8 zuN|LV>a4kwtcd276p@RnmHrE!0wSx>X9jA6)P>Sj{$X@fX&ALIqLYd?*A_g zE()|BY{|x-YyM)h(zHGHp6G|7(Z*{ViuKcxuY|uJ?g`yr_rc&N;j!1nkG5j=E~wo2 zu&r`q`TNXlaQXea(#||kl^w7G?)z_R*;&09Uu(`@PZK{>5&vEB#er8md+;9wVD6Lx zrUov&0zUjx0E}L1PX-tvC*B_~-m@G30RW~)3Yc2>@!-h+767A{(vty3#FKZuSc!j0 zJ^S*%3BYWZ0;UGeyaL|*Z@>wQUMWxF)Qh<{$t=M93hj1n5)F&>!X!qBY?)TW71v*f z6P9f zjf{F#!365qJ_(rU)$JrOVR8pEgXS@DOC5}vUV@1=X)s2}#N<~2lQ+s>vVl>rDwsfi z;P(M0dfhq+Oqk5T%%FKp+!6z0rk7yS<>{mn)~;UxOm3CIq>E9nDwsg_-(Lny^on#6 zm@s*NnL+cIxTXC?V`5GCixD#4lP3U^TVyclWYnt)CXn0ry@1IkIZT+WzRaL`Ox%+C zVy2h8C)RYn7$Fmr_W&jvWiaVr)T;_6kh%BW#QzUG9caC*Whj2U`Gux0H+9B#MBm-` zRAZ=NVg1p_$?%JzuZ1?&-5GoW9=p^(uz%N6wo2^ZRS}O@^~K{=J$t;LhW-0PGW&P6 z1|F{p9`7FlFnU&28DK;_Uey6RJC^hhU6>xh$4#4zE z0wd!0D!w?rif7OFV*t#4DPU^g`YPc2{yqSsSA&#T$`ldjSMkOBRXn@D9|d5bES>}p zQ3L;10SEZ^0GPd!z=(LjiZ3p(;@JoO2mo`h6fiY#f)((Be;0t+BMFR%8?5-^2P>W( z;SU2a_ecR#15a21SNItKMlYEsvy_SW!iq1>u;SSp{v7~jmlQBHaEBG}hd%_s+${-= zNSM0fR>TH#;6P3(6l?t&jFfpy@wD#$FM`s3tp~vWf42FB<`v-oKMwx?kHG(b0Q~ULk;>UPg=^^358XJr=f_8Pd=?ci2_9)Qu) z&`JQKin!IPFK)H!*{%Ld0OqI+Flr6lY8Bk-7XTQ&Vyg@=B5t+ni(9RFcB}safEkto zrUq`c3U2l102sX}s|+wAZnf%*TdjI_t3M0C3`zl01GicQxB4>xj9zzD1{e{yTJ^=P zRz17be-6M5NC8s=w^{|a`gs6GFRdyAjEGyU`r=lrp55v{17P~4fT@97t%6(qrvQvz zJyix65w}|P#jRF7yVcJDFh`_-sexOqf?NG30E}KBRR$Olw_5eZtyVp|)qf1Y9F_v6 z1|j4sgpmJ{?*9iHo({nO&Hv2L!2Ar%&%pc)%+J9547@gH;NtGtao9%%*FKSBFKJ>9 zT+I|ya~7_I3!2q9wUvwrEMN zj4fJ{D`1P3`JBUlP9}UiCw9l z>`EnerFya}mDrW)$*xplSE?twQi)xup6p5`cBOidCHDVpr{+?5dsEm9Xrd zp4F>8-Q}Tt_226Le{0}ffrghGR@N_%JRdm`SrGn6cu(ltp~<>$)@=jU|0w+V68vbp zE(Bq#!{>JzNT+fG=~B9go^&BKytkMxrkNWc!ug$Wjap_JBS6C}=uH31=?hQ^hs*DT zX{NWQLVhQlFzx@9-dU)N)4isf4VZv>8-+aNJSswIfbryDF1O=E-^f@to#>{&D)0HY zCuzIi(QcT1EEYJZT~3nJFt9X?HkRNrsSUTKNIO}-&j69 zu*&}USSDpWzFui-Yb(V|d?Pgr)pd4s-sjZCNG3II2<^7~8`nvCOOn?D5BH7le{J_b9=8j z&Gb#tvz`@(A&po!Nz-L}3oM{e#J{$C7E0x8Xy4#uj)e>)`syx@kEcfq3H?7xnbqvW zAk!WDc)-8H_l`@5O>j}r4hFeL1w*4jT%xn=>;%MjDo#L5WMGNPkH~8(= zgwhx(_*7cfZ;S+8zv0&F-)ZY-3$ehlCEB5AW3aBgC}LXCXP^iEAC|1VCGQbNl&s5M z6jdW%yts5W52!8J^4`&H;?MBE#4rdK?Z}$Zn|6d5SKGZTEf1&y7&9^n(AA2VHd~|u^Ee-Fef2zJQ za$Wdj=wjU~b#=kRfr1|g0ovdNy_!#a$X*Emr#Vx2m8438{Vn zXoTd@U@p#=k4omq^KiN^N29Ie$TRFemCBZ)9_$S{n7&%ysvW*?1B`pY7VVgg0YZ;kIF158%-$Z0vw|0+ zuNOJJs&CPv7ZTCH#L5Xar~%(v#n1|oCg@<|+L;U2!*ugBDQ49WrAaYu(vOLiCdEjx zG$}^HSn=wcwDlLRbD~KxtyG{%F`}LoRqJDuhc7H4FUbYlOM>axjHni*Nv)J9rXQ-H zjS@u*a5NsbalOLDQzXh|-%87;}hHlroE*k-gOM~R{(`PgQ(XeqYY8fS=+C9#4Q ziUA`--fauTfRUjeyAs7vADxO`xK{ssu6=xn{S0B+GT$(30D@!Sf#_^-_Y*L~F;rvk zC!i&{`w3{t`FB{4Q!Hb_Oi7F-7p`St@Ib%(wskoBlHQ8%sZ zbc;+B-o+!+wMnhNL3`|iUYq^uyf$HrghhMQq%DRqx9h;$vTG_U!JaMljGq9yh=5fm z06EnUD-s@wQa2U5Kvd27gEnxIEh>&{+zL9eTOdFR;?ub#aq zg^P!azN1zm7ca}wxp+C#L~hn@zwpLb;Peu$XuJE0K6$7>?cxU`IRR?@nkn^yUZZ!( zmbb)N5(+;ElZuicd-1(dm2H%-pg2bw3CeTyub@0f0t?D>RIs2tM+ghbbCi;xJVy=- z%JUSlU;_?BF}?*mU5u9`Ylnq2#!J#~nn4}kG?lt=E4;PwwQp^eertsYWc-LZ8W}HY zD9e(`g7O@tEGR#}M!*`&WSK&L)(C#{YlOty_G}3wo9H1)IG`eTov-_;Io| zpTy>lURkccQER*amDML!cn@|e?2Pk-ojd|^~=Ej?}U*{b4$#9bh{F~|LCnx!{OBCLiznllj|dJWtOTM9!NG$+=G%XisYWZQ7$3^g_8;=j`zV zf^9J@v?1$oLP9!rP;1sC8M^cf6>H+7>|V9zRg)GCHq1YQR3<3aIHNcBN5c*k@J)9yK@a*^ot| zX62soq9$sa0kvIzpw?-0UUk%fdAXYd1QtI%>c; zcL8d5_<c+cK~WT{6LLin^_Hb z)ZDmcLl%jem21X}ny9q{YTNxlO))yJI%>c)tAJX!AE+_s1gr);YHmEUA&W%K$}{6d zP1F=XZJQscwHcjP9kn*Kz71|r-wvbZZ%`Yfn0xkwZ%~UgEwQnu2UzPV|6ha}04;aL z?`nQ~)7zRtv8B;djb8}=Ab3llSQYvo80EuGj51#J3XC$o0pb}eR!SW&3&faOzY7@U zLtq;`ql|CJ4@Mc^8&%oHC<}^nY>}Wm$0!TRbBwZ}JjW;t%5#jepghMG3CeSfvYP zB8#H&c;)?YnNK#JoqN|xnkIbmoxmp_)c3#~pNwx!lus6ES=v_a06zIm4t%o6AaYO% zV=$XLZo7$#j%3}_Lv5l5atIxO$zFWS&L`tXQ`*^D?@1fN)JAwe(Una$gy%MzQqK~I z0Gqr6V#)7^DF}_&*<|5?7y^}FjY?dyA&J&mt=|qFelLt$`sAA7l8u2>Le7mzHe``X zSeazJsOjN%gNMH#^p|F2HyNE*9X7bpy$w*i&kxj?8{Mo1{QKs1pWBc{qQ+!HN3S;~ zYPakD|5braftGKy?23Obex&)w%@>-NHhrS$j@V1FBhep5&qU)5i|Y4A-WvXFxG}V& z?riW=fmcC5+j`*s@%RAQb+An=%6Yb0Tqt3D)jhN4%#!utlD2yfNtP8^T4`kbEZOfP^HDNgNU& zd>He+uA1)Y>gwrz)zvdHc=rdt$h-aO)vN#O>U#BmRS!%U@zaFL&q(*bJ%4dH5*Xb1 zNEdtP!1!tvKFv;_n;!VOMd7(zCfS$ICDS9>Xz$=)CNbEXqjmD6KVB|d)=nLA`u;&} zJ?UEIyLHmhg0QhKV^z3)KT4k=MAf}5eKCnO zaA1@%u=^gIfu)5bjUf!|P30Ai0fnTvx0N?UD8Td_(D zf>dcw{Ng}3aCkAPyIj)mplow06$4cI^4T%KQ2!s?!&J+`fr|-nm?iEGL$&C>rnj&A zCPI(#I8}6D;9@^kk&d~pK=%=ZBI9wYXn*oz9DMS6_t%_SM1a;c5Z6GcG6krveaVY` z0=im>RiF!)`?+xfT~vVTy64=*-f&>BZP3;8sa+%Km_8N7p|*B1m%Te3E&MAwkVy~g zv)LRj!pc+qd$lVrJ`fJ{rh8Ex$i+xk68$?0VQX|Gor@NJcXJfHmb?|2e4KtqjjwN_ zf6l&{P1BCu4Trm9uqNRTcN=Za9c+sp?#8!FzS`cIXt~|CJv9ISl|aju<}*znYj`g5 z(})^Qg+5UKjrwcr4noxbd`(sLtyK>OKO1Pi z$XiCf?Zvf}`8)bdsWmFu&fuCU>AvE6seeDHtsSqA1k!u7WpgZ8ojXM48{NXfyGD0i ziVIiYJDk_HkJkYb4JWk*S0qfT)p2QiFpZglmeGpBzGND#2>52jrJAEgwCxwq`)0+Znxlrniidr(;!@2~DX`+4 zZ&qBYIqC#h@sMv;T&g+hI9PGkH!CjH9F>Hf$}^F`*r7+RcU}0o&X$J~$-$vqqCcuL zbKdqYUE4QSO)eXkD0#PCOH>1HTb8H>-1aL`4Y+Mqq8c!EDu-Z~eGJ^!H^#~N`{cCh zF1hG*dbk)d>EQ*9gbJkDAeeC4PZOL!PENg&HleU|9smVB^F(eX!1R-N6r2G3dBEg4IZT+de@v%& z{M?TIp}TE8^oJ2LF?k#?St5gp%9vLcOyGRqEMTIS%aV9cn3H`>r+G}=4)$S&m$;Jk zNFPSX#AF6AfpSFN-xGx~uPm6r>Ah*d$Pm+AoVkl09mxDP(g>4DflDxV!nWDgD{qU29e`X@bo zukfcRB>WAeGs6&E^o>Px@F2r16}wlWY1*@k+7;th0Eu?WM%a}Q&m?k5ptYR3W!dA- zMhnl;o+Y!e=?%Y0kFL}Iw{@L+iAoDsHS7mBPS8<74qu z!cvaWUFG_CDjh$OUAI4-&J|WKq=9T9dEw{!#utVIdz3vae891;%cYF<4*V-N1cYA^ zse`+>LlU$#Z^2oDTKwqP_~pRScG<|_zD%J<6i0oSv$4eKL_801IB;m|A=W@g$z(R0chpS(Fo$0H5Y_tjv*VY6?{(RH4;W=xlii!3+O_<6 z3sC%9Z4|$=DB5T@ajN%c&W$%iK0}u+i9#p0WzC+W4XXd)Q{zqGXIpK4RzenSxT`$X zeD2(MqqF6$K3MM72GxJ|)OdrlkXfuSxt<2< zI1G!8|8>;yDLwvQ7+e%+IozCT`fTG@8kL3}k@tl^91e%xQlGDTXYJQ(epu68eQ(u= zf}euN-V{IDit#q6+<3xPxv}_tW)`^o`dw*b9w^HSSb^~UH?`EbUW~6LZLg<^@2W`r zuJ{tbE1o0x-vD3~DPSrP!YdHNe*l2dYwgJZBNW8@6UBQD<9{81SuX`lCE|Dpv2eokcr8YfXQkZOg1v+l?4;1WBUYPqF1+*z=XLwnCUc+iQCn|nBgUuST7C62$`6? z4=`CNgUJTQys}^d_XEEdFwyJQNnpa<49s+z$HeVoV9f9mOgcU7RKnc#%YexW8B97E z^U8t=-2VF>z(lV|CxHob?=RD79uv1~f65R9(NY?s)B)nlDvv#RiR*ChyDiZOk zzC^sL=ZNKAmUXa;{5~wquj{`7WQovLo;#DEy z{R03-zj##!7?Frq^@Y4vJx9DB17NmE0aJm9SA~f8_W_vOHd#g`CR@f-s`1Hf#O0;U2%umVx= z?*K3xC4mtMgB4%mV8wGF{2>5lgA_0oh=dggg`Wms^pbfpQ<+FCtoRZPE1sj_4+1b9 zQovLo99AG6{%ruJT@o0PGa@KwrCtK&rR=3v)N!6;{~twMK(#3 zX#NVaC=Z7N*}Yj?nOH>PKLKUVU7up^8m42U&6e%AsdX=Cca6UdJiB(YmwZ{#8+N%|ba{y+a6fhMCt5pcA{|tc9i?Yf9BNA4tzJ%4P=dk*-0L)$~U@8z+ zs}NTIDFCC_U6lbwB&=3_39D7lVfCK?FngqcsX$n*LRkIB0E}K*RR$Q5uv+yctX4gT z)t>=i?vetg0%5fZVf7yYFnaY=8DK=hYSovpTJ;=Oe;R<gNC$y+EoA zFd|{K>PuLydJd~U1;E@c1xy7}$W=%o|D+!O2kV~+!2iwv%#XnQ2+WVb{0PjC!2Afj zF-Kr}&-f8oM+MhDnPv}ZVm4gOE2icwTnPs>xu4dX#I~NfGM1Pw5;_B_b+QF4(UM&0 zO0*Z=Xl3W>Ev?N!+7A?t@r9?||MQhQLe7Rb*XrWN8BIvc0Pa}S$ zsGekrCn;^7WQi{+?VeVH;Qa$;VO6Mm}ex(w>Qa$;VO8iRohXU|;5~u*m+Dv6Ew6pP_H^xnnvd4( zt^Qv1BURt2+77J$arpBk_<{SKj>A%izu&1p5l{Cg3fDz+Cvx$jefdN_!JGgQzTXMg zsAZ%v0yNEnj`VlVjYB0I?tUjsGdG4Y%Vbz z%Ru|d;Y2J8*p2jO4qJ?Loz1iq+qBom|%bcR=5psRhP&%87 z4)?}~k|PP6CfdnwYKzAQB7q)lL62^}T{NEs+e>#*|FaJrbkah*_l*a}p~g;E+dh3w zZ|~el??}2ZBzG788BL^mN3)6kRrbe6lX2tm^-61NYaw6a>+x}@uCue_fRl>hWISUC z?XmnDrzEkawX>k*Yw>X?ty5AQO1r&<--|ZQPQaKyiHaNLs zE(sTXb>%ae#7Hix|3~3wHG6l+bF{se4v)vefh}9N=QN*qpcr58VQ_Sq8$x~f>p&y z5z~r30^RWcFlA*e_a0$H$-L~vP_^#G>80cM18NJlJTWG*RWOnsIg`kwW7aK$!tbKJ zQ1!fbkk&QiwhM~C@U&Eh+rC$MTPN$A2cA~R^MU+2Ci{?CP!Ug@59E!D6z3QyWj>Hc zPjAY6Adeoa`9R)sG3y-f;Qz!xUT}Fp)+cMlRwHC0hrI^6w)+yz|IYFpbde7i|-{0RrE?TKE3|4ZWU^ z1R6|QZn4J>>jyv3ux8|93^mdC4?yFr3>q7Ywv`19NXGpspm9bH4JH@2*kc|Iw>(_T zz!DF&rr~0QOf>!-&=`|JV?)ulvY-JOxc>%d=rw92-VG)Jx7cGI4Y%Z5%)k;fI=#+R z{|acFl0l=hXj@s(fVA7602&X9|2$^X72++vNpwV8mtt@Cjg6*q-Mn(<| zCcn1WV;&8++*-`Q5;Ux-wHP52jUNIUCuPuRE812TG$5=_ zg&A0ahV_CfjF5@OKLZ+R88pDLFKlgR{jR^Juson#Bw(LBo1x79(V$@jXD}gbW(3 zMcc}P1{|0Dt{(q41(yd}?rR=y`a#q6jqhxDH59K6*4$lvSJiXDufm_+^gmkrL<4wi z@T3;BmGeuC^bQ-D2*B3Cf5h_X(b!2YAAx$3{SgsVsca$Z!QPOA>#Ox)?bt*F`n_O_ zcGAWGq5CbIM}hBVPq!sl!Hdz?i|k(2hqUlSI2;&TImRY6;9D!1S|QQ|?M%IPej)_J z&C{fqSwoa2#k5I3CRUmhBgN9B7zty>tFP78Pt-fnq?lGp(4-hq&xWe?HIE#ds3R}Q zMcYe)QEX;Z3(}-kN))4q8fc?L(SlrVGg^|PMA4F5ZZlev%WXzWa=FcDNiMe;Ey+=$ zXh}Y|87*4KZMNnaVq{6KpoL<<$dGs2LNQ=u=*O-=G1ON-8lI@tKc8#I2ieaMrY-Xg zvnC)oCLV~626sLI(;dS!_Iv_bk~^P(mYjcw^*qJmf|y_Nyoui7BIAegAM1*#N?SBh z<8W^lrX9h1vye^F#ZypQI8hDvX6>@wn}unF*jMxnr$x1g0c(Lvv+4udT@zI>ck!2I zVLB#yX%?b8U_rm`zaN?iI$WBCX{OYrSvXlrHovYVCju~>s3~Qnl1i}9{@UuRIByZ0&M|m?QLVoQkkc!;OC!gy>vkJ>u`=y=Q4_hV0l91ZKyHKG>M|m?L9M?EkkjkDOC!fH@pc<{s0Hn0OYRp138AZx7)xY=f>R&B4=gr@uKECw+N8C!Vly+>{gc% zxem2{At1NN59Anz-);ksoEy)NAAa)AS=oNPsEOR=fZReqkZZSFT}I^E)%pd1+~s~C z$7BQ8ZQzk}%L%{_KS9o#8Gsixk-H3#Ti^$BZFZ~6h+Lam-vY>8<_B_2=78M>9yz!C z0sQb2OV7Gxs&MhwiKl}taYjy%&)I=_#$N!CivwHl$ zqiJj7M;cyiSRHvs-4hV=FAIj?59Rv-@)j!Ya^!7)GCSG}hx!XtxAC72qp8Q^bM|I1t777VLB}UXsim7Sb3mNxx|Zb-eu1_{0i$ zYn#?iwn)FVLIg6t#~h7}7d4D!$z(x!j#3tspPwUOj%99{LVwl>e)DsL#O@|td@a+W z6U)Pav-{3^P2&@!rtz-`G)()rR9i6tHP+AfoVOKrLVF9D6scx+zN$#&;rmG%CzSOK z+R+KU%KC!M-m^YMiEDg6*_uycV@HoH*GIL`1XNZZTj4qBvlN#tVg~h!>n{W+qA=Kt z^V(UjVFO+02}Y2v>^9zxf^La77&yV}w9bj^VGycMYSmKt(Ml`rx%=x;iuV?_TCHCK z@&9$ub7^jgIgf5Pf{*`QYeu183IZNeI0`Ll26u0v{6EzHyRLbtX{hnphG!$Mg!xd8s(GnqdG+$Dk>JQ1r}m$|C84Pkn*q7tustUqAN!N>+h{yb_ZLL=n+(ah zcNwTAwf+|Ep$WZE?loz9{(xXx3=3_@I_!`T#cmw5A&a!Z%0c5rO>f%-sBQ8CwGD>y zvZDqJ^hQ8Uuc9lB8pA-d7VvX~8~<#`B2nwK%ok;?9W`K|I{`Jl z#I7`I4ExMlz@z5IJsYw})U4byUerXb15oSm1GNrAdD&3|=D8hE)9dd_qsB1LtOY!3 zZoIQ0i$u-JJL5%7)Y<^GHa}2nHn%%FB)#FwU)jTB{$ZF^n^70gsv+-)zVtQM2;Rcu^Cz^?=%XKTuN*VDz zv9PBbSgR@jpNAR%&3884**M!U)leN-8a^BPa?Pv3TLSsA(Eq?F-|oaH<4rHYDB~L- zk+EW>)bX-FjH&hAz$kZvZ}5yVz9BytWxO_OvW-y|6zA9?L3xf*7L?~0WkGq4Q5KZv z7-d0ujx7?D=NM%{d7e=gY`}q0#t-vahaGqRfuE5^JC+>{|26sNv#U45^zFLJ( zv(x9M2fl7mcrKSo_T_WQ^hh?^J2;q04EE+|ojmD}m&=y5Q-_?se^6Ubx*MtwZM6B& zM!OGfbQCg+d(sm<;lR-~M-dAexP*!>Y7VAzy{YK_z1yL5Ev#|o7+6z`&RGZ369=&x zQ`x*#Cg zbK?ZMr~uV<=edbH!-2uJL08YGc8#QC`cxE$+SKdr=+8#Yk5Y{W}U_Yjh-?ixz%&a}>OmycL;zoPI}*uWzD%&c2yV(~jK% zhr929H3@&X+h}v{U|aNXH@;=^)%MOr%k8$^PV@g?3AAi!KGXEEhUX$bji}*N=mYiN zsK2)EAVmGo*Hl&CTJ>P?vw>I2k_%A5Mzx-ali&u+2DN3b9$MIkp!*Ur@|KZrdvPsg z{*FFVYK=;^Gq`3-y05ri>ffW<+KJIfAiYOhHphb1xkF^W(Jd^zYjoG8xN!Bo!)a~% zL>iE2IH@(bB4JXkj!WBviEJ)8GUy!eyNZ^y$W>~cp^y&y2w0)lmGQ?4=a}EMKXO(S zBL88qV%Rq;oWuSyT2To8Q(%Q&RmLAIoFo4-TA@;pIssOk@XZS6;J=Jk6gDM~gB8bp zv%-09Eu$5MeaR$Pk@U@qOEpIgffYl(S#hc6s6nt|&^IeC)f_bdRt)%N#ig2~5@1Eb zH!CjH9Mun2^!sMTrJAGSuu~b21jY_Ma=q)q&vmvult>N^R|&fh1eRd>lnr_;m5h)EAG zXe3l1&3eIvUO!E6{x~`HO4@|N()j@};Q>EQaQ-rt#e~ATIY#;a| z-FQ{QlMOdSJ{Vaa{#WBgVi*w z_1dn=NcR=?U-l=`nf^p(dwyU*&lJc(k%u$A`tA?)9OiW3T+h0cnunb)q(aN)3P#9G ztC-T3O@^TKSQoc8S~1537qd62Tb&GSrYFFbFz==G>?hf!9L9J5?8Vw>B9(_m|O%*pd69+ z_e5dLD+?xYdhap7q*@LW=G-3BX&w`|V|$q4C74(b>|umVOnwe9sgl8@l`*d@n84Y* zcWS#QgMdlx7VR**_{CT&d@2vusHg7*-TQg;+++Z1Gp!tEH$-uI^CR%tiAoDsHS7mBPS8<74qu!cvaWUFG_CDjh$O zUAI4-&J|WKq=9T9dExDS6Ay(0dz3vae891;%cYF<4*V-N1cYA^se`+>LlU$#Z^2oD zTC9ytoCS`y%SHzGWePo_IO@ZkjU`Se;(3TG50V54X6TJS6^Glk1*-eu?8KRH;Lz4X ztbvY_$!s?7sG0s@4!!gts`cF2i81iKE}QQGqbzH(dlOXq+5E(5p!m1iD1K*Aw9#(j zRPUK{6Q>}bq05#;p_ALPX3xfA)$+45E&EO%>z z>d&8=$T?f?Br9ThX?dzScj4-ZY$R~%$b~pNo7t#r8jf1ubmIZzjVsPB%v)c4Y}G2u zv}gRfFub&RK&p`IGFPgajwwy2nKKg%{2uwFo|!^svBKn<0qQsdi;e$v)bU9@{$Che z6lgizoND@P<5wD$h8>ajg+Ck)hu%`3uX|_h*K2-Q(_MXU)rW$gg2&zzKiZ1Pw?O5_ z6Sm5Y#qTq-z~$HPN*nV)SysRbgzxXyQj>ZyzLvDTo+iGlBJsQ8O8~EUj^KY4fVox* zmJ3C_GEw&3gZ2V;ys7)PXaJkO94}fI35D|Cjc0|l%5PQB9Xl7!Akr~ z>N%Ew9{_Wu6fhMC<`szM-wQh|dZj#xT`%U`Br^f?Gql^WNi;0h1CtmbvSeBfM_hjy zc32k4?67QN%qz?H1BI zHZtaw1rw-a`-^~yUfoUt6XxzPsy)+mjWMcAez~nL+Og1p)l?4;H zANXB>iC(u(0u$zDV5ZYNCTD+?xY`|r;KCVE9W z2~3!Kf0<76n7Cd0i^jxy;V(wWd`}(+Od4e{>0rz&3np;4?<`=_AcqNat1r`O9uv1q zeKEsJ-V^I}z8E1BlNrDyB7;dgV_sP>ft!1$N&FvpCeU(O^I+4d#upmC*3c2z8Gdi* z=}>k3!n)(NXKKDw{mtr)Rd)oRgvT!R53JwyD_bSj@2W_|tNIf0s-7d>kHY$Woy_`O ztw6-9Ld5%f0E~Vss|+wA5wGe?#H)Iacs~Nb+$;r51tMM*BHm{K82#c^8DK;rUey=! zTJ;?9{w@G>lN2x&h zC4mu%dlg>-U&V9e`yl}4Mk!z_5c(<*`#uf8=+z)4rZPn&_*Hy~eihH*?*{=GD2pe- zLsTICRUiQVZ2)GOBrqZou;NPytay%re+z(FDg{ghf?x%r;NJvbZjb~HXO(_d)#s znVK)vtb+LePUfv@0jKhXdKYSzDP{ zMB+aIWzJonV(uEIW2DWN?YF7*&ue#0ZUfJ*-Rvb_R`dp6OyAgAu-E}AWaRw$LzA~b zy6g%M>9QruzaqL|T-5t#Fi+^W)Jja?Z6aZHn=fH?o9D3lvoL|bRb~Qjt3X)Y24VG| z0xQ4eNdTCV|U_`=d)t9hZ^&D3JApp}Z1xy9PY8Arj zPXI7_^;8*PM8ay-m#|v(99DlEfKjD@sX$n*LRkF=0E}KBRR$Q5uv+yctX4gT)gJ?3 zTBU%gKnl4EDdfMe$N$0lX9Dnl^FQ+=Fh2tGBQQS#^CK`n0&mO_nBFtF1J+T&wNIwm zLzj1bn*Z zS>jDfhbLL$PfDjJS>jR322ZlYr<9GJWQkWPn>@)9zfwhKG~yd7nVnQmex=g+$&+8H z#IICOex(w>Qa$;VO8iRowg^n zdQBVe{fWYL5#5Pgd}v=jkxwuuK!opi!Zm6cX^a3(v!EmW&2y7b z35UDi3DZnZcZvI*aKcpoH+m+aE>72)E;eBT>TTq*aOY7TN&{qK1L^e6)4ju^sYJAk z{;Ig<{vFJ7>;G2{p4^WmIdra`m+a; z!}%1s*)d8Vh!!4zk{{8+Gtu5`bU4wQ&BJ9*(ewzpK58hP%|(ZM<3q`j1Wps}_Wt_t!K=G5I%0<`TfYUd|!hnYCsabcJE&8?z^ zbI=R?HtU7b7%BKvTIO$z1f9R()azer>nFP-fs;$Lqv24nsyHcPTG2Eup8ZNZi&#w4~1M$#i^5}9<&x@A!KU9=afp7##Yx`y0#LGc%! zmdbG3_bPAeWPS6%(<*sBkYC4SA2JIn;)(Ntym67@90R4y2lD9YO_>km(PK3q$XhOE zo#P$+pBTssE)U51WUbh0giPeH*Fe{HpQri%xj@VP&6%bb8o$+eTf@DPXT#5jmxpet ze^=epb)nkpYR*(oSG`hI6+9Nm`LPk84NU6Qe3}OBl>jjDEa^X!p<7}LfIjF_1^}?KnCusfQDY9M&jLI5^#$>=FxCVzQqhIL8H^_O!Y%R;{h2oI*YcI1r12M z{Q;m6lS6~ax-IsYN5d`S7LA5A;T9ugz8n7xXdIJ4qoZhBS@M84!BD+_iDlI&j2?xejj6ibt0B+MD_zkwzlb)rczt(2fiF`}LgRqJ1W#BjS@u*a=FcDNsbalOLDo*Xh|-&87;}>HlroE+-9^SM~R{(`P^o-Xd$=RnrDcS zCAoqYiUA`--fauTfRUjeJA-2QuSdg^Bl_oa?f4-38N#$>zG2n`1job!(b3?}Ct$i` zn8uz@KudDx6VQ_L@35YySX>bEE1oyeJ6vS^F#cm%G1b2Y_ht<{+?$1INATV(WRsMK z{7-G+WD4%h+GV>p3)2X(ujm<0i)s%8)}mjU_5a|~tP?PI@t0;{IwpE)7OqXpsu?3- zMgKn@nmq1sX%?oLQkQ1oWGTo$g#rLc7|oWXy(#avX0w7_lA}64_Xr7Z#L)q4BE&@LwI~vCu9*evf{$lv*(3SPa>h7<7s^hC1|B&#jvYVz#AmEbJ6_a8?s7n` z$`9l=*sU%javRjp0zgi$^Dd1X!^GQd;E{9V3_& z1CN{=&yOE|^3GY=e!QrOTq7X&kRQmk+pR7ma_wrU0gyZE2Xag{fZYZjIk%hu{O}Xx zteF9LQ4_fcAa}+O$83%Y#6S+D-?m<70Q|wlk5jjN- z)dF&PKagWG66`kc$hqYu;D?_eXU$H)i<-#Q=<$DJ;H)11?`YcE_>qPe8&*f&QTGJI z{L6wN_(S=AwD1&E+~vsI{$zHv7Y_9osBYsw9Z2<(v$^H74?#dk(>0MyX+kE&C8 zjou|&CYo3h3f~Enijp9E@!F`#Hp*8}oFk0{v8o_UVj*!^hq>Ha5T6C%@95}o0tk*O?L24TR zia^7(k8jadOhJwH^F8Nng`LpeLMBD3*`2Q{QhE4(lE!t)I<#0jI;B@xU$EJG)~6_O zjqfL0^GR&%=#k~nwOVKjDyxsJ@SOBnipv%;gL=g^7lKn^80^J)?X1_ZfiCm}BS=?v z8*fKJx5OI^oZ!`3=Try=q57m&EtMaww9=ltzaFJ{Z(*y|&{e=K)kDvvxh3X2y4?sq z{&%ezg?=drcue6aw5S=}T}kT^#a=~GHPCh>NC*!x#c%JSri0n5Rl5_7eP)%y6U3+LsFO+*t z+MYik*cQV=8?p{NBt)?r2W`kAZLo6Ccu~{a+5oksexSC&P+oS_fPq#4HNA?iG-?b3 z&04_E5pMjmA&W$<(=xx~MNQNcKrQMAYMqAivZDsXGLwVUz1Lk=hpr+T~l}3$Wo>>cc)ZBPyLl%jem3PLAny9S> z)ZXF;YVC&dvZDs9^UZ+TVn0x0SZCG(9yK@4*^ot|X62mmq9$r<0JUrVK&{PCUUt-g zalQ#qyT%XH7{-~kfJe=ZZ#HC+s9E`Dyr_xVYC!F3KTuN*Rsw2Q`hl8aC@(u|z%<_os9oU)YRo}~wH1KcB0o@THI$bfwN^E>TstNKq5pwVZgpam@urtxl<^Ia$XKyb>Udco#?;Vl zz$mW=-{2W#d_#UP%6M(mWE-O_D9*7(g7O@rEGW-0%7XG7qbw-TG0KAS99twP&oRn^ z@;sv~*nk70jBmltDB~r`hs44t<0XmL8~2@twgRKP4&GXxQO37uzO_P(GQNF|QO1iJ z#%b`Ekq~5*g&L+yuPxfLshfdO-e+T! z@lBCQ(RjS{dbr3Z8_&+YY9&n*KKWMQlh^2LV2)45wcAA7WoxaU2~Ey`8{)b*ijXoY~KbZ=yk1pGeN1G2?{mg5SY;IrwOfQwpVNF9|kp@W*x119KIxqWk__R5*iPE9xnCUp5}LR;lbXrm?^028+RX+q^^ zr2SyRHa|_M{EV~@Ot{TY6DmI=-ShVR)YeE~aOWdk?4bkWt5x_kJAH0?;OiEJ=W>~3 zUp|*ik7T30gM*pGU~i7r$&>zgxolZGb;#-a2etL2yP^8fMw<_9wENITMY7VAzy{YK_z1yL5Ev#|o7+6z`&RKV-r?y}cP#R4! zs&P+h>eg`J$m$~q8~ZX=h1>U|^cg}_-R|_%W~_k&ql|&w_uvdP#i+)+GE=vN1F^L+ zgn_-Oyka$R+wL9(Wn@v1D!nr^wMjr}t4(PuR%tvCGHMGwdlU4x3BvqLXYt{RdoBnR3}!Ej=8Qt_Ys65 z<8i8}J2}+>K6$Pk85kE4o3p%J=(H47Oc)4BJ+)IVc}h)yDr6rtM467YTKvY21qoV)EZooFsW9@ zrR~8)HkTY3bPo7kMN3-bDz(l~NQZq0tkCPq_+y20%Fj{M7Lg-Sgt0ahIJ%?jt>zl>HCHYNMPiUYn`;XJpN(Tc*pWE`y6 z@0%5uYL4myEB5(j#ig2~dclf&e6!+G%~2146?=WN;!@2~F|gup->kS)bJQ`gVvlcD zT&g+hDC|`3js(UIJ#xM4!q0WKJd{Wd4&@U4QJtCdws+~;zOia@*|yX{({8gSdP zL^a^HUx{kKZL<>9fU#3~KkTyK1@7w`s~`TygAmX*y7O*2gk8sFV`Rl}1FH$*-d zSs(sfcuVN1P^A8`x=++Cti7@3eD!SA_k*to4+V~khwWN&Dg%SnG_3X7uF6RF74~2D zC(@bzL}q(_U_j3l$U%{ZGrjum5A_`8bl_aix|EuSoiC(9%jOD3$V{vFImqoh38f!T zO{c?Vzu<%qn$CkkU;Sulaq zdp`@9q~tJR&h0Us<}q)1I9NUTF|Wv{N?1u7r3dkxK%t<)acs z9eWsVFdm2mHt*6Fmh!;lXna?3W4)*p1|}LGi>DHna*XaO*T+-o_=)Vg{pobBuzDd4 zWDCg)=lZ4+;lLhc4+|e~tm|?qW4#0aiVXqbS48UI?(L8SZOvP7mY^1&9h>S0j<(B2 z2KQwOJ)$`3!<>yJPAB4dh$|101PNy7jXxEK+qDI%`*e0H9u6GZdWbd9Q8Jm$<{dTD zKg^+*K18*iIy=<|zSm{*Jz$h&O?GdBYR~7VdV%8KYNPm_MbSpPiBr9~b5jpMK0}u+ zi9#p0WzC+W4XQtLYAObPw$ zd1-m7dE~;?Q};&#r;c2Rv$L6v%BJC{^-VV(Fy6T0?83bD#m82yvP^r%uM5LVn+K!{ zxh`|1s_B^0becIc!NBj4PwJT|WELw-t|_37kHBK%e;svvLXZC!1{Vcd4mYQoKHK<} zMx|j#xAJ%kN-&^&e;HTiRH^q;(V!9S8H=eLnZY+MEnFTJt zeplL<2g(Tua=tDi}AIj?e#SAT@{Jn6<-2)#d8Gz%K%KZ6fhMC;T4GC-vhwt zwf1Cy5enk{iQ+wn@xKJX1f_tfL>v!+{4WA9dMP~_U_>H$*MpV#m(+7C|8BUo`n(h{ z6$s`Pi00n~J1lyoJc(T|=G-JR0rNAo+p$SBEY<^)7$LG`S`9~Be*tz_&dKbsY+}qS z%l8G&M*citqSwHaz=Sy!$#j~>#O+WdW_ZcM(Rv&bBV=OoIAC&C29u49d1b)_>e$W# zCVF){2~3!~gPBhAn7CaXj2T{niS^Q8jF5@R3}7-QgUJTQys}^d_XAG@CVJgE2~3!q zftgP8n7CaGj2T{nNvEfsN|?K*0FzTPm~=Acl?4;H{dW>D(JRtPV8Yz{%XFH@#O>N& zG$z&ye=$Pldolr-VnBgVwiS;^PjF5@R zMZhF0gGoDMURf}Kn|mK4@qgf%K+9##gH5LzUugJRLq}w1_`RX0L)G;Q>yFo+srgd% zH>)>R-4T2e9=p^(uzuICY?WBQt0EDv>Py6{dX9LXfxzl2nf1F`frwXyi1&8@82wgO z8DK;rUe%X~SM?n6eh7fMLJF7)M7%0QyiWr#`o*gN(>5AON#a3YZE+ zyedSzzYV}#E(eSv67nj(#Jq~4l$6$pEO6M(r)5*U%VSMeqARXj(& zzX8CsNC8uU&{u)j_X7ZoUJX)WDpN#)U&WW`SMePF{yG2yW$`3q@^pbfpQ<+FCtoRZPE1sj_CjppxDPSrP4l583KLNnhNdhC1rmnaZvB7LOkX;JJ zTE7M(Wo}cvPmli>L2191LlFNz*Z4x?3W)#5A^!g{#Q*m}{QsGnFV(Dq`2XXBf7-1NRQn+=9BUhryPWRoPKLKUVU7up^8m42U z%@(~b>>q1)P2U8bUAx&!zO3jCzL>tTwP3LWRLIEr%tO{d)k6UOiO?7?H4A^(CxU zJ%`mF0bmwO0aJmnT7|IsSpY^akSYU=NLa1<5>~68!|LA!V6Kq@rUEJCDx{EqSdagM z_0I(0|K@+@M__&g=0{+D1m;Izegxi_BQU*ZdM&J@f@_~lvxhV>8?NRRQ*#!sgaew~ zPwP!$ThCk>OH3CDodMN4*#eemNv?DyT9PYTiI(I_+M*@7Vzy{Wu8b{Ok}F_~mgLG( zq9wVawP;DcTrFC(P^eZB^jgZN5kFE?PqM_5lr~SY#FvzIPqM_Dlnzg_#GjN-PqM_L zlntI_iBBmTJ;@TUQZ{*#C4Qxf&S=CpR5ClMp8QIs^OGmPQi)%wp8QHBex-WyE0y?_ z>dCKE;#aCCzfy@`sh<2wC4QxP@++12mFmf_RN_}{p8Tqf_?58i-tN__J)PyDeD$B| z@qbI;J%Rd{>Q~k+uYJDubnSwgkJjw1{$BMXRo|)F4y^xi`12+B(RNSY3QHaSey9FK zJl&rtTo=)u$i;{DA=(tmeu8YFF+UzY|WF z>VK(c8tUS7t?6PDCZOI%E(>=a<)Jh{CN_{x?>yZ*Jeo>GyXdcqYd-Et+U{4h8>Tl! z0*AFL$R#z|_)wxhpGp`v0%gatxx{cR1MMe=6R|8{H`1RykQ~mZ$jy#X`arbs0F?ZQ z7M_XrW~0N2-fSK&bBd-%$n{Y}>1-}K+#4TCjwEoJXea+hTRgon66nzu^yuc>Me|v( zy>u7#Kl{)@CoQylfBnET)Y$22+o!MT?VTIx9Z464qlr}SXg1Nm%KrFhGHyJ+ zUTJM@E#yo5pZGLX*V)-|z)8h$GM+Jn_E`RnQ<7NI+F8)@Vtg7(>y*@zN+k=H>`4~? zjn=ZMpyf-4r=h6M<~5GC7#>C)4coS&7TW$||1=cQ>DonJ6PK~Q#~fz*rs%hx<%ZxI zu`Y5=m+dXEfI<=ft3A_DDrZC61}AsSCE=p4u6!nw7|BKT|0vw7X73Jpj<)y0;b|ot z*s^tt-d?Gj!n*Ve$DF!bN`SWgIqm#(E6l{Ur-Vt!v0_7ZiWtX{ijieXsJiPS!UMJgt)F z1Nn7K_93&NBAz%O$Qu_a&M{ERd?1gW-jw-39z9m`fxP8n);Zq6|A~RT;PQa1Pu7a9 zM#w}Cdku7L_gR|%p9{3y-<)ZBq48Udw>8`wc{cofczNiS`ghemT^FjouI5bjbk!?W zRl#F{oF5wj+Q774&8KP5UI_pb&yxN#8M-C50O*4*WdQKnEXziWkhyI9p%$Iq3ztA7 zY-Ip2jnM8FZ4jyXDYea&e12=FxD=!^I3N@lb0TE=I^ib{3D>zC4)v= z(YCUn0a>;G0BCHNLxZ`cs@P*54YwPrFat}_uwGDw5i-&ECZMrR1`V}nTUpS6yQ#hb zXxt`;26Hu4vBx|bZr4&_29}^N*Gs8ZQN)`Ptv2iI3a|D_$9z90I%V2gIr#sH!FEu2Sz?`BW8C0N0W(btRaUcIJ; zr;mgKV=Kqlqy~IzB~vRznxLKePoPQfgW=|BQp~I&N|R#Rq#qM2O^T6XX;O@YIph5Y z(4>c*Xi`ioC1_HNsAogf&`%#ZHvKm8l3cXCBpAhJMztVKYNbRmdZ>XmN)#=~lbqCAr*Yv?Q0?jF#kbo6(XSC5o2hbDPnkh1_Oqo*_n-P~T9P}TfR>zp zhxI(g;)0l8@w|!N;UeRQ@gM7o>EGbqta}~q&BC-JcyAW6NxFFYS8d^R58Ruz%XV)T zrV(Oa(KDPD)gA_{MZYxbCva)jL72PvOS3Q?6TLJG*QRCFj1jP+|Ko?I4>(+!g=wbL zrCB&x3i7{b$?5$tnk`9ty)sNIg+*R$6tX3H`=B&@ZmbA8KOMN1UJ}sfX}BR}@JHIx z>3whoK*ClFwp#Zb9d8@#_@*7 zA}@x&7`{4mW&N?b`)i-7d4KiGRo|^@4X%TSeEWfjI;}4WWFzWLG)yZ7-68{p5An!w zZBoN6+QZX&ZT4%j+L$d979CNOwiw3TZUgVju2)$JYPR?@egNnsQmuxY0l8s6klSdt zx{SzeRKrbxoL<3Q8aaktx7)xY=f<()hoAV2m1)O|n#eT*awq&iZiC(GG9tG@4L1OC zdYyM^XKjAj|KagV>e!C4ka&A07e)!2dXJz~Gq9$^6 zfLy{4^AVox#bVwho2y4%_6{yn#ffFa=m^ar`oM9BXX)54gzuy_<}7vaAFxnq7H$7CefZQzk}%S*ry zKS9o#oq!iLkPE%8$N!CivwHl$qiJj7M;cyiSRHvs-4hV=FAIj?59Rv-@)j!Ya^!7) zGCSG}hx!XtxAC72qot0pY?)|cNho|LOe#u( z?8R%NCfg`qL2-^W5|roYUqN|}1QwL%s9-^Pjt~}<=O`sXd5#<_-&KjF+U}v;uV;zWUMl^eK32o7PUYNWZm01TwzI9F2?@HH>A+WI=h3QWlh- zpCe$7Wp0^5f7S?o^K*p6?j~J)T?L7<4~7G0_nq~c#wSQk<6jYInD+5XxX~MGte@{W zZ!7GC_7*ZJQqAssRgucW_mecPQ`X@tw4>8{mGuRiy=Q%j64&^CvNfN?#*Q9Y4lmL| z(@F*FFmQsGYn{_4 zVGycMYSmKt(Ml`rx%=x;iuV?_S`9CN_zap}h8mx3csBA{__c6%sJni;Zo2lRnwM&pS1+#`368vRYX9k55}G>G z1jq%4?K%1Q*q@BwM&o(9zaX;TWJu1v%Rn`$;dRn z)H)31Wk(H|=T(53UVm2_HHLX+E#OgejdM0+k*HZYXS}G1+6q9e+7Hy)4CQ4<4H)O;fLfIws4&=1s9LwVUz1Gaf7pce20HHK|wE#Oge$)D$%w)gGFD*bmg0eFD}39yK?f*^ot|X62djq9$tB18V2|K&{nK zUUt-4)$nz2g8CuoHGhKI=*8TlCwzihq-lwTJ>9@sLiztZ)BtF{v+2&p*@mfx>d4aY z+0d73UJc$7$d`rw*S5`E9iPXuh>Vj55A`j#0*o8pg7WvY zChOjA>J#nS`I*aMuos`S^U3($6jrv@bJB(|^$|W!bY+tb;klirpjiSDV3XUl_L&7R z1l7ZKHd(kMhCu08qZF5HNTO|4!z%2{Tn7D?-nnMDWTPXckaJ^_4OyfSRwfxQYDV}9 z?54GV{nCu=CPR7IVS^Lht$*bO_s#7*w;_u}jkygSJ>Qt9t=HrKs{@+? z&EIa`-SmZ~2O59Uc(HM5!zUYVkGvRpApE27`EXPHqPl&xlQo~O303c`x)A(y;587? zw$3Opa0_DE0yb;QW&oKKV#$$gE*l$#GHdyfWNw}D4EcSGJfd!kLTR{M;&e_grA2-P zRo9%POMcIs9(GhkC)@pCf?n6kHxrb~nV?V;_JIlO{WPJqawfD=6Yc>MNS!Lb9I2yn zCUj5}_JRp|Rc_xLsl9S0v{MuA1`}@f(}cFlnb1Z}*aIf4@zaFL&q%w$gq!>{q4G1* zU0}j$KTW9ojCAMQ^E0a=fx(@Rbg_pHjIUPV)9m!Q>4C3X6rRgvl70DHGCh)w_6`na z5`(=tS|?BXAw9)QE8y$tr;$7*PmEpkAHAfK(8n}dt zE@}>@bG@nP{=M6wbS(EQ z5H|K@tO~d9N9i+!sJia-%yO)O1EY+A-S^-OG{vaKotc?s;XrI{3}IkzDz8{g+_t+% zK^a*Tq)K;WW|j&lZM7+F#VRcbQl(w-nH$1^!;4AX<&u5}Wt&r}7@*RZ&yE3x`v2e_ zrdqZS%tXOqmbg0%)uQ{F-oEae2tCH*RMED9nd`BNbj)=Hx{n|f8IMy%w}rfJ7+y>QjcwXi1P4|f}F&K+!v9`44sOupLQ znP|D)w#_vE|CKD|Bd=<>kdNH|9nkV^{rJ820t5kr7XDs z6>L=Nndt&IST?9FbM?@|J_OyDh>^F9eA|m_Df4&qnNn+1vYo*-Q__9K^-}-t*VfK# zj|9?tv}JQFSe-jW<{RC@!n;OyU5X1=-#Z-Fw$E$>BpOa?4X#L-RIB6C_Fy8LOO6aW z2mG$0B`tE5T4yMv!`=s0=yhfMvBEj#ckPdy6@|#Z7p&Oon-$Jse;KVP1pg0!6?#<} zf3R?l{L5&CNxB6y<^W0iSD+>FPN5P8CzFBdp=BWF@ zid%fM;!@2~N5G0rzFBdp=BWF?ijBTmajE90!(hb*->kS)bJW|wica6GxKwl0A=s(x zhy=zCJ#xM4!q0WKJd{Wd4&@U4QJtCdws+~;zOia@*|yX{({8gSdPL^a^HUx{kK zZL<>9fU#3~FYL0ngZui%I5~fxoL1c>7oAQI7b7M;yr7X#fi&v@6WaVV!TICl)GKKd z3QOmMV1nwW3C>@pvY1d{O}b}cz`1O}^VSnIW2m67f%?7!?!q%-}A%=Y}i zfSxIkgCY-SdiC8O>N(8mz`34vDK!r}Ur2?P%@vH0nO5<3$nCoiNb2Z)EnM)Jdb z@NrHL#0FCN>`)?ma3~Qae~QvS>FIlgKSd$oZy=o+hTx)aESiG{8D^>2y%J5+o{a&o zbPtedr)-2>3GqxKmjqhNsauvk?rgO19PL>$3!C2XoAl^9{eN56xi?Nbc3KP1?2QCA z@6r~Q^1$S1d{=Q}y{HrhCK?}$rxKQOjP5Ge$5ZL}iR`-l>2$8JdLa#D3&{(o`eyD9 z2lgm?SonZrU6)H4>mB%4YzPRyB2ou;Z-*pkYuIB;m|A=W@g$z(R0chpS(Fo$0H5Y>9} z?95%@dtEl)14dcaWcMbh_H=&cPN4X=+9-Z!QMA!+;#BX*xtU#%&(LK{qR`20S+nP8 zgX&M6nz;k~Y^%-BO30!Oca^7_Pn?^%-P!V1A1rrkgX&M7n(217+(}l%^3w8D^U#H> zXLd#cr;c2Rv$L6v%BJC{^-VV(Fy6T0?83bD#m82yvP^r%uM5LVn+K!{xh`|1s_B^0 zbecIc!NBj4PwJT|WELw-t^+_F?|{X||2pb8p~wFVgNp(!hnrJPpKbg~qtdV=^1kqg z!{N|d>hpE)to?e;4{N%s@2&b!@Kf;Eo8m`XF*5>{8&B9OHx|Fo%mSBRzbkFb17%qO zD-gcFOH0k@#rRs%_IjH5u8PF(iZ21Y;yHr<1pp=`1xy7(cm-nkp9f&{T6;3U2nF%} zMDd=(_{Ra5<5Iv>B94bZeineyOX831NT3YZE6^9n@s)3C#$ zSIU#v^aXCo&86TJqW z1SZU>NT$;~CT@o!F~dt1j@ILl7$Fmr3BaUZ29u49d1b)_>e!A0CVF){2~3!~gPBhA zn7CaXj2T{niS^Q8jF5@RMZly_29phpd1b)_?gxGhFwyJQNnpa<49s+z$HeVoV9f9m zOgcU7RKnc#bAZVMGMIES=9L8#xc&E?fQep_P689=-e0EEJSJ|}{-QClUigaFL_U_*ZE?EOibPZnA|UeNjqa+ zSulZ{dw-V1|AA)$EtfS9Hl1pGq2X%{9g&^k_lBMhRo5@9J6?OH=1bMztln64NAO8_ z>{9<|sagHXR*ChyDiZOkzC^sL=ZN>Wv=y@fnf1F`frwXyi1#-ku+nd3l>tU1;#GZ# zcva64?{7e0^{^B$6^M9Mh(dr=@_Y zK|m43|4%JgB8z#@Gk=}87W{Y5D6;~3cm+{(M#sZ zOl2amu;NQFtay%we+hsYl>(*$;jjYn@Gk-|X-Qy2($p2VA~u)}2eM0{SnJndq|9xK zckA*0A}H_`Ld!n_+t9T)`G>ULPryc$eoL*y z1l}eRR=4>QR=0T$t3M7KQJ2X~;B6HMtJ@%~{sRC;zlK%<7*!;!R(%PpRnKAd#{igS z8DP{3gw-m9)xQtG=oMRKfDs9+RbRqt)pJ<=Q2?e<3YZFn)hdM5zX!nRMOkHl5echR zU&3nDb6EWm045>@Oa;Pf6~gLg0T{jRsthn9VYTW@Sgm>vtA7`O2}uD{fv{SIu=>LQ zj9yw*1{jgBTJ~6egw?9&u=;}lOqCQc6-XghA%*<6_4q$n|4ab>Z~kX~ z1m;Izegx)6V15MVN8pV)0@HhDuY`3}aP5<6_K+rK!_~ZEYRzONK ziRmJtGoV^0Tfh=6$(61|OL9dk(UM$ATeKus%oZ)lm9a%jas_PBl3ZCzv?N!w7A?t_ zt3`_z3e_rtUQ78j;zx?=NtSq$(&kB)_>$7@NtSq%(&0&#_>Qa$;VO8lzLlV7zFzY><++r4_Vr?WhiuYOLC|62m@3Dmz- zzp`$5?en##YZugfv}SMh_o^SM`cBn$VEvE7pD)3WwtIF3EOq$%o%$2;bbq37T|{>x z7a!V}PvjHK2@v7?op6m>Mj9hP(=6yn|IN8ssD#7a?}TZlr@O@cPB>wz|F3&yp)O9> znl3hB0_tt#vT)~79!djbVgu>)&eOfaqp3u+i~g#(=Hs5E?Y^kpFuODoIILYkE~&}J zhZ6nyRKmCsC_9$TC5B@eXg@ieh-Cr0k^by~hvrt5*YZrM zurB?=F{kd95}F7_Pxs6I$7U5@U%*v59HS|*@w)6 zig@CDAa7ixILAOK^MO2idQ;{DdGuJ#2lAGSS?72M|0f3Wg3AN4K3OZa8X*%o>^0D} z-KS~(e=g8+e{-hkg~o3+-qvt$fcrObX}i$_`86HUe8AY4JIwO*kgzFgCA&EGjcJ8nrM6n(CCywV`I^_vY-LUxc>xb zbjYE>2WN{t=FxCFG>aKn zf`;|XEJnyg5gFI?OU}Q;dY)o&LCmjs z-bC+kk@3U$k9EcLD%_j3)#2VOOgn=2W+9uTi>DuI3uoU7_h#*~-J6AJgxFW~45vl4 zhXHHZ4wq*A04~ki0&^FCX%?nqqL*gj+O({iF#=Zf|M{WWTOBUV!ZcIr(kz@T1^E>% zIlCD~vn6S-SB7b&u*i#zLbgP2AC!jAjTJ%Xrvum0O9J{l4L76=eqUQUdkb6vkg!#e z6Hr|=&&j8uY-=``9?bL(>#VAnhZn|D4Zp0%|Ca|I3^c#q+}*UJalGNN$cy1GhOZ7? zS%0kV{@SN%-e3K4)px5}gX`cS-+myXew)4|kd3G}(J-wTbc+lWKExx#wMmVHwTEZ* z+U(b4wJ}>JEIOhlZ840w-3H#5U9Ykd)NJu*`~c8Nq*{%H0J(epKyIVm>M|m?QH|6C za(V@KY2+Ao-EISqoEyiEAAaI9R;C>2=YZhSm`_z7~Ia_?LXAh+KS^AVox#bVwho2y4%_6{yn#jEd$ld7&a;n|xG9m~01^)@i?eYUTCd{{ZA}_X9a5Bf)M1kDOax0)F@j za@Onwyr_xXPxbh}F>qFo|93QPZTv{Xiw&zI@2GnMV*X{p5d5KhKS16>#a)iP?N4S$ zd*M)jf$BE?(}7ekN!}VVm?p^9ROHGFSIz2mb+72SFzhw57o~9VaM5?vO61~YSvnUl zXShgYk+yBNFA_MnM9bUGzM^*?N>ID_&PWb`8d>{U9x4Oi6x=%oiM2=39=Wj zjhbwudV6(W%FJ?3a+yr^L;OC}4-bCj~6{QMjN zb1ZYq6#BD9@SC3_Bz8CH;%fmU%ErQhv-{3^P2&@!rtz-`G)((=8Qka%HP+AfoVOKr zLVF9D6scx+zN$#&;rmG%*D32ri*|HYud=>iv-hk|QQ{ikPqyZh*x1n{%aLX+Gz*p0 z$5wbw`Ygp|i+dV&$8E4z)iqo7;j4F*oIQR|$&4+f$7 zq*g7JAFZ^~p1Z#urFd^)tJO#Y#Q%q(=hEB~a~|Dp1Rwvq){H{G6a+k`a1>h94DKS7 z|A+d2*EJ6{4K+U7@NDF@@N41jP@#ZtkD43zY{()}vvSXPQ4_UgfZC`ZsC5|1 z%Z?f_&r1O{z5cE=Y7FztTEL^`#ycCbNYt#nGhWn0?FK+?#1GWk4drD=tzC^o0kvU2 zP-9qU)&d?iH_q9RMWSZqobjS2YS#m5DL+taGnAJdwKg?!9iVo?57ZdOnYDmN&5dt1 zWRa*@`DVPRiP{oC?YJMPsfO~hqo%5nw*YEMKTu=XX4V28H8-x=kVT?q<(l!LCTfcT zwIM%HQw-&0M@>;9*8*yTexSze6R;NWsJZdXhAa{_E6j% zBUi%->IC$fKS6EuV(!rsK0z(gw8X-mZeU$S`TsoB0BF9m>CVR4hN*_?$kOoH(3fjo z4c-#Smxcb5VGicE^eF9y>rcQ$wo&?A?L;<8?s0vtV}Xq)Qs@gK!kr9?3ZR_ zHyO&y4jY{4z6nq}c{(W;h&uz#eQDbgHN6$AVYOD45|LVY|K=ZeocQ<{Z z>4C3s?PSg8YeLmKt1bjT9e51{w5^X{3j?y@LQ7E&PA4%rc8PAa4$H*hnS5S4$NxJ0s z%;{lARdll54JPPyt$Z^lTIQa+zdbK9@|7WTU-LUmn`!ZIA z+xMgN8A4RuZRy9Gum%o{G6r_vgEP<+qZ+qn9&Zc>Vrydv1A9|>#cJZV-8~A*$f6)s z`qs?j4FXDAZAx3QN(+Ki>6ZB8k#OMfVp4axq~Af==2R*MsPyHtV}POlKe&gfmRkoN z4}-%jad#N1MfWwmecd+^dW^@ZqRj)3hp>ut%yk92k02Bok5fgrBp+&FZ$%~_r{7WI>znAGvu|e8v}2tYuKNG%oeO*%#eMki^uCvbv3VNf!w+m@Y-?8! z+Ze}S*&+r%u(5Cq7$Kid(iur7>+Y~+W88Hbpg@R&^|G5lo6__V(xjwmn*0;e7cEUv z8t5Y_P4j4zCLuH-P1>d{rT^c~?C$OC?5%#Yvv;ON`m~=IX@4{GoA2!G{O0$Y`JJhT zH3@&X+bnZtV_WoaH@;-@)%Nd+mfLM>qw)W*g_<@t9!q>G_FVL5Q8hBy@ZtJz)?Z$? z7p(plYO1QQuevY%eCV~Z!~#^XP_66CwJ;18_iKwiN2sw6LH8xP$y-Lg?Zvf}^*j1Z zsWmFu&ftnEIef+IrQ`c&pL>1t*Ux_NfYqJtnfVoq7E9|l_fZ-dM$I1Qs7 z3~4=%t1_|&js2HB>Fj84dUSK4uTPH@$U~BcV?Fxr4;?wo>A;zu^++`jJ735OZJR3? zAuFungtquh2c&+yx8S%C16?ztp2)2Pn0As81t$RS2PT{4m@sGmm`3ybypH~%n{7Yz zhY_-v90w*h$}mwG^~%Bo&i5SyCVILoiT8v#*~c`RXX16R57WJ5DA|wnVT3FuW55K` z5&3^l6h^(WFoDy1M}f&Z%EAQB z=G~`lKXW}WsokjUXBWR1ZG}(e;R^Njy`XzPg%f8sKyIdGL+pkquHM2hd^Y(2WSlUq zKd)VU2J#i9hEk;(Kt@RSaAD{me4Mj=-F<_F+(0_FcOac4e@fCn>EV0EpOO&p*Owg~ z0^{P~STYX}GM=Sk^GXy=Tb2XA(t2>B9dS79N{El9^BHh!xi!nS$DK&-tEyW6t_TK8b?Oj{(h zZ__?jL08F4E?01^nf_shdg(*7)_aeiX@#-Z;TU`1QI=KNy9io)rf{YOT>P6HE`E1W zw9;PUwBCUeXHrWjyv&LO>H(3$QOUu)m51zX8%<5?9$epKB>}=+6Wm9w1{-&D`m~UKhc46N7 z;$zE~+onD9*GBi!;(?SA>oQZQn$DDl)65tN2IC(2q#l_ zt=s=|!}CH-`x^%n&&R(SS7KYDAB=oF5^1=izEJmY?Kf(ERMS~~d(}t7pM%HV7C+k3 zGy5QO__pBR z2VlCSz*NA5SHOmU9Kh(g_GG{a8S%kb@xIOY_X3#Pq`*|djt4{jF#w~d(vtxrV##|R zti->hzHRwO0nBbGFcmQ76|m+XfgKh-Q=Y`G7jtfsnSl8j+UwXP8jJnFBu0oVnO49N z*I$4gmR&MCEbAHd%JO}Ivyl%26Fmo>1QX^|B-3b~iPxb>O!tz7qy0D}M#y6F5HQ&x z!(<(!URjtx9@__jiJsj~f(dhXFwM^?3V^(ge)fS0VcP|Fj>o}R~9C4 zKk&~36FqO81QX_FV5ZSL6R(SbG2Kg;wENnrgt_b8z+{^YlXgbEvM_<$e@_7uJtLh2 z6XxDurqMhTuWNtNOzao_VuY;sL<1%_$uMbS)GG@UxZC&VfJvtu6XsT5rqMhTuS6HVch?u-eAU3b!+vxoufJw=KQ7T~dDq!}00KgoS1S4Yi zDuEcjif_yJ7XeI<6qpK_z6#jB=KzeJ4N_t%Q$&nkB@pXZ@ooN|1u&2nPr?sT0sB`0 z19%$1+${-4!~#|VF@Y7|Ht-aHIUoh50!FX`R`4W%xl0m^h#9N|Vh1a}4dDp@bEg!T z3RuDln8M=#Mo*b1GnI+h!b%{4^sOz?F0M&x%i9mrC|R*2KN6?!2Z7j?EgQgd8uYO*#Dmn zzZ$;vt&;;#n|B5-&&}@6ve94|O)MhtpFo*$*QXe} zhG`fnvq{ej`@7mLXYPlQUAw_gysYRAzL>VLxnQv!WJKhA^8Pc&AzXH;k8s(N>0c2| zFfZ!;9hfKdTWTdH@D>rXx+M^^y2ZCy{Rx=B$7Ck(mI|2FEnrqZ3t;qXXeGd?B4)K3 zh*_=rHmiRdz#NeQqgKGIR>7?PIDpYJw#tAJF{{-;%xcxQS^Y5pQ;-5v0kc{Kv--CH zjGmNL28@VVtp;LNtG>sASR;z)S)v9l^`ZoYfRtii7%xV?P>R$&idiGQqFd}BP8i-k~ z`ZlY74ZsXZfvJF5t%6zoVF05iNR>>*9ehO2eO)QpKM;eaOh z(|nEC)-#vJ64OND&VX#4YywNPB$v7pEy*RVL`!lhZPAikGF!AHm&O(?$tAEwOLA!` z(UM%!TC^met`;q7B&t;eM=kZIk#VG`zGTTrQd)e;lCh+;`jRE1Non&XOU9GZ?n{=8 zC}piLSu&=Sb-rZDs8ZJZk|pCx6`j$@*igyrr1~0HDxIHvjVqOmE7jMyQpvbdeT^%X zj4RdGxKhcuQhkjpm5eLZ*SJ#2xKe$ME0v5Z)z`RE$+&9qHLhC7xDuA#)wyDYud_VV zU;TT!{ofQi7pi}yep%g;+81h%*3PNv&|BGGYkQb+8Wd|ED0r@uaIk@wv0I2~+yZf@)tw(!?Mh4T#4*IL&nvZ*umir~` zs_|Gfv|qc3TvC%u4WxSugK6_dpxjt4pC0NSh4M2)>Fyk`8}7~BmKiDxlA9fq^ns-D z0Hpj#8qXwqa>=1|Pp$x$IVH2hPZb`hSRv3Xe0kjn?D|nhPt#lUAp>C z(LxS%FWp4_&jB>htrpt67w#E{96KE?yY)4_vvI>c!&#$Cb{7AcOb_;qy)fza4=(N zvLjRcH@cSfMlH|pABUtm8&yuSts}VAh!(Ibj+kKkG|K~$ZcQuYCUW|V`enaf`=(CX*B1;;stDmfUrmms(%9>-< zr>kD8stVs7$_KF#p!JRG*?bcH&P)IpdzQ4HiO?;v1wbG4NCSYc&9-dB2wBU9#TL7)AA+D^kI2O+YSH)>&{!x# zV_i|Vve1BF+;0MnE9Gb~vAD$+^EABTa4{WAMyNdu7b9fR_y*8eAVXtqQMaZ-B-;IU39@RmB$bG`w!8!gMU5VZWdXBV^I|YoIY#hK5?ytt>R)ZmPcm z8W+mZV6LVrwwR~kbuAU9V+jrWtyCBxi^g99jX5$jl%j5Bp#c|Cy$m!ikfXsIoGrGP zr{Q&I7SpkWhW*SeM#!S^63}Rpq0wB_tt>R)xa^m8`#%w05^B1maVYV_#KQQ)vDX_? zwc(nb)wfhV7ycUj>23d`wT`cW5gR_Ng&pbq(!)JNW+Vc5>)=1S3)zwG!(2Q99ZAkd zM5t0(BkI9fkjv0lqd(T}9$yXZp0iOq?C=1g+btYNfp2D4r_ETwi_zDM?q0pFMaEY} zLSxIu*q{b{X(dA|M2et|`4PBDuZ8aB-K3aVL)1-*DU*Io>~2zw6ze9%NLVx8KZBce zrJI`+Q%VUpDMr+{qH6TD2k#!ghP)(?I$jb?#a2YMpqte062;U*6?C{n(SlrTGg^{! ziJ~RB*k-gO7u$@M zLqB$=i{S?kMaEa?pU+i~^|PNLOj*_&W)DDc4LlI-4eopbra7i+?D+(=BzHamEjjxR z>v4+31u?(kc^kdMMUNk5{Mc7auWIwgm%H4Xg(*ky-YjI1bn*0kZSMFoxHoH?k_-@Si)smrBVm|{v@ znuU|4kpD@`j4y%SY|1$Em0?OT7J1zx5G~Pj5K_bE$BNMT>A*Afl0bdFx*I|UzoRW0 zUkq0Oq#ar01gb8o=QgGxZEG%{?H}zK(!Hu;Bi!gqHTrGc{=YDEU#RiT#?Hi+_+zn0 zqAy3j9J#dN;`+Pm?y7yd=0nx5R(-FkIlLMk3hW0g>KpYXfviQno|rNeba$Y`meD{+vWB0V3QCz$uXXIrw%+hFMm9~`w6*rsdp~=CXm|@1i5yn)@39I{`&t0a@Pexj`7+%b>PW) z`R)bD***ApQS+UPz5(Rc2SKjQsdX92fp7o6fZVzu$T1#%rw%+hFF!xN`^h_J_x9sO zEpq<}7Gm9kZTQs91}U<)PX1G6+eLQenQS3MSvHz$o(6TYYBp!>eRZ7mc_YLWXXkW+#nr#Q7PBRPmg_z93}4uTvLk>J#UC+8KH zfbV`n&K{kB7q!U!i*EnNL&tUde@kLh{1dU4V=JQXse2M^{>9-2_(S=Az885&RbIkQv}(Xid}H(l5st+?nP~xul;@nlg7TazSWupG2n)({E+s*E&N(b7&%1~P3vh81 z<4bV5i}8|V?yxzH@sjkLX1I=H@rP35+u*HDta`jj`mGgmAmdxixsmarrn0OvSx}yH zDGSQa&Ji%jGPg{jKWl{F>>MGnyGa*cF$k1>XC!od_i?{re1fiN{3`+l(>6x6rQ?ue z{bbikM`9=RXd#m#t=ZdORdnUy+er%7UDmOPc3@o3vOZ^n|ENz<$2Gp4Y|baKzN1H$ zV+~rvIAm5ITk1RLvy?AeL=8GB>Q9Bow?JplFKEa8nhkWJCul)>dbja<7Tm(MmIIx%cZ)%I__#wHm7h`+qaET-vw9oJV&m!Q20yIit`o1%byT zjzWuC#$65d|3m)2D;ozA1Mz2L&qm*fybR4dlW@&X|0>?N9n|o%uZ7Ul7@EG9_o;WuP^wu|?Yb<9edp%d*b+0l~5uFSIG^ zvO_{D_VPiSvPd25K4`qCHQKHMY6pU#w$`k??9|q(u_RE_v*=1wV?5BT2K*f1<$pG1 zk<{94^E+PDqP7sI-5CV6cC+%bQ)^daR{}LX#jZ3p#{0}_z*F<`J)5#fYIffP-x)7zQJW9c_60$$)vUbi z)LPZpFc512`yBMhL34$8qZDuv#sd@RDO<5#0 zyRR89YEiogsO=7dnqpR7c4~?mn+Mcx4T2i8Prz!xQ}gmOo3co1c0V&-)S@;QsO<`Z zTC-Vs*{LKQ zql+TP8@^KWdic6fp)Bry@F?Hw=26D0Uc#e{FMwFairuA-mj%a|8e0t>3 zWqb)vk1}48d`N5_WxOOA_2zx2u~pzv&cIvCdzA4dT5qk8M;Tu}=TXLsn#!^sWkGq) zqbw*tJ4ZM?%7XnjJ4Z+_Ypw;4@_>s+8BapcqbyV~U3#t57LWIXM|royql_<#Op4~? zrPsqnf3o@P%&S&XH1Q{21ODVbeGSa{lkugA`jbUUmL4lBz@MCU@h6KkA`6)?`UbN{ z94B$nk*vEq>6lm!cL?=DXU{+E^e5w6W2|iL=cG+xI!1Uq(bJo33eP-fjG85gfH!#= z#FD3=3#x~l-elp1m;$9=jZ(g3QxdJS8oL@S{DaVL>78rFmuxnq6ge+XvMGyH!tP1N zi&_@`Qn2uQK!0hE?0U2EvSWi2-AjPlJwZ@oPIR*x@b8<~d2Uk{NsYM;9X;Pz)E4XZ z|D~bzp~mktZcltEaZmiG@kir}VxNuO6n!~*PvpmulaWOIyt>`BkJo&$rlERk)v563 zLvMh9wrQdoI&Mz4HiwPcawtG1h3?F7E}!cjfi!D{;Y@zD`3(7eH+e+8F$t;R^68^_ zJ(U*u6=Yp=lP>u^b9>meDmvMAf(m+GtH4xHDyM=%RoDtDgo9L}xpFErQx&#=3M5Ze zPy?y0aw@b@6*{y{<9b%^zy?xlEaWrU z;asw(zkf8{-;<|#@}xgr9$VIK4RZVbL2fFl-iZ^cWQcy}31!<-0M#u94l{PynZN^q=2+~T|rp9xT(Ej-(?{dj;2WguJ2fKmF z!9s2f80!CndzjYJ-Zwr917?BufT6YMV@)q#ADf7J%*Sa(ZGGd1u@&je^$PScf~d%R zoL1DD86SZ$xzPJHry3E^dJSY~AgZziXk9It@vJ~y%|t7x3q1ETbAr040If?sF+Lm# z^|$nUj(n=uaJE~Yic*kUJCo1t%qES0CHqFRL;7qs!w_NRY5j_J(fCj#)RXN&H6Y(j znv(9_VwkPT;cPx>{O;N$j9T(mj22S#J8FJ?)4emc%_^EUtQih>55k&+KiqAWIkT}X zdbk^3GWlxz_e9I>wymM@|F4CbHZ~qhd@A-_^k-2uGT89p`ft`>Ubh#l{ugShs;{rQ zFZ_JywX(zlRIpI3YvLjp28;W(#hxS7*oUC|65Zr2Bj5JoTFUwzeWuhJm278l#grVr z;`P$;{Vr|Q#Jp%IyF*(%!-CblL1ezsH8kEey6aM0xCY+fecI-Uxj-UzSc`d*uw1RK zOWVEaTs|}0?{4sWik8&KQ);zYA?^0tK?^;vOfXuw+x(vGk<-Gk{JTJlIe}^6ZuXZ^ z3&Z%|3tH$|WrAVhZuyr{3zd$j+dzw^z_f5T{>!L^u_?I+v}g=W3-`ITj9M7`lDk2R zL||H+uQ}>g&>|j~7Uye@+67v~0@LDr%~3l+i)dh4oUb`*2WSxqOpEh1M{Nf!8UoYe ze9cj}yt^<_9}SJ|dvKxW!q0QI+?USu59HImN!>H&Z|~Bxd~?<0v2lr#_u92Y72vgH zi7LQrzYU^MYDJhSO{ts8AcE z3ho~#w^m83U@V>A2`bbCse=2LsVpiO>*kxN|Nl^^X<1_|@&3e|_*DFo*i*5qq92K_ ziF_flvEk{4X#FE~pQ)Q$dv(pp>L;px5PmbfFLX%UY}YaqtDv(ILt2mHs*LPGWB+AO zIy>5%9^G8%>(e6z@{r`=SdYH@Lq`sCI&h|EJyOlX&KI&m+vW;J$O@}CqAi}d7E(Xn zTX0;6fv%ZRPvlktOgqVlf)jxE0h5(-OqjEOOrv>zUPu4X&9)!M*$$q2{BV;kj0TW0^4+1Wxab z0+VHOOqg?fOrv=wUdQ$@-AkC*5A0!tEGCD6$<;DUni=)V!UWFdjcD5^mI9O7joN;8 z@r%(`_*5RQP+#8*y7!YkF|h=4Gc6lpH$-vu7KY)o$p;|gglYZ5+Qk!)uP8N?D%Aio zLb`_wLkHpGobBuG8!Y4o(z(3@=_L76lKx2#-!uM{gn+-k?C1~}7YE0Zd3ccVEEStq zqG;N(A?>1xMc_m`;&9lN5FbtFGvL;8YnE+~JC`(`qbV*`L zGlCcHJvfn!gmx%9SbX4PU7a5^*E{g9?g4Q46-n*gz8Qj`?Qsk45>(^N*u+Bc(RMhT z!3Rf;7Ez9RH|M(3N7JbS*p+)pfCSU^=AVk)?O6h?yDv9!WhAt3(>_)~SIJB+S8%PF z{$YlC=|i;E^zn%WF!nkeV-GyavMPHQL2FMHCawS%|0aiv-(3`~w3j%o_uz?%`4G?0 z;RvG8o!hc%&rk-f|DGcgm%}*QpyU0;!=0b z-DE{HFD*}NzU$Pb6PH9oNA5h8VrMglE1R05_BY*pzmA@az8Dr8|L0uC`*r(&Zg^g(X@BEj z;`#Vj<4SBx^n;O)M5p8fnPsZ1jb>`E=H&w)bR{}BM72g*8F94W#NP($<39o<+ z|1f~jbM4815i;U~vEqH3@ecu*>!rX{!j1<+{y_kvr_z%FBVx&W9<0Q_q`qzW_W+pd zq`*|bm{-7>|9RM9(KF>q?0PZhCYcGCpP{{uO`@^b4@_c&$dYLV9C3X&?69np*KP#eLdC|-5U8o!!r%l^>gbE)gG&Psrp;h z>#E)vehMBt-#@T^*RO1qSih?x7OxtJ#jE6L2LOzIE2|6` z5sOz1#Nt(bTfDyrU~ZKHQvr)t1&j9_fYC2rl>sAS@v4D1uT|d`?^yt|QwmH4EM65X z-e~}{Lk^50V)811*u08wqjw6xY?lF}RKV(0!0ep_FttvPc?4^V<+b*%t}rcI6ytViAG=1j>xNKE>EIOv6Z-O?qC~k88I~ zq+w*&ZtxQ?D|&-3rfqC4SZoIw5jh{bf1(${WtaL0mo1t8710FqqTb(vc|yOXR$>Bg z5izS<0x_#we4EuDg$ewi%mm(20kgUV%<5+VjD8KR1Q=DstX2att5x4-^+y2AJu+a_ z3YgU@nAN`tVDyZwGGIi^YBdnETJ>#K{|11$TMA4C%xV?P>R$&idQw&yFd}BP8i-k~ z`ZlY74Zz$b1*QUKwF+kShXIV9cU1f5aTWdL)#6qpK_ z)hd|PzXV|P1gSD$M9gY65VKnKZB{=8VD?IZsXz$13L)fA>h^!Q{$nBdzuBMJ9+>Tc z*&dkff!Q9I?SZ#u51igHF#zkR@T#Y>>>*9ehO2eO)QpKM;eaOh(|nEC)-#vJ64OND z&VX#4YywNPB$v7pEy*RVL`!lhZPAikGF!AHm&O(?$tAEwOLA!`(UM%!TC^met`;q7 zB&t;eM=kZIk#VG`zGTTrQd)e;lCh+;`jRE1Non&XOU9GZ?n{=8C}piLSu&=Sb-rZD zs8ZJZk|pCx6`j$@*igyrr1~0HDxIHvjVqOmE7jMyQpvbdeT^%Xj4RdGxKhcuQhkjp zm5eLZ*SJ#2xKe$ME0v5Z)z`RE$+&9qHLhC7xDuA#)wyDYud_VVU;V6Z|2Kush3a3a zUskuI_J!J`wR37dS+lG9`_&IteYa{ec>NE-pU=Y&-0yS*mOA|XPQB?=wl{5D7txu{ zrv`Qx(uFj00z~+JCtRVnp2i5!Fbmq#e|BO5GU0IdJ7J3H>MU`;6Hb`c|3cRUcEKx%-|?!Ihx>(QQ}k->DbgZ`?x=Hs5E<^G9w)kHoT+OJ(iE~&|- z2GYHS!L)fJP;M-jPY-pELiw4YbaxKe4fp15%M29;$<2;Q`asfn08)M=jc1ZQx#Uo~ zCs%;WoRZmLa(&c5HkVHh^`r(e!)aViw2^mu{l|=Kvb$ zRts(3AKo(oId(c)cI#_;XXA!@hO@5B>nI7yJ$)$UjJ0Bm(q|C?HD9z2yM!duy zq$VJ{&epcu+^QJLq()7l9kzetDoL+wZZ~TA{nP}c)+t%b;9$nkWJjj>Z*(o|jaojx ze*%)~Y*^{4i#fvRK*N@;s7Be(^-e$%osMnfHSy@%Tdi)UZ;F2FS$+Vn5$hn=bUEGv z8x%_Xb2}y=RnFR$wQj>PpMi_MItrts>EV1*{}1D4HD`0kbF{qA?w=Tpgf?#4sFzpj zrmznE!ZEkzmJ*<4|DJYo;$E1EBWlDvKsyj=2v-#+MNBDr4|Kx+!<3b?-Ft))CG)Z$Mb+5voL)4M0cvwLJ~<|_ zRWO_#K9(NMcH6fMjNc`DAnSQgKh0~%Z5I@O;cKZ3w|y`7w@%hK4}7hXXASw)O!OhM zpdur2){r+ZQkDgZ~>1dBNoYS)Z&DTaA#39QGRM z+U^rH{y!gTx~p+C@nZbj@f%{dN1u(n5Lwc2UHxR;Gj$EMSJoV>K3(-%RaN-zP(Fx_ z0IhFQ&*qcpcV+^>*t4YlOoVQUEdctUM;ZWpZMJ12M#x$={=Jr*jKU=lX-66WOd+)S zMcqVhfWVip#=ZtL^n5-NG?=j5VvAkY4?)ndN91A@wP^eu&}fjMv973FS!h5o?khl} zUXBJ6i(70lPs1w?7t^t1gxbS!F+vuNzXcj~GBnl}bt?-Eh`{|S(9m<#NW2?N0B*6x zJPohlTTI6i8tr~(s;>Zz8W|exMcv9m1Hx|q253~v(O{x(i!J79ctzZzY1jj9F+$e6 z@z+43N`^*TQMa`?A(IaZ-*36JJ|wF;BxQwieT|goZt|79(WQcu7l6JRn1(rKnq3Xh2l$mx0C! zIU39@RmB$bG`w!8!gMU5VZWdXBV^I|3!rho3=OrYTUltp-Bd3EjpK4On5(IZE#_%> zT}y@OSVF^oD-}k_qVeZIRYOw3x5s% z^tS)eS|=}s5gR_Ng&pbq(!)JNW+Vc5>)=1S3)zwG!(2Q99ZAkdM5t0(BkI9fkjv0l zV?WUDp1cIwJ!hkK*x>;}w_7-l0^iK8PMfiU7o)Ei-MxBMi%eb|35_iqV}ly-rIie= z5GjH-=KI>o$%~-7c{eF$)(~}*V#=f+6T6!fBgMK&F%s5{_g!txb> zZ$;JEKRtN&UMr^Y^&JZI@Vg+q328;}Ow{0#4j12wQnJ$L!JQSI{ zQ2%_cdaR%Q3}MQ$-Y|Orf@|P`Xm4=m6EMv&Rb$U5pe4EU324dLcUX^8EG~%o70=t~ z9WHwOFyqI*V*0i=Z*q>yy;+!Y1nmRgRCYxaH;xEm@G)(l;EL@qkRWnAwPXAl?Pd2(-nuRH*)TLQCSqk}^T4pi< zz1fs;<}1ULVl48yM<80F=OCnp&yN+M^V5N6>Lr2te04X341PmfG#Q600Md>uaspKs z)pHxukhV3K&-Rb@4C!7~u@P?cr5gLXZvS5xx-ZoDW@Be!OZ>6eBhi;5UyfYbaB=^z?1XxvE#d+j2XM99WQE;`%fTubr9s%I<+n%Iq<>%2awbAyi1c~Jn>E) zcyeC;czpL0a_w^O+|PjAk|4;nJGCw&Iq=v2JCIu(1UbfQ@6>@O=jFQ>Bxm>F<3+7^ z?%#miq9Dk%Ikhe$Iq>cOE0DV?2y%>v->Cyn&dbk_?|$;m*}eUEQH$J9fm|{Oa;;9S z%SaC50Dc1G76w6%i3V`$z?1We3BY$hA!m;az>8Ys{sqWg83ef&r`Ba82QdRb26796 zAjd=wICbF3dBqRlyPuG=M-kvfEpo2|xhsMor#iJRBRPmK_z{qs9|Soj%D|}uPtGgW z0N?$DoIToSsqScHEDa+d``j)_Qc>cErpic7$EKOtw2PQZ&= zIy(_Ht}R^gVS?g3Z4;+yH+l-w!x%A>%IRyzR~8Mta~-zv1dO z|I@{(UebAM%3z8hTT}7qsY@pHyt)^)of`6M*^9bx@wn(aYIo$~Wm$JFUd|jM@doY2 z$<}D-!~(70IQxp;c_`u9#WzN>1Jro^L+YfSqj$l^_a;~;6uuFbD@wxIi?5BU>~Q%C zigQjQL3z&oD=5!7fd%C`SFoTw=MWZ@=UhsH@|<&6P@Z=Y3l`wwD8`rIbQj|#$=qRc z8sjDDH_dPz$Lk(SO{(zLCRRP(B>mP3Igs%!=G@45QBzsgnJg&Jxs(OvXXgl*W0_l~ z(4RHJZ+4E5*xjUyuUa@@p+rK*cOUl~#wX~S#=jy^Fl}RvwsaD5te@;U=}7E^9xY^2 zq&0i{tBS5Xd^<_uy30CVtsR)uv#ihA;6Lh9)Nze(C!6z0tncWN<#?6WFbSE}$CmmI z`Yh$k7EyzaitwrM7cpf#!SE42G3^+dUsWu5T@f@LvYXj9f@hlEt@<%2e5 zkviCY(0EaEw8iHGway@@tu-qzJGHfH{BoeCXVI0W#(1Dv4fr|2%l~Z3BB{08=6AfP zMeQ=6wj~H^?Pld=r`E2v@2x@I+ zMeQP>c0&-~gUk)DS+hA<)9%Xz%L3ot$wNaHF z9%Vss&RZlX&v}#uGg2YpKLxm z^Qx5;P5jAAz@NNJUjuXgWPE9&{$!DorN_!*@F(wd@h6KkA`6)?`UbN{94B$nk*vEq z>6lmqcL?o(&YpkR=}*SD##q_f&qRz)Y<8$bm;uT@|wD3wz|p(<15_Y+s)8CwZIx4@ zjjFH_RM4|>2R4veE2ly$RpEM2p+86!S}LbP3sqqQsL&Ur3YDLct^*a)L8?&s8EHMJ z&>N%*m7kH;y}K}(iiY~PKG?w?IxxRl#;4isbJGQ1wC0S zlc#y|q(5FBTh?w3a{K;4ZawK{r~#DG58~jP9BVe4y-(YSkS;FRCH0Z zH=FMnOzzpW8B*868fS)%wZ!P0)t;T~!PYpKE7)t?ep?r+MoWy=*fuzMPb75biaQZH z9vrn-xOoq%K2wNR*P5N|#wyr1!YJ5zD^5X6jMmsPI(c^_)V-=3pdTwo*foR;r{X?}~)>&nJ18OO888+dMef4O9*ma$~?y{~z4L zw3g<+$va`dEbtyMv=)7=>E-KV6H$-(IIU<+-{c+Gigf0B1^O64RAfF*D_WhI+z(@N zq4#S}H6oz(8pzNjW--z892Gtmm_0?+--oS-f$K=i9nxpB8HNZePwQW)T{L-nB-E4bK{X)XO`4MK-C~%n z$>D51Y5eZmB#c_}R*V)>^gC*Pebc=&w#_Pw*e97di z?cWnEx7)UY#{a(-YTDR%Eb*z>bJ3qg)yQDOhwHyte|gb~&vq1Vb1 z3sAvAwXVqrU>Gd!*A{z@P-7p0?n`u&w~Tz-i)$(Ccl4Q3YgDqG!4*?-_=?v{$M+s> z)#Qn2D7!;jJi~(3y+LHY(KR&QHM;9kT(}0_;N9Bh$@_ss?64N|Bw@K)U6;0d)46%q6#o~D!0HcdlrUoU>+y;?~~iAyX2zN?ct)uq?;Gi5;B}-9iYN+kSe%;oZMO^ zt%9+1-V7=X1*wAjm#Hi&80+R6ssH~_sA*YaEb;!tocL7ylGszRtD+x?u8Djhva#Xm zhG_jGb)Tu5TYGiQ$?7Mneh_{$yf1V}+-%n}Qx`*LC5E&f$5k2GgU0^Lo^*D!H$A$! z(ATF&3gjWl!?7NH_lJ%g=5*jp&w8Yqhn+8Eg|^KVjF1&paadbCbrGa~ytm-E5CdH^ zqn^mE1ekV`5d|jzM}WyZIVQ~6Kc>+%EAQB<{i?uPc;IQ+Kt+NcJYhRR`^sN zu25g!3%d7{IWd)h+)T@c*bPx!y@g@;Z1MrfIAL1K(0WF8)5JWIvql_;9FtY5om zDh5upBMygM3GvZ%J_Bwow`SS)xN}M4Ioh&J4mQ2vH`$TZ`u{es_AZ<@tWS$fMWdk& z+qAi*Mqp+nwXL|ZUaS;4CYc&b4W@1B7`>}plN!vX?#->#4^BlQ zp&iN&79aRnSLX-K^$z^2djK4MMN)gWZ-yXfd)$J%1l2e-Hq`(=+75>^_~59~BFa(k z=3ICBXgXB@yK*lHkYKvr{8O>JJxic<-;`seTdfD zeSE48#$JbG?14vFR%P!ZXzd3IQ?=mY-{f%dyNjZg_7bP{-gRQC2I3hy96=Pib6ZyJ z8Oosb-*IHB8phcs$2cn?i&orIp4Pno#8j2L=FI_U?o|e@f8UX*u)F4NvLc$7mZvq} ze(KVxP&9Pp&QmFNHgmYLsX1zY)6ECWH?BClFmHYFvE|Ec)1LWjqkC!bK+1@9nJH9F zXG+6qW{d=bagTgbk4z!6m@&ET1=sOOSZw^Ca~{VDwaaGGIh3dC!BD_?OhTEq@ZgES3UO0b^bPYyJV) zVbL??N$h$t=O&p6n4h7&j!mMm*bhu%gvgR<1srib0Xr;L$?UMKXVfdp_XW;I-VaRl z9C#8;m{XBVqj@G?haxfEOBRmyGOyF+cJTSRbjtO(CFVkqAiPxpRnC>O-iTyfXjF8172TU%JVbaQ|R~9C4 zbMGjz|3e=OHC@oypEweKG4}OXTXbvW0}anKRM*d~J5+nD=B4UyRj;dhXZR_2?0o;g z`dz=WRbu_FidejAAQrFc+v1&q_4@{y^}AXDi&q7UcM`zpx3bEB5wUpHKrCL>x5YaF zVAe~4ser|+g2g)yVDyVuWx$A7ylNoMYt^^Kdj`O)l>$=%i&q7U_Xz;gE(b;tF?p3h zY+l8;(ffV?(U?44?gdd^;_OAj4@M8d`SrUwh1*`;O0xQ04;70+>8YwUp zFoG4Zf*%1et0lpRn88XQcCg~x5dH-Kvq}m~1uS6&OyP$CjGi)2W-1f0g_S^zVa2yK z{1AXyDFvni=CA_x@Ph#68c8rBVd{!k5*y5h1KFie?D=ajQr0%bdvyDM9;Ehb+6VUk zbMY7BOTqqs4DA1(fc<|5*#Cb}^HR-nu>U_Del>jSTPFvgHg5_p&&}@6ve94|O)MhtpFo*$*QXe}hG`fnvq{ej`R$pddfrtTFd}BP8i-k~`ZlYd0x%s?U@Bl%t6)|?31IZpsxn|i%xX0dvs(3S zR(}w{+$aU60%o-eX7vXEjGjGJ28@VVtp;LNtG>G+!gO^~|NQ#59q(Gay?ho4^t+$)&DD zOL9po(UM$BTeKvX%oZ)lrLjdzatUnFl3ZF!v?Q0b7A?uAt3`_%iE0(WQA_=4WE?4~ zFIh5@lonsIWGpGIzGTU0QrdjUlJTUp`;sLiN?GemmW(N7oiAB3s+9GnsoTS3jfM|4pHDq54upVUjIYz=kxFb_dDGKOCA1xr`~ib+nYA7i|9<}Qv3n_Cq_ znbfE$w8QpqTqWt1&Fw}lpGi$YYMqj`3=U=tO?G68|3=rc-l*l%`==nO&W4q)x|k!3 z4m51pifWYod%aVTM5kjLc}+a}_ExK#>6@b8dX^u6Ys5OpHC>Lkzy^g9|J05tNR_j; zWv$zA%xB=Dua3g#XnHuG)c?b{SeNfUHkeiLFM+L=JlmbZz%h8vmaUHQm)X zns_n(?f4C`+oR7$UWhDdxUPP(?wPuV+AC|0RiCbUt*R<~cPJmkMu65grDyX=^gA;F zVC-4aekMY<#1;U3&?5~1zBb#k5hG+R8^59@r*d!!MB0%C08xUp{*duZ=idr=O8fYArp|P&0TUls8Fz#OgjS)E-Oe}7(#XJqK zI9yD}k`ZbT!^H?$H2xB3WMycqE$UVl8W4f|GSJX-)JVJ=OaN}N#XJqK;9E?`5*qD( zXR4Qg#*hq+_M&cOp#fpHUj`b3ax|Ez+hU7(8eS2%Xd3o_Ta1wPZu|w%xL1ZoTT!>N z(12Lm7lFnhIT}o;ZL!5X4X-d;OvjRU!yaXe5wd9fInc<+&}c2{Ru&o%VEZMYF(5~S ziLWiTn5W?tTZ`#fLc<Q)vS5LNpHpwTBsgSn-u*kYcB*9}#e zjwLkg7gS+{EE<0XG}1CO)S_->p#gVOy#O?NR)LaILj8VBWQFb8LgE#_%>9h${-ETLgPGm8W+Rz?W7sv_hl^+L(U?H|Yb=-MpI= zGi!*tNik*8kBQw)ijiX7q!gXa9mnd40i)}_raxPJ{Bp2I^mgHia(UM$jGg^|1ZAMFSE>W~3 zAKQ!;HDa6XafTRK5-VtPF<@lKyKQqZU}WgW&U7(+al+IGlVJ2dc*7i z2(Ez#qP@YLPrx+CRE<5KfR^OWC!i%~-(fvYvA7`SS3GZ{cev>B!;ByMis|ccZ`LuF zd$Ta*2;Q58ERrss{$86qH3s))ZFAh4g(-ySR~#8`i)tSo)}&vW^)nO}!{H0l# zhKXL9g)7sxYQ_lI>HppRQ%77b&B7E@>e4KnEQS0EBmlS%db26x%vXjf#aQHZk3h6U z&p}8HpC2ni=cfbD)Jp>O`RZ;68T?yq(NqDh07yHs$O%+kRL^ZpL)z9{KHERqGo*V} z#YVW%mumd0y8VA)=)O?nn~j}`E%C=ff~orZ0qcUzXFx9FeeSi<*?hc+8zT z@MGEYDl5U7ZQ~i=0dx|9QT=Z~ZcY&7);YB(si86YF$Qh;M@NZ zkc$REj`8q2b>PW)`T6nPPu@AZw;wNRk^3Q#iv&Tg)v0wE$w3^zKLfdjAjmP%08Slv za$Ye3`0gjVhE0L=HH0;K_N#58%6>kh4b- z;6*KRuL8N+Ajqjst;PW)#TwwdpOCXh9N%-+qIVujxOVZ4 zk?a695q?OW)^qeO*!bQA>x9BL!g57PID7H6QI#DoUqNxsX(TAmxqk)aIVZ57Jm(4) zl;<46g7Tb8Nl>114hzcjE@Hs~TpY#t5}fX0yd;@BY))gmB>kotuH!`Lq15zzcxw}@ z9&eIQ^g7UL-1kAC_EmP>v8sRrPM@Z~$QorE;Xvyiz zBcbEFkNXYd6Ld}EUlAynw((7E=``e6KiPHCk=O}6TF9hGYxee66rg0bl8TJ1|{=X|52NDDEXJgMs--x^s z>1^n%KV5ga_LZ7fYL-+lsTvLszjbo|>01(-I;{Y?@Q^bmA8-4U{#$20Pxlu@_M1$} znRgjzO={v|?fz*!QSN0~XZ(O*S&SFjly%u5Ar*W1piNn%4t5_jUep|IiHm^RnjomH zH7hSWwY6$u9#GS>=t@&#JkYEL{2bxse>P>2)Y@(HJ6_bHHW#R^3W8d@S$Wy1wX2B> zftsFTSDG5*eP%V_sd@RHO<5#0yYCqRyENG)RqT9jqy6O8t~M-e9opUlA7J; zj2E@2C4kzpAgHyNm6x4bi<*c7wX1`m#(11r4R~r^{$^7aNzLwW#*13iVnA(a5Y$w& z^0HG?)kGAiEeV1e<85X&;Hi1}noU_GHM_4FFKSVX0JX(IP*cpx%T7&E6AeIZQ4rLa zeF9bko|>1R*_1_6v-_Fxq87D!pmtRd)SAu8%TBFXP1L~&>Lj$9KS6D_V&>ixK0z%~ zw8X-m`n_wZ|Gxk^02*&j+#G)*_P$tkbW!AZ!&hov4__B5l*RoI9_5X09%a1hB|OUb z0*Ga-*j?&)S#XT0iACU1z8=N~?@`7V6of|^UmI1~;ZYV8=e$LN@|;ImP@eNB3(9jI zWkGq)qbw-Td5Z+)IghfSJnvB!EWpL1j4#3IQN~M>4~fmAjF%*%-n{QLaTR!!H^5uV zdzA4dT5qk8M;Tu}=TXLsn#!^sWkGq)qbw*tJ4ZM?%7XnjJ4Z+_Ym(qmzRty?j3*)J zQ5GtgF1;3Ni>KFvM|royql_<#Op4~?rPsqnf3o@P%&S&XwC+z%TnYZocti;iU7)k(+1723(^cIfQ+hn@apd~1xA zt^J&|DNM%*Zzpd+n@`ohn(JI;f9z3rC*IwzGPDpt+SfA z94!1+Xt(svHRDS*8&Zm#mnYejMJi$UB;!Rb3;!~(@LNECX^-rBv+}ZITdyWA1!`&# z)R+_9tOor1=5?OiltofwZbL`UHx{)^bo>9((E3p0cN(`RzLdBp{?quQ@kO!E#%_wf z9K9#<FzuJ6;{Jxt!qTZN<)NuLq(Y&5Yi~I_*uDMB<{GPcz>{=C_Y_9_q^t@Jq zsi0I&1%;}x9#pt3NEMnZr$RGTVI8QjCrB0ADyKplRbeftpl9U{Y#_B(PK8#gLOZB% zYmh3mR8EB!szMv6uq#LvDnBE&f(ko>RH5=SQVXcCBS;l0KO?E{E=+HahWfWY*ufq; zFuz*Hr`hdu(*<9*Bs`ZN%^WP`Guh!>vZud)G~M5mr+Md2a_RlAImrIU2NZUL(*bP(;7II_2Q2!s?!?c!b`lh$QfLY)@U}!D+Skueb z$0nj4^Kn|yioWR%Y(+YAy#jrVASyB+rxh*FOmBuUxzPJHry3E^dJSY~AgZziXkE)P z(>Ds#)l9U4y1;WkGbgBv3edW)J~4emB-G#1?>X|RUc=dLeJV;pZtYAyw=bkVuo6hAk!~O0Czo%$PjXb4Rn-$V-zZta9 z^U4IHg}cq~*&aD949kBTXpss`3wN`>j9M7R|2shoJ*!MGEZi;sGHRjH5p@%2(G!>! z?#6!^wJ&mL>1t*S&1sZ+^KvA?6SWLhHqdV zC-?7@+p4?dqSNi+qQ#_}7t|6moMszAg?&M);Qn!PYn8ML#?twEP~r9SPCMV>W zFlYamM)Ult+=sXtQ0yX znHozCrfumMy{lZ48qB8d&8^;(&E}2O3n?IH1TSO{WTASMDVN5=_^d ze=2sjX9=|KJ95(_kYs_b0^ zt$kl%dKg^%n;b5FcTu#`UgEUg+fPgnK|Di;BZxwGZp*4YLm9OGy+@`8VVrGpjI$E5 zXvIC{Y0bBtn7-Ft^X33F_bP+dzx&AaA$QH)WJNSDEl+E{_0*-)nP}+9ou^XlZ02xf zQ*+e*rkf9#Z(MP9Vcz=UW6PJ@rakl5M)%U9+^UB zF=KMw39jP-SZw^Ca~iG+a?%sC&5f8#O8!rJ>Z9S$!DDZWA8qN`3n6pky^hR{#qTq-z~$HPN(=KqSysRbnC}m0gJ<<*d`($r zK23a6MeKJa5CdNEZNZ-aFc(OHselQufDL~?fYEd9$$$|u;)Ai`eVg&e0ZgM5m`d33 zV8|Z>FnTII889N2yywA6{7dTFmLCH!aVanrFyk zl7*xFI3z~MVp0Gm4KhsDG3u3t$vQQW2PS%UI|(Mt-N8(wc_v<02V=UIFtJ}6j1jV! zlJ~@Zoi9enVlo6wLNZKR8THD-1a9sfB=&#kW1*%C8v7GR;xER& z9&3wkjeMZtnTG26xpjwXkJY?X{jKVCRqqTx1&^KYA6UQZSGG#5-&GNdR}IACRef8$ z?}zpK5}Eb8S^sAS@v4DXysB@D_k93nkrbE;SiCA&ypIDI{o+*_ zFd`PO8i?~+^=04h_&osTQYkPMFoG4Zf`1;sTp|fZ#0*vf zv4a)ghVZ)q%*9e*DqsmKU<#iCFnY>7nW;>~7FGf=h85q|um)h}Nr9<=Ijn#^{BrbD> z`~MGWUaDCR_W!5DuZC}Z>*N5`=ADJhbF;g%Y%~}~dBLlFkwwxVnqSo}KD#aw%I(TI z(!?SH{|S^CcYTVnYnXf_+r||=7Pm`kP(sdEBBvm zhj7`YKEh>7rhi2=!Mv#VLoiS1x712Z;4LC%bxR;-b&GGa`j=q>Zcvftb~*Z?pO-0Hex)Q7d3pt6)|?31IY$tukOl%xX0dvs(3SR(}w{ zG)sZ0fLX1AS^WV3qbFsR0V85otAUu+s&BLU7Xi#_DKHfXo!S`EakR(+e*Qvl`~DKHf>3QPq; z$W;g-e?qta!}T8v!T-(v%=W-+56t$!Y!A%#z-$k^HGAOnjr0l5DP^56Su(1W^}b}uxKc%DG%_|+GCQfh#+6FvCtu@ACF4r< zHLg@Lu2f&+N+sh;^);?kGOko#<4PstO7%6aR5GqqU*k$8<4W~4u2eFvT6~SG7Ba4c zWp{P1SmEm|5A|1nShxS1LgzyDuhcKATT=T%?a|sfHJ_~6RsH?y2dlnYwHdtrhv3iW z;YZtkb{i~p`1_rD)2VE4+PE&FGo4Qj>@K7WY32lo@cmA>LTx>b5ujlfw5LCJ;w)ss z;qG_B6w}pN;(jNbFs=WyU1uRLPRGg)HedqsZRB%s=TQMt1B`a}WwTq4_6&^-rjs4? zSH(3S_arU%Gul;WJENif+C}7&np|oi-CGz;n>PaG#&Y@eQ1>X5pBYMb=YZXCZ|=6t zP+^eV?3knvB#j3kMW$zDOt+RT0XJ=EF{(0u+mi*bA-`>hAmrBjk2HZJqt;6I<}G5 z#G`L-wYr(UDf+Ev`2n~_tb<(B<#-EhP$===-foy$o8Mx@HqcA#}9?mEA z|1fS=b2f)ON6Y*8{5uo**)wB5|`kk2oF!n5IKNF!_Vheyi=#d5h zUz=^&h!L`ujW21*vvH1vEv5;T~w++vGe)(=6@ zut(%#6t!sl1<<%dhQ_+0Ze^hX!MHC1js0>om{{Cmi+LJeak!X{B_q@xhKmuhX#6?Q zc$W-~wME^^LIWajzXUY&95oW}1`~i=Y%x#6EBF@Ev4lpu-)rS>ps`nmMq5$0ve1B7+ZTYwZE`f2P}^dQ zc^Y0}wwR73?}k0f79(WQ_*0;&}rEJLHYs9RZRz;W5W~37u$@M%8h8S5A zD`;~uU}VUuUTmuh8dxJZlfN74Y z8hbtgEy_3sX#~OS5pY6!Kp|0)TGl&8CbqUm2zpW0BWA0?`sZ2O%|leyj+cpAI}z zFA3D=tGgj&@GrGRXYYn90Md>uaspKs)pHxukhV3K&-Rb@4C!7~u@P?crJ8tIxBo8; z-4|+nv#~R=CH`3Kk?6~jFGntIxVZlAy1QzhuK7^)t5x5tY7Vc4hXVTni~4?jNg!)c zucu~O(dZWGD7=YBx(n{G`my%FSv@!VWjSrk5ebX7s7YCj$K0s{KbAeOvJ$M>HlFbv zKqnCx)vp7&V?mHx=hV84c-Nge@Z`LF?D*~{W5(`j$BSCzehB1_ z20?DEQ|mI410Vc9135jPW)#TwwdpOCXh9NQ^coL5`|zWWI|dvpR`)FSsy-Tsex9BL!g57PID7H6QI#DoUqNxsX(TAmxqk)aIVZ57Jm(4)l;<46g7Tb8Nl>11 z4hzcjE@Hs~TpY#t5}fX0yd;@BY))gmB>kqDuH*lFD0Qv|-rB^f$D5?zS|JBAzQvpy z882!o%Q}+<b zz1w&_jJhS>VDJh4RBJ!`0CYn2VXay!K3Zv}E%$ysO8LEowN?{90sH?1v|QS^#GFTW zD#6?To;jn?F9m_eB#uIhTE^YKQ2#&V|GTnrATbbsHuh}vjmR63&W6tV({-n7U#WSe zW=Zvus^RePTPOFQz9pfl=Pn0w;UQ;CKHl~x{kP71p6)M*>^GT`Gw(9cn$*S%wENHL ziE=N?I^zcf%VNCHrmV{j38~o22W`qCb+G%O@uKEvYit5)mj*#?tyy{5sjXET8-bdh zMOT^{;wRu5MYcne^JGC~oF$&c5{JYZB7|%1S0Z+}#?`+B@YK9~&894pn%&oo7qzHW1GRV%)D*MwvQtyk#wwr|3xXQ6Prz!xQ}gmOo3co1 zc0V&-)S?y!YSAF5HJg=}om#Wn7}D-O7lBsuC#cO<%-nmzC#Xe=mRQ)+4Xpp6{{I5x z0BF29adZ5M*!yDD(M6Ht4PU8wJ$zlLP!{(;c$BYp^C;t0FX2(f7eFjy#qLta%YtJ} zZM*_J%1dEv@E&D+K|y$w@wHKv9Uf&tan4&LD9?G61?4%9vYqT9%Z~F`HdpI38|Q;Zc?rC=yhj;dqV?7ad6e=$D z=N5rSdAGx(j4z5ziss{`*TY4Bvia=Ht5#C9?oV#K6#U6o>1$xlpNua})SoO;vh-NF z1pLWK7k{!yBeIYQqi-;K#BmZA9m%?@la7gtwUg%-LTArE?DQw&TVt$j?dPOTVLC>5 zJJHjdYzogjXpEXAh*YbM7iq2Mu7obA9&&nseIlBFSX=r_@@jH#%6JJW)6aQ)a(fFd+XJa=-Uyj}r`ElfA zBvC)FZg=hDHD9c0sNPz2D*XA-8z7);o22xw)RA{3rD4>F#l{>J3)LJDqS}&b38D{aYXGU=JObUoGR) z?Do0og0EW=p39GB4i@s6>~JpG)89Xu?(fOdJbBU|FOMy2w+6X={~))XbTiZd%4i9o zjMf0kXrqzEjVrU~S|gzYD-R$RG;j$OUDWK&=6eQ{dv>f6zsee zr=TT9Yg{&Zt~nCwUe%3IuxGHK*j2o7dzXSzvM5L^y?XTA8i7ii9hEj?D>VdZrAt%i zR!2hn=aan4CC444Z5|x#1}X;&xiMg<{}1kATFa8YbE{y$EbtyMv=)7=>E-KV6H$-( zIIU=L-??kC73s|N3iL68sK|VrR$)m)?izu* znu%6W7kKVx<^*+70a{n`#JLrbP=8Cm=g6me4QIRcsVD`xwKMtL&TP{7SF&$3JEYHM zGYk<{p4PumyXf5VNT?^bkVuo6hAk!~O0Czo%$PjXb4Rn-$V--vV0bd1Zpp z!rkWgY>%84hUMP@T5JzY3wN`>j9M7R|7Ora&ngoP3wO)Ej9RF4MBNBl+#Hw|?#6!^ zwJq>hsOvzBj=;1yUvt!Y*s0tc4UO%4aG~eI&vUlim(KJL#}8WmZ)SDuXjiMZyOQDeuk?Y~Ud@{~Z@#lL@6CHN z50X3_Ya_cq(#T;>2j(lQk!l`xzK{%Mn=2TibXY|YV*9!w_2Z#f)`b{o&Ww0^ZY999 zla44j0oVzcbeY41Is3;nT8W?5(LZ#v<%j+-L}^Sq0F$5@Oaw-}qF@5&``Q5$k}k`{ zd%~RTV;Zf*MC)K5rhDm7k{{{A5T!9`156+tk@5G0XT&QCCUAQ1a=@hB945@UJ*Lr0 zOtg;eVY-)MB0sQ)AxdKs08HAxH!Uv;XAj0OV$>?PE7YaeQO_uxu&=ka5Hk|9#?=(~z&Iqpu@Z1Lz0|^vC+zVQ~(3 z1-g1;k)B{=XHT$(epMrVMZ))zU)4asUsrgr4~&cUp_(WZWIRim=H;kbYS~`!EA0m- z+L0`WT@LcWU^E17EwyIJ_P8T8$#PQ5LJ`>Xh9}{Hdh)ZOUb}LsVVA)T#`~Pyvi0J` zTq7_v(6K(Vv7TWH9aGaW)X^K1rDN1)xwNA<+;J#Uzdam|Cbbt@K_nTxu&4d>UI({{ z-^9WPKGynZZ%W^R-vT}0@XH{zbK@!qf|kcEs8dK1?;bjR8Te>dXE}r02a_$5a3tLv z2?UP@J7Qp0?xX<{OxLHr%5=A84HEC2k<)t|-0l^-SpiikLy<^Km9z9k9==i`iR+Hz zr!R%Ew>oR=fk)Y_$l6s%+_%L}?*G^Su8oZUIeb=ZJ=!Yf9E=0N9_NJmPwpdgE7y8H9rVD zEF@E&iCr(|+$1vrD`#k}W0PoDr!7#_tN)7ew{CdD2)lkn4b=r!K9H9 zuPB(n&AnaJ{^xGzyw#p=_Yv2lRZmwnIoCKw?4P%nRZgrpSbog*SlP2>E!K-Hw?eVE z`y=)alPg=&^W(f>^wQ zVT<=Y08EuBV8+4X6~N-X7JwlaubKfykHsq##Ca_kws@}rU>v4^83&730E_o(0LE?( z7+#Ob%NNAvbFMJ?fed_WuoF|Gx_M|7BqR-*0=&wgBw^yDcwRw!U+60E&}_;qu(@ zwlEtFh7n%p)xO9or68JH#VNyc99(2eBr8oUBJiJrl6Ti96T60K7_HMw?(rQHHw@Rn z$Sz-IBwkkU4ZfJRvAJNe9h8j789gyP8^UGh8wi)pnf?{ggw#d7Be1fPTWU>A;66QO zwXYy%wa>6wJq#+(OfwU>Zyd~OADGpr0T^-(tqEWRJ!Z8~5VKk^Y*v31fSF+i7-1aD zY5~mZj{q40to{H1Guad{<6u?`U{=2$fFY?> z%>bjvtQHDlRttvB>YD+WiKc)V2eVoLv-&0ghGb7Q1B@QCS}2HFEf_Yd-v_`VbD=58SkAcrNHsmL<1_*+ZI`4cGJ)Q+X3t!U0Xir(u!4t!G6VOH31~ zI|H(HvI#8Fl#0}qXi7!WN;IV+r7fCLk<1oNsYqjsrc@-bMN=x$QlcppNo&!R%5=48 z(qy7qUgxNl{AqL?@q!^*I+A#wAzC_?_(nstbTsizhG^+{;+qZ8(hapSN6s*du0r_NwUxHFi%E@Evk z+R?Ku7K{a%6Ck?pcfwUF>uC&u6lNjy^j#;1Arp?`ekV*dyVmBo-w8)7@&E5#!;lwe zb=_(!mk4)<^|vk_8~;M@_O!OC@HI`rG=$$u8NP`L!n4+cpphb}q;+J`n0i6<^9XG&CgR zC2sE+hU_|Pns%tM=nHiWrieDle#bEh)-^OIxqPu>7*gxxN2d)uYO|R+7 zdJANrNQmFIX&6%F_ z<+>?sHMww1t+}}fq`E&No*F&}W@1NEe7*jgTlFT+K`Zz^`GwLLDp)FI^EZYnFvf!}Fco;hExohuvb$Op2IVNDr)q|6$6C$nHJDkkWbCh@gV!6E~ebJP%NtuzYOD z#8yFnxc^viFdUF?86=<8v_aPMwr(k}p<=rr^T3d*47YtRFs3Jy%>zSPa>9W8@c*JAuTwps`pFV~+6bM*40I8rx8UQ>unc9dUN~?|E6>Emqz$FmDtTX_aN~Gpz zWYcp41ipU3^9Z0p^7)uRg9*#cv{*%dC- zRp!uOVsSGqR-&O5hl}Z0Izr`PxEP`|8s7mlR+>S>pOLL7Xg~z+w*d{3qsGL$!35xD zTC7AvEBF@Eu@sGFqchcmfW`_lXf$VJD+(GAcKbho#`)&ZV4`j_EmoqT6>*D3LmqI8 zAxgg+4*(j=&7jegk*z3bK&4`?hi zgGOUUwxXZ`0k-!68t0lrgNd)rv{;FTR%|V%V<{T)&{_;p8jWuP8m(r~@MUBx3K|eq zdoQ5TVh#=Fma0sPm1t<)P=)DOiiZ4xDhyE?jc))NelutY8QF@02HZ{cbwH!p92(5k zRGAhl(a^e<3e&L^4f(B97@{;9UjsCn%%H(%WGf09a3R$_fJUP^G?;_4nHDS2&^k1W z=~#+}{LCzdD2>M5fQHWu8VwoQih>3lm;EZS|J{~(ocA(MpZg{EY}d6_uh=`vEw;^N z8?5(QegR+o+keEy;TE7p#ZaVYyq|!Z zbc>ps6jMtMHz|hHu%LqH$5$U1-b`PT>$6@G4C8b}wN5vw+$DqbU`!&1gzRY%`it;SxntDr1|`q{-N3d7L4Jmc|OoTnrc*`fkfy z3>X?|u}iraesqmvcoSJZm)y|JE<>2Q(r=hN06{hIKyPm-&L?1+lM;cbTH#bt}#6h_hxNWxi{|j49c;rd6kt;&a&~0?*M5Mfl>VeAlFd{{sr{!bEcRw97a!)&+G>zOJ0J*k8Am`7Px`@bu5B~Rn z9Le);8ac)jpDlwDIW2!YzWXV1&F0>@-vM%gLLk?iEp-u*1AqN*0l5Q(K#uX+XUm{O zPRn<%6FIpDA5WTo=Y9jo?Jop!P1#Zx5jpVfKL^O|D+F?khd)~eC30GReth@ScTVo@ z$CIX!`!yi9w-CrRW=mZ}9{ zg+R`iEp-u*gP4J50J%#GfgBS#kS&7}Ij#5seD_o2=TLvX^TCoQB?x)DfBM$JSY2)++!XIKYT4^`;o{+v z-ch+D7f;K&bMbWPA@Wyo<#5=^ot!Dgvd+GucOG)McJYld*#Uy*^=pJ-lB0L#^7pz~ zCltOBX;+j9XD^azZ`xPO>rS)Z`Xc+{s} z$2Gp4Y|bZreV074{EBEFhRo_i^9=`m=JI9h5kndkFUKvzJ9im< zz1w&>lDtj4!Qc~oNo*eOhE6CuES8yykLDYsmTSKrx%}R`r4~GY0{g!UT5j66#GFUZ z7D8$NYvznXzZ4WaCUF#+G;Q4dQS$#o{=ZqC9(RxHj;cGHZ#v#|thKMLys6@*@+WOi z+UAwbv-Vs1-#NMe$d-gCjFbU#mcHzm{M3Fy{_O+We=Vu<(*AR+_L{08{#*?N|D+kn076P@V6!W5^)+BgsfEvlaYZ^7i^UO-1L`}=@ zoT5ddCigqzNzGoCb! zngviBDg*joKT4TBH!DHKdpq9W^)||G#j8dJtNzJVBjmMgHE? zeS%s~)g}~sX#?vYlK(#jIRHEtyDxUVzv_KeWzMr5$L&wpUa_3Z#fswo7gvsW)jZ00 z(Q|l|@f9R@zUnfYt$12+j0xT;*9apX7#m8DGQOfhc$D$nB$2Z`$~wsv-Xfj!3Xif* zdWA<>C%wX>tdm~hQPxSX@D}N$S9p|l(kng6IxA4|DC29$_9){i=|Up&DB~&Vs88K@ z>YaQ|$A}x=TBS!BUt{{M)#Xvf*RSv><4IG(vL0of^a_u%PWsXuk>yd=*?*-u!mL{3 zog~(bxKupKcqDXslyy~1RWI*EaqdVJc$Bwgd6e;0(Md5?JhwhMw*boGXhWSzuChoNqf?hx`;i>F5H z(AhH%XZw@!tx0OO@^jKDVrh&h?Lg!eOoR2#%cl_CL%Hgh@RI#o6M%#C7_OdnBxaFJNn?NA07^#Jhn-CBuuu)rC z0?#=<@(ABCM=|)Ns+@(I`o!MIS-dH8tv->zP_%RaVjh zTn{3UyjBGhfge8+cu9mMAj14YiqJ5AA~Z-MECvy1o~l9(q^9u`p-B><4n!bXxeGRs z8pltBMoEN4Ai~*&6u~!sBKRZ`7J>+K3Ms<)&qxbEgqlK%F#a=AEr>9?kRpu#jC4*S zHZse}b+5U4HGAkFwQ41oX0_#J7p!hIP%b(cYL7)j;r>WXTX*+hu)8fP<;gR>c*#$J zAsVE%{y}a%)6EbHP=~Jobu<>BjwUIx*gHQwGSk8BuiKAMXy6hmNvYWxj<)sIY~Qj9 zQrCixlc!_TWYU~9FFbM0eq;NaTAZKwuB1GFi@&I^-fYfXPRT9bUXxCTZo zeJciI9nw0QTI_*( zHFLXdQ>FO-KXTsXo@4I2s_u3E-6=SF?VqXqRppr#JHhII$Yw1&&w7RBLGF)5i3J$P zO2u6xtuPGcc8ha0M`&^%LfV%I(6@}P?HOGv{Y)y8OGl;K8H#Fh4qyFzrSW}%xMZZo z$%QwGbMq*y>J6gv4UsVUu1UKt8O62W4PGIx8u0@XRfokYO(fE;RyAF_erGTe4fS`c z8~mE2IVIAhN;8zY{d|yuFQODlNB^e-Wh+q!G0Yr0^9?3U%YZh*BgsCC>#Zgn~(-KDQQ8isZgzD@ef?Op3Q_j%ooZ z8VV-G+cih|L5igXlj7}~qnbgA`hrREcFj>uAjOh`N%3~gQH>zQ;(|%>cFj>f*r}{@ zazneXo~^0)Y0j3rgQ4!8Xt1+}c;<}lU24{!(wa0Z92C9Qt|h7htu0Gb1zP)+s0y?; zD^V4sb}9wfWnTotw_qM8_4P?j>&~fksuj*yOvJp9ETN>+j0X`G7E%QD#YwGIrbS4q z&J7^KfgrwfQRPz5HD)-Iq39egQQ>$*Rn&Z69xzzE9W4ZlPc4y^v z757w3EI-F~s_gyN7cKv??B)*YH`~R~$TsLKcc0jnbyY^VJ-Pp~Js2MB3=Xb}b#;+Q zfhZ(-IMzmXf25JaoDR%aRwLCs?0g{^$~IRpMCq`K0L1ofh18FSVp$hrpgA+*>A95v z(@r{~-~`|Sz+{U#OqjEOOrw?fX&wDTH(P$_4?~p3WItfC*$gHEBVJK35d`l(z=WjB zGVz`;C;OO2D>2bJ*oWy}I+Wx``Y=RkO!fjMkdDasd%`o~6$KMs@LmR(Y%qrjb8e4m zv=S4oV|$qHrI^SM>|uz~nCt;eE;fTn10!BhFoAf>OT~>N>j9JU<>DSz`Nc@9`&1sT zQbX&7wD+_7|fTn(TjB+wu0Ylp=- z+!g5RjYWEbk)1uk8v0d@^c4x;OMX=Y0e@ZL!9Fl9+J|bQP>}H~Wn{}ywbV1az^}9x zoM=a~9CkU#2ZPZNxV6-pCEMeU)FjJEEel0p(;J?I2kObshI;MFrH1Vk9V2U;+_LrJ z#9SjVG|;g=v$39G3LR6^G1So;l%-?TX1TPZH{5Y3QolVMjwZDiT0tZkys)Ev;PA^JwR7Vt2!fW!EvQpS5^ozCSq(ng)mhHq_Q7O}BpgXM zM*_j4!HyW%l{;yG1k?4YuQJ`OS%buTb7W+dgWJ7gH!GkjWhfGfsdARS$ir7EByrtz zd}Jkzz13M`4?N0dMb@rD;=Um^asjybS7f>P)k)DpYmrNQFFrZ40^%7~X9ZCZ=eAj~ z^VA{nzv#%w`7q8_WR0^Nv}nOK=_StVPL3>Bm%O0>l55o=@n3UfiCkwVLz)fR&j0lGq#s)Ys)UNe!=n$DE4ph5$BI|LgvOpS(zI%>ocR^D)o2X#}p`v z2F!!`J}CB%kYs$`aCSaTd{gzm<6*~xA>R+ckW_kRfYD>gYaXn`zodq3`91*VfGJ?c!I&qfCwpOs zg=ESzvFpX0n`99cTw=;nWb9XS) zXeB0ER|jLdmtrEnG#Epa#suOtM=mvkiJuX#D44+gz+Hd|$y;Xv6Xs@MrqN1Fv@Qn5 zbT7rE+0afU%w0jiU94tR(d#vo)vKH$_mRq6N z+x-Fkom|;!LVp+ZSiC|(EMCE|#d{U#?^l@7--U6ocm=R{-wnW!TUpHjqsQVE3S#jJ zhArL%024I@%s5!Q0$99p0ES$=Y6ciR7Ozkc=e1zi;uQgyK~untgT*U=#rrM*=CCKr0K<=i)ysp~dnEu9HVKR#yO%GB;maGgd`|)}{ic8!2h*1a z+xG+jL$X1dn96uP#xGwG>z6ld{vHQlAT6E=Kg2lLzdRVg#{if^CV|mo0rLehfqBC= z@DKoV&=fG^Ut)AW~MSd zwlH51W0*H=4aWeOE>pmagE`EDJsbsKf+m5{6Q<5b=1i(>692EVhG^5jnS<7_~#V?0f^^vN_YgBASr8sQ1HS?cq z3Sw6K44c&-66cRrn3=$R<6u_%z^wit07I^!H35vE$E+3#Vpa==&FT*TFg7#52;*Q@ z3t(2iAAlhlTg?EY$E+3#Vpa==&FY&07^^8@#=)!>z^uLrfFVg)%>bjvtQHDlRttvB z>h}RKoGD<&!K@a*tiBO+QIdDn3^01kYM~%zwP4t+elHjaSDFH59L#C~%<3Be7?N7m z3^01kYM~%zwP4t+z8-)%VG5XWFslVHtFHrKNcL1S!00ing@TyXf?>1zJpjxxQ^1Ua zSuKEBeJub(5~P{|Mvqx76vV6+44c*005C^Q0W%IE2T{T8pMsrmIDh zCKJ{2I!CSKPov|A7YxzTk;MB9(bBQRHyWa)qls@aL`%mL-)xAMjws%5h?b5izQqtN z9aVg*AzC`F1idpF9UB6jodiSUN|5F!L*q)I<4Q0zt^_)+1ViIWpyNs~G_C|Xt^`Bl zN}%IPFf^_NI<5pm<4U08N-#981UjyKhQ^hTjw{`?kO4rR0su2TO3r`DUwxX`|6cAi zSNUXRZNfjPNtXC zL^^tcow43v>PDc*P$U}c3k*X2p}t@s0@(F;Ms|eyV!iZc#~P_XO|k%_{HRHmscDPU z^aa}@F}TdBCfrZ2kLn3WqBVVO9X+A`AdZvN$lJsjqf?#SE^)#xB7Qb$ECRCUZX)@* z01Z^*A~o-Gmybe@oz=c=M5oVgTz^}CIN2qeGr!gZd)o#g!OjKQ#Rozisp3odhK7b@ zyu@caMj^Y-nx-9UEc!wngDIj-vfpt`f^`keNiP4{F$$@5a&qbI4J9Sn6w3UL=F*zv z@*jIfA*s%?I#pRxBTO1-*t&TkS@&l;Mh<(B(dgT)>28+Z6msiXv+debPR%JKXBYQr%af{VUZ#|8(3{m~loNAhO1?B>vNQhlG=Gg|H7makY& z>dSRg*lKd&m|Aml5lD4^OguH}g_+pV6ko6Z=2pFlb1*9yovwWH3#BnsY_c2-g*1P| zsn=0)>8Qua9iA!fci1h~%%q5^h4jE$_#dXMi0s}Y45?v31@FjBXOFr8wF%3|hD>Y~ z^oRS81qZ_c`IbTQSxp;cJ#Xul@)|0(3o;K3smgHM_X1;jGTA&Zq$QUO`FbY$kWr}U zNGuuhsf!fzbQE7Q!tk&dvmYv1e)fnF!q+TL4l)jWhswZidW$P#Yff2`Ao+YupuvRYW?HPGKNJEDc|qyN*@}V&1mk`S(5N$q1`~^$ zX|WOwtvFmv$I=%}9)^n{N~3W&NRAp4?*-vl%km_egCBU@3>fUw(p0gYO7XfRQ?nHDS2(2BT4qahEt#So?6jc))N z=a@mGDI;4^(12LmuLBzM&7r}B+Gbj;L_;gg7Splx-H=DwVu;dcd=1c;X9kVNjBG_g z0|IRC0W{{CLxYL0&9qpFhE{AXrei4@^3YlgQ5ucA0gbcGpyA8NRunWKs`jgZ#vF5K zFt=1?TC7Av>xL>!$5J%p7gS-0(rDZTXw;ZNL&(Th6g1#&s{aNwW}8EUxtc1|VkH_{ z*HU3RmZBlQl?p?YM&nLEW0n~-_>62tK?5$N`Y%9Zra3g2gR_|yE78z8G>hq2iiZ5m zEQTnJ##aE1v&^8;kddt@XuxsVJBa=7w#?(amwEc!FS%#CuC02--cfF`Z7$njz1Q*! z`0C&OBQ}nn2P4*UShQrN^9%O3^`#;az*`4@1Y+TVz+pu^f;5t{i|9dBqD@9UWLKmz z^abya!~>(tpxqOei-)s3KT%IAdai>TsvTm38t}E{ z46V>ph18hGz)ji;-K}(!VrC7!Zc&fV+@vjPZcv&#^suJjuw4?s{2JkZ-4it`DW=A=Yp&nKWM73UMsl%;o=#3^PJVx`9O zZ}bjlJbsw*BiER|2lr+*soa}|sYmC%S;#6S<>|ZP#L-5$H*0;?y;+z_sC=1`p{7(D z=rAw2H0u$#G|LBbm-5mqOvChEnuV)VrZr;-O2d{tY0vu@K0$WJ_H{{sr{(9zcRzjSYq7cY2 zQ3lyED3Q~OHNba2MNS@ZfG15O_ftUb{6ZkdXG>i~wO^EO3;sY2SC`b&K3D9NikxhdXf)Uwy>!o|ZSy`yqRE}oWk z=i=$oL*x~4<>;kO?&M4{mUZ?Oz4MU6wTo|z$qo>_FJB{!k{rD=m%rD|I-&55NV}p; zID7HjB$2aRzBD(c68sjOYH7(^je(9Qy(M#a1buYQmYx=F#c9 zK#uiOyG~^#c0!L9Iw?w=wf$8YS028dv~udQeo@>%O0uj^SY|xxQ?KJ1-%d8?lfJ%7 z9$9`tw2y8FyJ5cJpwC>sY&~K~qvH9vWpo>K_KcW#+^E?gDSA3BNKNlHUXCPh6K^p1 z1pgp5k8XudC_5~cnTn6*8>E(NzaF{#-nyk0yuSzge+#tSv~P(ykDe`r(*D=X8HIi+ zD0ocbC^Tu>xci;t|A+j4vphZS9@iaJcR1g4yy;kLUt4)o#ZBc;+Mcw{E1PHSxAebr za{rMn2~imB1>`J!*)jQe+n@GdORBuIzo2KoDMguomqFqrRNKT8qa;!8^l*0kfX=!Y zFLa7lWru_^*78B8Xpu6=eb9K))M%?N1Jn){0yTe%dC^hx3)NOYjbza^jT+;DW+hP0 z5nBG|6fF|9X4(9XCrzVf0n~a5fm(BldC^g87OFWwjilH$jT+;9W+hOfrsaE1(IQck z`=0TnDb&1g0cu@^K&>goyy&Qb=lNfN8p*$F8a2lA%u1j{P0R0`qD7)6_dDZB)2RIu zQ0puNYK0vmjoLo|waW{Envh~%bkxAx{C7YtPzcl* zZ!;@_5;ZMfbBY#;n%vinCrzXFH$d$`AyDH}%!`g1c$)tTsO>KVYRogRw1M@i zjbUjem@dAUm+ zPYaGQp}HD8%12;qC_T#fiVERT#&eTI&hjYhBv*KgbkZw4$~x&49%Y^M3Xif*dWA<> zC%wX3q?2CZQPxSX^eF4BK*ghsuOZu`jHjdviOi#nr=+7kb>C^V7d*;Wz+0>IDC28P zzqPtN%J}*f9%VdfN?6vTtdm~hQPxRcnj^A2$~yb6G)I_KYpOlqQI4s2l<`RD^eF49 zn5tgYZgK8t6g^ADONXA_8&3DpjWB_Dt;DC^7iChKlUiXiu^k;|8yqD0FqRNKMA z4@0|6?_4v!evsd^N47P^yy&pC3e^>WT3;bh zV@`Cl5-8s{t@GR|S|n=BZRqIvCXHG-vHz!Wt(@n#o{jGBxi5FU=DOZ>cGcIb);S+{ zUhep_`Hi;k+U#X(tZ~aXxi^78TrpM+9XBB$PGF<9vIL-$LLk&1iADkg zkY+8`ABxte%FyQlT0~e`1F7Mn!J|=%tZQo2IoC6_!m6yK$#wyVK=N7@Oay-X zMBpV6YC!}~Aw_5yKM@)v5zYY-Xr8J<4Wy>=6QM~GVLpgJvT_$}AT^Gk2#t~m^FV~E zLWt1v9YWC1U zYSl_E&1%cdE?C`apj>n?)Ep$PM(fUlSy;d ztngSlmSb-uCg-?u$1W7dG?~P4X78BI!R=kR7a?Q&pq$~V?I?aJB8lBu;juETfL#NO zfNQtn1Wc1j9A^xUSsh$pNdO^WTW^e)i@0**E*>RhhEQU9=HQq`2h)ZurVUu8NkWO~ z^o}vk!R?tr^DgHccaXNZw>JQ&w8tVtfFb#TD=cxD)-`$x2Fy(D0V8oCV~x~L#wLPK zs<^~xYS-wMSVjbM!hnns1fx`OiP4nM=t&rpv$bDyNg@g~;XsE5f>oM8;x#!mdO`=U z1}YWs0?&PZoPd`^An}@Xa`d=^>-KePj(kb3{&0XyMIDe^I~0v<4%Z}qtLYjH_mSBw z&k$khCH@n|$)m>{TwAye)qrS#HYM13VbW}^=?_P1lFt^`z^J8f#bB&MT1Qi>Z?H3O z+tO7_4Lbu4cMpM1qCDK4sx!Z_t@m&@zGk{=8}~$;+ija5#sB}2^Dg%sbKg~Uuk-It z!O?5~Oy#dC&#c%9R{ujbYuS0$D=ZIke=JHYz&KVa?ixEAhQZu!ajxbFP3}WT`w{{A zmeI96qf4cqNo8{BsB}9+QBBU_tDmnlzRwkxjLmU!;Z5S)JPNCNgXnxiBuu_*(ymKJ zaV>a*Tg6plHGo9bVX;aRiL|R#Rkht2j6_5I-RcIvCTUKIG^y%S45e;w0VzmcnZih+ zZu4ul$D9;N%ij-D%qo}^>SlitrAQk8%^(HIDpMFN)GhxaN+C!istKext6);78~;U= zBDpEq2vW=_m=x-BYZ0YL?o0YWiZcr)#oIMU2_VJvf=Tgq%~3o^F|A-yyj^ot14uEo zU{btYbJS9hVoJfJc)RAPdXQpr!K8S*=BOpGQ#r}W4eh>qwx;5zIa}@yhPr#A!Oj}u znKQO`sabzYYtq=bMA2*QTB0h@+OkAdptWC#sz7VA5>-KJr*bjuvQLEJTQHB4`ue1% zb>~z%)e2`UCSqPlmQd1ZRtF-SQAiQg7bmq=nHC|bIxhkdCKOTx^_8h8A|&GSSJcthk*fi8garCyeeJL~hr0q@y|GA7FtW2JSVO<6k-j3~ zd&#eAAmFbnJlF@uMf*@q6bdq)rA+g3R4uh^i#T~~DLB!NWI60|kPil27dFf`DyKC`i&VG12n(=pW1 z8^yZy{8t?rtA%m4B5R!GphXL= zNiT6;d2;L=b;%nFAh}i@68{xP#^$R_u12dz^4#BYWc=>};0h%9hMg z`J0|9kb2`XvkUXqXNoOYAe;75kCWXiRS!rdV_ouvsu4`3aGJc4U@-2HB}rrooyC%q z>v`Zho(syx|97tAWyJoUXqm)$_jr2U54wKl;;Sxnjyk^NaM;hPj8$A){*3Kq+uE{A ztY5Hv1B(4ye8l-)Jp{&e}ne~}baFzNy?_&xSMFZx+eD4)|$4D|hZ#X-jCcde9 z?03E(20U-rfrjWW`NOS$!i|0#J{A5ZTW5hW}PWu#=)59!J6-a9Tt))&%~}5b8eEEfR!_}*0D)6 zEb;@B7$T}NErcVkLD*qgV`hh?l@YHf>kFKX>;z0m4m=Z>FsCA!Mk_JVIuwcNUaC0C zk3(XJ(wKArCacY0(!z*W6iix#>UO|{WVbVc33GQa(`Y3oT2}{Sx|d=izcd&_l*XhD zFj;8^6F(zfQ80n~fiDM4NZvXVm@qd3GmTbaqIEGarh6$S&4zX=VeSe5CM(Qf(#(ig z6indu-vfXN$w+4c6XxDurqN1Fw66U{VQ%gtcY#E4fEOqzu1eSpb% z<}hJy^<^5Z#6;^-UrhJX_e6f3FNP?M$zH%@nHfwP8S#pONuyAG8MXhp+c|Hwr`vtR z^=Q@8RZY${juHFk?PZk{D-M<)vprV!Y*~x-BFn8%?Ct)5{!XrJHKD%?dMsX{AQrD+ z*y23}`g_2P{w|Dz#VdftdnEuvZe=wCj2??uD2T-?7`Av%0x^y% zylMs*Jr=J}5a+dE*y247!0a^z%s5!Q0$9Aq0GP|n0mJJtdHI6ayu4wfcL;#lV+I(0 z9IRd*%-*8_%%vuQ(PQ`W1u=Yi!@*3C9t)T+hzZOawt*u6%nnn)jDr!(gB3goz-%`O zj2<(XFNht?8#aUw12Ef60W%JkFb}5i002W$=9!tw^w`3DL5yMEur(Y8V78b7W*p36 z9_-0c2|NL|!>6U-CjmRb`NxKEE+?JI~`?K5mvzYiwxu$c+mHx6dC z56tQt0T^-(tqEWRJ!Z8~5VKk^Y*xP)fax;>j4%#nwE$-I4FC+u*lGqCJ!Z8~5VKk^ zY*t?nz#K9K%s80U0+`j;0Wc&fs~KSQnAJi-%xb}~S^XXWCS(ejaWJa|FsrWxU`XCo zGr;IEtA&D?)q-KO`WgVH+Y~V4U{(uYR$mRkkkqPXfYD=C3k5N&1;b|bRRB!T6fomp zRtsQOzZ-xd*;CB`qsOck3Sw3ZhRx~(0MlU#m~k+x1u(1Q01QcxY6ciRX0=cdvsy50 zR*L{kn<-$%A%t9j5b}2s``=P|I|u)jKBXQg^+2fyNtqvHqA3-rE76pSq?Kq&MM_&V zr6QRvno^O*7EP&0V2h?yq@_etDw5WsDV6DJ(WJ>lwY<(zEBVvtIN}9Ev~(o#K0~y0 zEb)zoXz6I;n+(y?@x(V9qNO8>_Zy<6V~TGvL`z2%-)e}Kjw?a$j7G&+jw`{?xDx2N5)6$ifsQM|(6|!lxDpJFD}jzH!O*x8=(rLLjVpnU zE1#ip<)h_S3l|zX%Om-#KS=CF9Q7fNHOV5u~;Qo^a%Tg0VfN1WW@nc{wj-D1^e2$LRI3;*{ABN5rXN0=s4 zjnjyrLiLAlI(zI2KyAWu7#T1q^tCy?RnQ;qKNcJe2jp7@$!9ffkoCN+Tgq#w*e=LC zFr+HOZQl!w>B(gCz>t<)GUV%-=tJd^+L9rkx=1lkNAV>?9zDJBB|{$FS0zJUb}?(7 zckutBA+J+CpwZn+^sVevrGUK#Nw@nDvHvG{k8$4r_5Q{Cg7??nC%un*AMt+6dyn@H z@8`Uq@SgU*-+P@m4n<0zQV*1Rpwt7U9w_xdsRv3uQ0jqF50rYK)B~j+DD^<82Xyy9 z#RMB?b$CBsQB8h&{-eT6e!9O<;UPa=U#W1DpH+8PxX4fEH!G^hPsf85PV&?K-3kZ! zS@~Fno&2nLqN0-gEdOOi1^H=vuA-d$EPKAfMt)jft|%ivEw5Kt$xrSLo5jXC9Ock% zi}wxkZ!`U&TaF?UJiXK6<{SaLg_|LMy~*l`g!%@0gMmo2EgGpGYU}H*3HR3oc5Gd_ zedo$RcW=18tvApg?hFRr3df@L9g!;{9X-L$SZ{E!J{p4LIg2xY4}?0xkzhx-zf=BH z`J^w@+Z$4Q+%c#^-{01+@-!BKgh_+obqn=D^=d zsHihozcv`{=-Czv#)8a#{jRk-iUtN?2ePdl(k(`Z&}7m^lFs7BIq~{9F_#JIBNcQO zH!hP;A1CGnls?i%oyCpI)TECSa}H7;X|vAa#%1#B>a?nKZ> z$~WjOZ%C>w_!*DA%>>;#y4{X04cE+|%ckA!;AYoPB=gauiAhj`uEIGm84UKf^#$vt zAAwkSAaGcFEy59);HFrYFdQW!vs~qC6q;+Ul?q*fe7(XGnUz((cA>fETEWm2$k#DE zky%;gYZ{tsu9Xd4fqZ?#6PcA&zSg0+=34R470A~;Jds&h-Nm>Yy?q zEweUaSmsMJLvimde$OOgY_bPmMSFZou{RJr=pFRBFE z+`!~-vcNxrRwatnG{rDKkv8|9ycNh-Q#99Ht0}qy`D%(MGApZmHAQpHwVI+UkgukA zBD1o}S5q|CT&pR%0{Lo+Co(Imd^JUL&9$1ME0C|Icp|g1%2!h~*IcV9x&mf3)j?%K zT4puHu*|udVu-U>Qw$Benqnw&tfsn1)mKwY6_t*g7R?hiRbz?12V4AaguqL(g4~CMIr=MVNh3wkn#DSi1T-`IR$aB#h%tfbds~ed-P>x!s=b|_qt&`y?Pbs;2|xNPl5#5~%KBcV2G9`dqrL6q|3D9`gOT$wp@9zB6i1#&{Q6^k?Zmwv zjD|pW(<&0~3Uu|xB0a%KR;kEBMyGQ56pSfuoRfT7L2$~on}Q&f zPeYjE#$}RETL@0Mc1;kZ@@WoJ+_+5gX%WFG*X{^{RK72QDQ;XQ`CbTuQ?6YQ1gU)g z15@0%O!7Ss1gBiP9SBlpp959ixKzB}X~M)j!RwqTcqV@6|wN6Z3|kbKYR{KI@cA=e!}9m^TES^M+tz-Vk)o8-j^>L(n;I2qxwYLFc?7 zn9wx@o%4oZV%`vR&KqndBYk7ur*qz5(^;W;F}@USwI-(*MVpM8iRkvQo)hhw)0;!}^aR86_K%9oN<*}PnHttaRTUvkh~bFDk* z3gn9qp2)1M@?{9kHP>2%u0XyZ;fc)3Dqot=TyxpO=n5FCSO=8}X_*xZ!!qZ(g(1#f zy)ZQF8it|Bv6AT`RbS6ARb;JdSdx4grX2ev?|*opz9T#sY)G%4@F($9fFq*h{Z9`! zsZUDY|FqLe7A~Q>YO8fvdWgx0<(7nIZ>tfj>Va6!s_Gn;SXC>-600g#SYp-QPS4S5 zwaDz7bPsD1l)V2C`i1Ht{a~bg+p6PUlV^%jboO11sAV}r=BADXFdr!U>g5Z>E7X(2n z-~YfAH!hQW&jZ0J*KP-bl-cJ%l{YR`$@?z_3YNV8%xXyI^^*5LIW|mq@pzrnM#=lH z9ETh*AUNf+LJ*{KBLq|2xJ=C61%efyD*{=YmW|8M zd?j*6WN4BpVlgyJXGLLXmd&9{XbYSs$`N^?$uT?TB5&aw_+u_UV)xNe7Y3T(M(1$3 z>kZRHiO4Khx!VrSHCN0cx&mge9nrheB4<`s`PzkUYOWOwU4eWZ!xNd6RlcU7x#rxJ z$3i4u-_Trht#arJ)*z0#!WD<_6~*7}Jh$Ws+-C2u`_9BO^%V8Wg6uahc>=6M|F8`cFjw(P1ms zh!DM(D{=urD%XNA#f^^$x#ol5lq+%pLCR=5sPcxSVl*6WHYMvnkfpt7_ZKOg0~3te z!{{wI0u$U6>k@{e=IVs5K)y!diOk9>U$4+ybFEtF3gl}Sp2)1M@^uW&HP>o}uApT7 zLxztWsXe3x(k!m>bq@_o$@(WJx-i|vtc^GV6Nr|qf7K*78mlPIm{&XElcH4JF35pl zil)93?0+Az|2>{7Iqy%s-|~LRd#g9*z1Z92J=PY=PWa3TG#Zp4Mc*SzWRn{pRXk{#|@8b z7g`5{1L46a%$GmCI?6+Eq-m`%6P3H2@USpxxZCsPM; zxPLQjYC#5|!Dt(tq!|nbf=Am(a`|cp6rDcVdZcZzFE#-20l^7&(#FYFh%0D^6hh?D z#7cNHd$Khe9PA7Aw?%{G;lc_+xy}mNX+q&n*p?j{2ric=gr|FD!NKL5cPuD}W-jp| zn&Y)W$?20bt*I^}a!-~u<_s%5nLU|l%K{4_TZd>$IpA0BeGdM$obP?Z`?~jK@AKa0 zyxYCYy|X=UdH&*g4t4?V^$dG@J!?E?xLRMR!*Qy^> zja0>|R#nY#{=s>l^9ED{Vn?=_ECGM-Cy~y%Eu}{QhBIS zs4TDeUd62yT@|erT=|d6PnY+VHL^-?pYbNDj;v{MLn+A~^_Knsy^OFoA`e7hZzoz;9~Wh2+2l>TX=P6Um`r1HXe5 z4QS%P1}0oQe>)*rmxpA1^)^y$rp28a4C|Y=k|*7k25ILr$3}Mc_S{7aw-B-|2(opp zn@Q1do}y|&M{D^_gfxI9g;1w83pNsxO?gNlp6bFy8%W7*c}mj#+SR^`N!gA(Wm(th zs@nC0XhUA2l`R*Mf_-@k(wCHtbGu;iIzqV%N4dhkmXtn(Db2XF&#zlU2zQ_em-{b7 zb77nm=hdx7b7@)TUxntvgdCh(w-U|8-|D{r&4mdqZK+#<=F;ryT16HvT}uAm zYw@lPLK4Ysv0w~bq_Vq%ve%OZbL$DgYE6P#-x5-&BWEGCHRPPS#e`@_ej->s=AT_h z3a`vr7}kw>^A?dZYjT#ME5_U<3rVTnIZLr?#o2Qfkm4&eiq8>hNs+dkMO4;@nsd$} zH0$%x%;x8laslmf%$hK3{yajmE)U5}elDVBW*s@(k zEoCw@zkHINJe+SS>kD-ZhLaW9CRL`N_V$L7Ps_X&L5k&d2VXRJ3k=(NHU$7K7;T|2idwZQN8fL$Io52pR{L<{_T zUi<+3yDVM}|DG$J3;(w6BmVz=U`Bg?;C;aRW$!1w@AF>i?J3#+^5BV*{jWS+LSHau z7AV>O+B*ztx?9Qqm+Q(zkaB#ueT=LUlZ7|b>;S*D+ArV z;r6!PK!3P17$Ell0q&2S_hIjtx7&N3x7_oj=aZhJo|T>{?q}RzaUXZ@bkA}9&GiG< zm@DWy&sACV$EruG?gW4TyQ_Muc2zB_np0(WzT$ksd9U+h&R%DW)8_c0W5m(pXmoJ) z@7Zs$x7(N4ZIzE#eyp;;^1MoC#m_1}QxUCLRZ(63bouAYuPwj0+-3WX?Z>t|Z6CC~ z+ZME4Xlt@fDSN%__hk>2{YTkH%C0Soly#QvEc2Clt-rH=+4^pX0+?m_t>vqhQCkxOXOT!;H`Yy7?(^G&;X(CF|0VU1`&_d~+%?u<{=&@8AotltB5~bdxcIpLY!->T z2E&EfCFK5Nkw{!O7%l?$nP!o=YcO0GEsy*3B9XXmFkE=9X`v%-67m37m zgW=M^eL@h4%f`B26Bl+Tk^A^LB5~hX*W@AMJ~l@rejV#FC`i2=f&N%uJM8?0y8>Ok zu}BZ3%)E7$NL)GArtC6PJ{xXfevoeB~lN~M5`Na6y-+G5Qz`Q zI@8L1F5253=m|w5+=yEwP8UM+S$N2exJ2q+0h-=Oj2oFH688z=`5CEQQktq%N)K-M zERi@*tbA61P$UwACY`Plsh?zm)!jbGot`BUZwRUFld83S^emD1K1gj$fn;qTaf#ID z0gdY#h;bj8B@%xIsqSN{)!lNINIV&&I;KFfx(~ZW>b00`b(5SW+=pk0#63ak8&$3E zLuZM^8A0k}3MA|Mpj)I)2&ujX?t}A1>S0)8-9}&TfV|RFmyHMH zMx{HJNt(0Oe)jBy?>cVy$u@FnZ#NO>YL|?)CsXXH*fByr6Tn@?6Gc3 z?af642;9|6Me5SP zQt;eWOGWDMz*01D?>RQo3fVH z30|b0A00G}$9a+Zhjh^Rj`1S#Adzr9wSH4;n=r(S#M7jQhCj-SoKBjCBWIl=Bu(l~ zyJG5<~E5*#K7dEhkjKDVdQ2Kcp7&3=f_YAOljnU? zW8^tVCzYqH$rrs-quF0HP>MrmMaUP>TdCZq0n#NMpp}40Xwg5WuQgkZfU z0fe}8F6brYHffflYC;Eph?InPBRzd6ondUKOZ)tTghV@3b<5he<`5|s(yW=jkZefg z<%@a<-8K|mQ#UD#2zX{fUk)tnB4mh=>el^DK_mwz1ZLmDP9%qxy-giR4vZV*vW4wP z4*oq&ZAcDGNX(@RFGq4{-rW>Ha$rJZE?Ia0$)Rai(|#le#`Uq2*#DPsKjpmN@t*d! zc^f@%K)nBlJUyPJ9;^Fd_prOo?Q{LZ^{HZ2`k2MKLX*N#9R~%>Vx119 zDwDxAljERXS*&x(BiegXN&>q#4l0zzIu)K*NFg@vI4DRKE5z7vZ;~jRnm8yw7Hcc~ z{m&eMHnZZO>sYK!q=;&)&Ez;JHk3Q}PkFe_lsG6a7VCUM$VA*siGzlc{zA6tHX#mb zh{ZaCwk8>OGf4zJ18R3bEwa!Xj|d6|h-Qb!H2UTY5flrH6~b@0H`4()ZV{9Ui*+(Q ze1inxG>D)~SRe`qe|u%MNYo7LbSw9dj-ifT*k<^HM+_OvziSbR4q^>y7FAoix!*R3L`$*m z!I8q^&HbiUB>IbWohAt^-`sNxMWV~VMxfSK9$mn>Ukf5pZu0CHu}e7j>^UM)Zu0CH zZCT6x$}bY-#@emf68gIMn0K6ercNZA@B2k1;e3&(!`$?ThbRiUWSe;1pA`~AV_d~x(R3j80b_@Imbt2J@P<#aLQNKtOBorTJt>?a9C+@@Y;kobm z#l2WQ?0oRuI`J|rp9by`zqkj>CeDIw*@e(Yb7Vg`A zaTk^kJ0Cn)C+@`Z@pJ#<7k6O!u=BwKb>em`pJwh`esLR?4?7>+Ung$G@@eAk^NU-s zeAxNmn|0!5ET2a1Uca~r%ZHs0zELM`#PacRU-yd}uzcA0;A?f_#aKQ9caL9OkLAP8 z2Y1(r7h(DE+*keLIxHV{KDet+T#Mz?!2Nf#xQ3KQ_$yiU;Lb(jg@h2{6>a7It65x) z@@I&MdEo#E=}C$o5f{FF6<<58?pa4bN6xHN4y{N9`s(|b$Wi~`JCrno{K#-?ibwm zxvz6y?q1}2%k_}!6RrW*I#*5AD^*{ux~Xbk)%vQL&fhxkbPhS!JKc_7IzH|=>}Yb7 z+aI=n)V|-or1H;|_g7w3xvO$|#q$+ks5nuvqGDqCQ{eGGRldHw%Jxg!Cv7*{I&Jkf zuI$_3hZ$fy>*AmUAo(=z@W(+3K=NT$neBCPPy&#AeB1nSPy&#Am{n$L zT^y7EBp+dmKMqO&k`J@WY_5xg5`g5xZ}P`M2|)5;R+)`;aZmz~d>S_R;-CabyIERq z8@uClacvwF0BJW%n`-?TaajB<)|q6BDfKq5_s3yfmxNoS?+%B$IIQuKaQP|MEsMjt zZn1`exprx_{f@Q1IIQzh*C453%VW)=IIQs&>!!Rs$$busn<$uiq<$R?-uJJ z908-}oWCRvtG&g#14+BQH4ckBnir$(oVO$nYd)G+%d*xuEcIw!%))f;k~pmVXkPx- z*7&(PcrlAp%aVAj4qnav)_98!Ud#g3yd>_|!K6VQFi;K?g5p!Kz;pU#f!_zoa!@uY(t}Xf0k6 zU!sFoLtRsRF)5#aCS(_`MGNC~gf9O~2p2?D#}|>Z)M1jmBZ7}FB#-vOl#|N~mAyH? zV19f7p;?`WrnV_wOUe8!3Ja+f@+#hrI zxG#1ubh}-DcKy`#HP^>o?{;;&HnjTygS@&9(SYEX}Xu009 z-!c~(ZS+%@pnAzBtJU(EJSPEQ2gjeFnn(~)ni+fC7wQsJC&BQ6Jsf|6Y9Sat%oyZu zt4mOQ1jDD1`PH4qFRW-M}_txHf{1j7f;Huw`%`@ryF#w7P2bqT76VEDl427iKT z9vD81%EEoREx}3NJ`LS4xR*6Ey#mPdeC8F zg6a-4;lZaeCmlQqsw$9TpQJ|}JPE22P&#_pVRnM9_=GoeULbSYVP=A^_I!4U)Nu!Q zf-dV5;QLB);9+)xuHB?Y=EOj{#*fZS&~=*B$dpJOdGI9Y>P#C+4?WCI(1n*&d0Ohl ztRoq>%uLV)msH7=NF96dCg{>S8NRP1v7Ow9XD8?qO6t5twayRCOwi?$)X9`c9ewa7 z=pvb_6AnMrCg=i4#>EDz0p&dYuslH*z{65W{RbeHC+JeBNuLQORvv+vo1hEiaxMPc zO?-kbhw1z7QmKBb^GZ=}6`n%y61fQTwBc28hLhy-dvYg^+;3x#2@am(1!w`JJ zqmKrTL+}Z=J{mX>!6#fe8rhKuK2fEQ1`b8=38y|9I2OSt9QtVBU<9AA>!X3A5qzRj z9}OIiSdge7H0$z?H!jhbC?`b&(g23T;ZmS@0Ip#U#QH$OSkyVAx>M-hZmzJx_L6@R5K!Re-IIO&U@(xl4FA+l~f z;fr6Xo2v1+FMd)tmG77@enK~uFyxCL*GSiTjBKo?_77ePPjI? z&Zv5>>OZT7s?M+SI)Ci^wDX{InUixo=J>Fq+p*aGPy4s*AFy9)Us(B4ErUYG-+n|)j)>`&5$Nhd`g09%>^V4y^ zYfaE4JfvMVy;RATIqtW7f-dpItkY;;niiHh?l%h(bm=E1fCddcm#`>7m4FS}Md>of z{o0qHIzgU|9(I}Io~=z#jUdlP&lWhE(2}6~K(}^d=vtM1Heqpsssme4q}d5o>-ia)C3eM_tTbyq$gmCvwPV;S)4dP6%rIF_v4nteyWjRinF`f zKU$pFhvoza6bO=wQ+9Pz;7#6~nHIJnS~*nsB5&Jp)7PF#%U z)WqG_l30)C#Lf}lT%5QF%?S=Kv?SJ{Ik9uZHx?(>qB+6wg_gt`G$(eB_}b#cg=kK2 zfT1O^8qJBFBko?DScT>UM;KZXE76?TIpVIxi3`x2;1Gj9v4RxGxY!tTnY*(taXz8M zxY*zrgFmqx%ZHsHzEYPs56cG*GWZkAuzc9*;mdW2bFqBjD1$%Iisi#j4!0Bkzt#I! z4s!qf^&Mt}l$ulOfl?2YdZ5$;r5-5tK&b~xJy7a_QV*1Rpwt7U9w_yIsU9e^mT_E} z1^$uj|DSW-KY4#{s>%{^sRv3uQ0jqF50rYK)B~j+DD^<82TDCq>VZ-ZlzO1l1LNNV zXILvC9s&$~Yvm8Q(>;}sai>EQER}H6BRtDIDu;!9^vs19Cx-6!n-`K~9{O`E!N-nfv;dQwQ%k@Wjs8Pu{rqwgbz~dUn-!AHM&ItFC!r_BTI& z^KE}xT|4F;`T6ADH@^G7w?FgJxj!5JV}rBx*9*muH-;Yi_EozZpZwNqn-Biwfvcx{ z|La%nyZzg-*Wa{VJmbh?_rEsu(dL-{&R0G(E7a6}!|Nws*nIstfzfZo{^Y#(nZ#4K z|K?cp@;45?TDjNtk6X5V^NDr0uld|h-gn!dAN=s7v8n5pzwpYM$xqxdq5ht?Ha`EG z_dT_5-ETi}zw`J&9h#0Jrlm~ts|?V=fC&Q-9P)@mb35r!Cjwy;O}?NdhJJH zU-wr*Hn{wclC#*rCvW{pYJ+dQaIG1CRV|_?0(4vhDu=U9sTR=hr^r-sivS#n%_U z{;T&4pZm*YW8Zmd_>#U&zx?WXtzS(DXOJ^iD9eeLd(9f!{Sg}?p&*s(AE z>E@T4eivKz%d5S|J`(Qv`b$6h?D0olY@Po^#ow-s=`qU%;sCm8r&MnJ+6sVnj z@A+T(;{TrdpXWZ>{m-}VeClZ62X{R?eBacK<+pn0-}CZQkDtC~>N&Ijar-sRx7S~J zMeNI$ExjO6vEnDMdmsFN?A-}mjN2bL{@I?{m#kS@hAc%=wi4A?QX!Q}Au2U0+80Wj zktMX)mrFtj*S==&UQ5>O${La)Nh&JR|2$_tgK_y@{@?rh{r~s(eZ1~H?|GJamNU=! zob#Dyrn&BD8CCePVmGJDn-8J4HeKkOd9<>vi4I$+UeW6K*)_&uxY8t+oM{>{Z(j?E z$J!A)D>+fc zQ&-|9u@8q#8B(yYK)(0NQOo6luLkTJn>~8*Jw4sP{P5h%TL%Wb;jQSJ6u)iW$?JEj z;tQvb>sM`k&fca=b+cgIHg->jKRp{i@!7kqZ3WkFr=NHwY*+iINln$J^Rtcf&F4DK z*6y?R$m+0&Hu3w4v~_Ou`((u|+i82yi+Q**w?eb3`hNAvm}QlRH+5d9C+}A<-l=Z# z)vVu3LmOLfIaM^bvhU-nk@}?{1#|MBmc1A4d%oh`s9rBE2G$&_xT=|EKDRnRIg4&2^o(^2J!)>V~aXTba(;;P^bt{8;eQjdwbqI@C6CKymWW-6c+nlQPr$ z$159-Tb{e0A_>#195SJNShJj~K?m2CTdYbn-erDbS+j(3sj<-?2kBN?wU6I%xvrpM z_*Bc-qYJ|uw$vOcsT`iBQC58JdUV*kVdpN`4ScljxRu-3f*})MZtcx_-c|F1ou7kc z%ty9o*v zXOQk~{-konpM~bFcP>n|9OtBUc2;?f$9=s~dTCN!|3Bj7tuHOTYgqKwT;l8OzDjI0 ztHiI=w%xXct$Ml_oNjpb+nXsxedg9)aMTf97gU>6YTUS;b+wh}r7fEmKd*nFTbp(N zcr#ylqxG?*_gA}2jj`Gh(#~u{jnlJ*wb3Wf-A&QGpR~4Fh(S_TL($qy-^X#+i9ek# z${W|88J3n?FI#J^)#~LRI+6P)jzOn-cT>-myN<6D*G0#lk8{b(E}d}TP}zxA&NtiG zbZVHnlHq34r)_6W$Jag+e$%~W6J{A2|IYWpld79uy;khM=aBHS^_@H8SDY{QT;1+b z|Dq#hkx%zc?Q-CG)rgvLU2n{D^WznTT>cPr)c!=X;@1!RMLp4F1VjGfm`buV)U@~>09G$!@UXXk_CyOsBV4qnDw5L+m;7E z6n9YwW@qKcJ>P%Ps@oIBuB&EY*5wx~9bRO;Fys(BN)A|99*Vsf=d#kK%h9s!iLOCb zJ9?rkxbm{eD}~@{gN;wE)3W_*2cG=%?XJwi`a$u1Rt+k$K69>Nip8u4?s->px}_Ht zzBd|ITyJQ4J?qs`#tFOpp~npGbY6Sb{Bri4g^%2u(LYXF(SC=RL z-72#h&&3YEl@s5hz4!iE!m?EGf#I}hF-_Q|kg(^lK`0Y~CJ6He~y z{@&YTqR)hadsUWkv&#*RhiHa=cvNGW5!S6KK=Yo3CF4NTvl-p)*B6coyiwrv+JDHw zo@wr4-EKu6P@NyuIrVVJPV-R#w@ohPXMSo*&02CwQaQqm@!lY5f#AMJ!|QGb6LNPg zoOU9{^4!?f={mwJr{Xxx_=fuJNwuTV6+NTJyxIF7w5pQ>6Ee1^uDR2racgAOg5JHZC#{=(cIv#WlosRGh4npK;(O*!dUp$xQAaP{ zb2wn(k=6Jq6-dg)56HGHap19VWpKo#8XiMmaH9=)w+Ve}&-=%jPW$NQ`B)$A_ zuQW@q)HPimw7>W%yEOWvUO;Jiv&W;SJy`rIGy3|tL}ANCyTk|l9&Dz!+%>zi%bmoe zB-_kK20JRv-_|v4&sn5M@o@-0VUeEMm^H55Lk*MiK8{t%hfO?P!iJjZZkg3kaAbKe zv+bvAHw^1@Z|S{v3r83&vRv?v-a_1Qlr-P=)95qa?oG*A$$AEs?*!*RIX(2ecG|k{ zDT}@bT_bRJI^<^LA9i;S4oYiTyhFpS*>vJqzXqFTHVn vgS{TM;(&aEqOh*I%#5 zA6vVt7zKpVl&mEu`Q_H-L%ch5G_>1saIqfS+4u0|DYzwuR*V`Ka;M;qd-?D8xQ|w*WOZqpEPiptw6}O(_>0>I zlAl=LoiDg6J{@NJM~6UDzqRY!&VNc3s8upzYN)7+sR%bvP+UQ0i6c~iQu)59Ld>nh%bo%#Kx@2UOzg#r3O-uzBt_AB1;eB-SzzIsJ2{Uk8l6PdzzwU ze07QCOWP+ot4^e3U9`IQVD7G-yDuFqd3JM4^ym$?6^kX(w2cY((!w0?+VyTzdaHkn z8i!c}C%F5cvC?wPFJUhnvVg!>+Mqppji*M+&XKl|yZc8Qkc>Bg2<7gTL;lPUNV z`Yemndhc4-{D_`;Yiq`ge$n;t_Uf$WCob$ton}|Fl3(2Witfw9Wd(z*cqvvl^WL?M z`8e#MbdsDropIJMXuelhvC>0?h_ z*7rSmCMV$S%E1GTYM)XO;<^!)AJv(Vx zA2VBqnqhJ6xlS_-pA;^cJ!x|~qcJA$y2lE=;eqFpic|b}d`k<%nDI{w3!suRO1#$qS55B|0YNryC8o z?A@xUioeQvI^lKbG~33}X?9S{eHjI7 zu1*`fqABC_swRUbt+jqbPcE1yKXBhXA|O*IVe8X~*nQeD%dbS|W=`4X)4}L*{mljC zSu>8`Hj?zvxxXvm)B(Y!bhbcfVdR#fYdv5~AIszEHG3Y73>TL9RC80vBMxJKXjfx2(&q6Umuyl`~$ zQ;o)vqIA!VHvOG%nQpGm(qEZmnI7Kz-tww=-#S~9C za5pW$ZNv$~CQrT;Chx?EcH zG-`e5fz=}IGYfy?KP}UZPiXnxqA<#Sn&ZHNS#SFkPRP(JyJE3;Uq)s6Q?m`)cg&{T zaH}rsmeHY??gqoO;(eEnExJ&3Dlo5a#(hOZ>n&z+lLjogXf-=Lxn=r-h6NfuC$Md5 z%^D}9mgSbrmvnYGP(&irATD$qyS=raLVDE*}1B$zhn9)4{L?={5 zDJ@2w%eYooI4%0oK6>8Xb0bhb<5W@Au0prQscT`>*@&KpO)~xBBUk)+o!u`|J0_$4 z()@*0&u^j}M(h=P*>CnIL2l=lSL>5?4nJQQt#i%keB7|R@pQK#Np^nwEykV_J>T;& zzd`;+tQRt3!17AsS?py_R@C$A-7n*`%B;53IbEK2Yjtf@)Tk3Z?zhl+ZvJxr)}kTg z(Becxv`>Rq?2PNLM@Eg}mt8eY+xVd4)49tyj=8RJx^1SGu`LNBa|*5R zjy*LnJn72m2kFQ_OYEmlcZ-x!;UkU>5A3E!y>@zvh`MEp^h{>7Mh>C_OH4chs#|G}Om|wSQcU zk>l}Q2KghdWuK{U{wjBC;`#|XwR;;ACSL8Od%NUV_lGsPDcvS~c-||`wr0z~r8i<6 zd2V4J%XH5jXU^>2rs&hc%<7@%iwit`2AI}1mSiu>vRIzzzTWggYI4$O?UK*~S99Z+ z#o8wrwMkrfHZIss=ahHX2AMM)QwZ3CALO?g-gdy<*PRQ-e@r`>>VF z3o`vRNH->pH}V2l6R)}u5}RGT3-u!8k`dleYn;z8+C&Ps}v(cQ34$8%w0?oQ+>(pVz#eDF|!jpWQDr zmVPq8Jfxkc2Y=5@M@p8k|De^AAX> zntprrK&olBr0x6Cd*|;bw;#ODY-xtjugNg2-BtUI)>H31mrH!KtQ)FRv!YMB4J_z* zcC1|tVUjOyyuE0ibZWD_Cvn~0I-O2?x@qa|0}pO&-EbqX(~Lct$zi=m>%@AWSoQQu zl}GEkdFef#Mo({RQ+qs_XSBU`a>11GO_z&Tmf45vC+PCZtJd`-JkD00@m|}iq3yUj zGaUz=Q%~HYuBZLhSMy{`X&~@%_39?d2ZtnXestu8&Q|?;D{wyS+g| zOPh3ZVa%)vD`)I$O0T})s@E>q=5prU%&h2qWkv;Di8vPlRWsRNIpntz9BSo@i%eh0R2hPY_zOD@nZKfV*8nn0K#QO9Zo6Ypj za?B)2LrT2;gNCd;TOOWt<>)Z2vr*kzZL@f5k)k(Pi?-G{BQdG+l-DCxyK~dLmao*Q zUOm`D@xmj{vZ<)Yoe!6n`H343uPV@~b+~+NW|*zM+nF_<{NZi2Zr8uauyZ-_;s(>X zz!M$))3_0(6Q4E)w`=_|t?$brdvsdY-OV)mkXIA+&eYU?_d2r^GrANf+omV*?M5y7 zm^b)%ujhl#)XP3xicPrF74Bnm5ux?GR89k^? z-tCb6+x+$>&aJ+z+1kXjxbA^tS_i-Peh&u3&;SN?aN;0N?7<2bR&ZDW8+Q}}ZckA*75A5k}Y5uZ8ajbP<j% z*BuEQJ9nDr%H!KZ4|G3KIq~h~&b#&dZ@0RgdVE*2kCwxq zFP*qEI4MhhuBrUi=~}Cc5jQ<0ovch7Czy`C**$iXJ|P+^5gf?{*>&VR{wf`j7Ia)q67O%LR<9|#2NHiJNHwG;$YA^^Y-cYueIIa zSfpDqsQlhQ*QmKu<0CWLUv>L*BY4yPy}!G9obzg^FRhqx_{83`Gm7Svj4%FVKGkw( z)cS`9AC0@WL$UAo&CX@}7eDFnd(O&34>A`FU}$O7Z*tuD;bf18H@xn?A8!;i^8JH1 z_NSgFtce)?^?wHS{eSBF z|J3*Ysqg<&-~Xq+|4)7YpZfkk_5FY9`~UwpfB&D%|A&zG{kf7tRMH*D-vE?UNnS}x zB~K*}@y!3FT;Lx&*EDDJw-*^1S`4meC%I9neX&fXOqDo9;PuFLf9l`GowD5tP1;-c=2+qNWn*OSoodiuV3?$w&K)C*&wUs1 z=)P*&Rn{lDYFjqv2h%fMWgX2!-><&mt$aSsMRq{$f8J)9mvTC8xlGpCR?F_{2IVws zg{*7&Z>DV7M&57Wf`_tC?dpanw2K=q)^geEH%F=lw1M-rT_I~f zYUo+tXqe`1mYvEwVJi)R^G|pvvx&$leGvZl=Rar=OEY+H2icg@4rY6e;QasQ`82b| zveAFGxT2G?R(ZT{rpfrn3XW`LCVhX}c$)0iHRp|qzO$6e_e9Bd_i?iDZBQtuWf8Iu z@6LN$yi8P1Kb|V9p6O&Yz5R6MblGZ|p7~1e-(xl_r(dm*8Eo18$c>hzoG#ofizS|3 zPHDdnPv_ovC>w9+*I1VG_varc%!%qcprh<^&=udN+>ZG5|I70kd+>a8X?*wJbSmDwg2bq)z5$QF7@f8gX+_? zZ~kpsB>TO3b5)t~HRW}Xd|J8w<7wZ2Ud;;GtE)}R>e+DJe6u+kOu(`CD5cNZSgt_eS? zoW8YF_GDl2ptnaLZom6ocAwnrJ(_uZ!86tr=H)E4* z-qT&r0_tIZE^B1nx7@J}$ljt{-*APj`dIfoBX5XHEtkk7$41sYsLN0;k4~1EAL*I7 z)^?e4`r2$+NLI?o1G2@+=|#~pML+4E3Xj>!X)iz7cBc8hnO+evJxLaO%WCGsX~D|r z^%m8yE7dR3}drFvD=Dt@(>{MGfTUR4=- zs9yD7*sD_czpCH-=S*{E!$^gDiQKBHN2W$9>UFHD7iy$JjZ~K`}2i2=8FKSh(su!wPrFvDWS4FMjS9{4{U7zYzm7#~~ zRsV&(DwY4M`ptjNG-oc1ROq$jR#iPRHBwQpV^zISBNb|-LXA{V+xXRy$FHtWjZ~BY zh#IN<3r8wx&K&4fIoHXTQPq1;y{ht}R+XxHp?X!SSEYJY)GB_pm;BZBsa{nXdZ=FY zU)ZZs`M;{){O3$_=EF#Z_JQ20sz;_qD(ZEtsuyacLXA|YkqT-XzdG{x)%B^7iZTFE zBb9&QNF~jg2fZqzl6)Cey$98+DlckPsj3&MSEYJYs#is=;#YggUtOQ-Rh6NK>Q(=R zy(*RetNP7<&NOEsj8p_i$gQe+WNM_MUdO6>p++jyNQD}yptkX=BadHQpBkws0}wS* z`4^5<(wqg*t4cPIFQcmWpn6s1MXf4T^+NTkRIf_)s;E``YA^Y#>r=g|GW1Zr>c6m8 zRp$R=Xxtt2F{&Pf^1rQtG-ol4RhTcxZK?WXYOJDOv#NTb#wyfUg&M1%w(+ZDk6&G% z8mlM+5H(i$7miiZoJ1TNkNjO-@{RloVqndYnk**~k8+^VWarba62b*!ovYNSGqRH%^(Y8$^g^7z&D zsga5@08t~Af8j_a%~=AyD)Eqf8CAUp)vGEmYE`MK7phmKdR3}dMXlmjd&ys2pXybW zp@-^K|AoCOng7$d+iB=O>K|1DR1HuyK-BMxY4Ua0(^%KxeSAGM8N&F6o0eJcM~1|TZ`|Ci?f&BP`&$!p17$!W z=_An**NJb4cZnB>!^IQOD%3x!2B;dKYJjQ%ss^YUplX1s0jdV58lY-`s)7Hd8tB%H zpfUP)m23!!iZqId^p1=$3i1vO4G8u#ni3HbjOTd5oE92rUF}xlY_x7KBfsD%ztF(Q z))I1Nd(lob(*(~9jtUJ7iR>sw^LywC_v4j)<&oY2fq0YTW<8bD9YuH@!4dr2@2}HI zh}P-ZnSTmzCtU6u{-b^%^DG$u5rC5N2TmUxKQBj-f=hXl7{qqzo}tYYOh1A-!y2S;v4hai{f zGG8dS{COFA;7KlT!9vS*TQDl|bNwTO0@431ndHn?^jiERm>JlDf#zzrqJ6-tMn*?Q z%A+GS=md?Y-2$&j&WesC@BcsKt)xlnC2u9=l4p{Kk~@;ClJk;3Bu6FtB|9XWCF>+B zC5t5UC9@>4l1NFAWU|Cl;x2JQ?-2}<43zYh7)iQHq>{Fh<`RK~C2kbI6IY7M#81Tc z#5cv4#b?DQ#fQav#M{Ih#cRaN#Hr%B;zV(@I7~c6>?8INyNXANZNxHhe{nBycX21N zwz#!eBIb(eq6Sg5=#{8M^hi`7x+c0H`crgFbU?IIv_+IDS|wU6S|Cai#fhRs!6HAA zmuReLv}lBAsK`RpPt;SSFX|v_Cu$)QiU{E+;d|j5;S1qY;eFvP;T7RI;VEH`aIbK? zFiW^rxLlYfoF_~aP8Ws<{e+%EH=%>jMrb8874{T%73vB#g%Tl0&?I;-s1%e6iUb9M ztAaeiDZydEZb7ynQ?OEyE|@1s6igR{2>b+|0ylw!z(!ytFctI^bQS0dGzAg?hu_41 z&#&Z{@{9Nd{Hy#t{we-p{%(FYKa;j$QkB9_FJm_gz~#E>`>(IlgZ=_F3X zG!jQ53c;O9M3VRu5hQ*@ILTxpj6_a^lK2uKBtAqi$#^1&WE>GlGM1Q1;!aEv;7FNNwX&tWQ%6Y z39>=6eF?HXvwaA%IkP7bWNRkk39>O0GYGOR6EOtYl!<79Y{|rQf^5jdG=gl$L=-_b zVG9m1lev`VFcN1S)l~kYFQx!*=Sk81leYp^9i!aGUpOx zi)GFs$Og-tO_1%CnM9Dyl{t$bTPssRkd2iYLy&Ei8BLH)l{uXtTPkxJK{ixo6hXFA zW+Xv2Q${*Lwo*n4K{isxLV|3gj0FVQL>co5vV}5|39^AQ<`HE3WF!(~^JF9tWb0(a z6J+CL#1UlMWW*9=(`3vb$d*Z8PLK_gzJws#C4DhLHcR>kT#DX8z5~ing6#C&!PIxx_%y0%i2&;``#$;(g-Z&}!5_ss^YUplX1s0jdV58lY-` zssXA7s2ZSZfT{ti2L8JmkhEaaba`AZzFke;>u%kgP3xhW@zt$ZeeTW zO?EN=tIea&Hi&tjr%jB=-Elwf&csM_0{Il z?zEU}T7Tv7eCJ~rZ5V8t36IA^wQ1xN6%-Zd9T_l9ZsZd&O+?3gS1v)5=v8A0m(V0y zVjGbkdO-c7YJjQ%ss^YUplX1s0jdV58lY-`ssXA7{*TpwVl?`YGd-zui}vG92J|)w z8Ws>Jm->VRg?fhvptMjLDUXPhPLfAXm&=2t9Y3dw9i`sEzS52n0YSc#jJ}#XY>?eZ z>!DKg#K3@HxwM~|G$2?yncU`Rd1y#@O0Uo|tOxKYd33FJClo^l^E5(h!ZGKPk1>Mg4+QdvBtn^q)RO6VqP( z`oV?Sr!Ty2y-fb@g~NaUPtu8o{-gd;H9*w>RRdHFP&Gi+096B24Nx^e)c{okR1Huy z@IPGxim`MNt+#@sYpNNmX`#uBKyO7wdVBhWgv&kU(cVF!fpVjs-1A%x&%BR`w3o3-f4q>)|C=$V(%{OdOjeb9$0LV|~(zf%zvE_aR! z3PSHt$|L@M4Z8D?(bj`ptfel429LB>dmr-qo7A`SmD-JRv9_@uEp;4iXFq84Sm_As zvC=`VE)I61(8lboN4Xe&XG5dq5mAAWc0qr+Vz8DM? zwk+SvYvlox<8Mw%?G)*Q)K5fh;ZGj?m#cAze^l^P)g}8YhioXd z_ns0G?hx*auIgYv@~ZxP`o;!|eqxSh_wI~Y9q{>lv%ha9=`yu5`ex}bpOWfqP$h;( zIyx&`pmngX>LH##B*q8bWWkLSE&4oV5Fz4dcId!R^!(IX;rlLxB2@NMPZ zsNz)ZO(s;XZglUAyRTMtKLxabqMSx$*;h&USXezVc{gKL?jzsx41i<@_NP*f-x_#mcy6Fr^nMusT6>j-$&Pyj-E z!Lbsa;%qk>1>!F+w1Gm0$I@=IlRmo@G!)ef>};ocq~1Ee%>T)QP}Lf)w9l;qk3Vn^<@4PI z*3qHi%8UHFi}B^>ED!PyMxFY%uke_Fh=54}fdP>*YD-Lv;h&*&NFW+Ys4XP#pusit zx!3-Dusuu+6fHR%O+7ujBIK)!+&MBvHA2J>E!AH)oD1LuKUr0HUA1c%ZU0M4`|A-J zD7v#bn*ICJ6=C08U*rp}dI8lh1J&FC>IeJzBK!K8KY85WRve0NUt-&r<2O(w5FAYt z6M9nDuPX8D{e9Vx>L}Gi^Iw<$vLVt=a{JWPDIl}x8R~p0-pQ&n>3%Xix9QkH8 z9;M)y{Y!&WyNU+0$Wa%4mPFawz8;y9&-l_q{cy=Qq5g;K{ba>_snO4!JnkG-Q_8MO z*(Iv_RtF{-)95gC{IZ45NB^B!EcL7Ha~*wt&QBiice>IqmzrI~NV)f9XFIi*ihNB* z$|pw#g+xSt)xUoIvN$r>XlO+ri(iz8FcdlDo)Pk}D0#4t{F_Op+If7lP(K(Qiy?ZZ zfuc2?%tq)6E4)nM-g*tS;66_81Hlk`5eA zdjp1|xAO4SvIEaAV-gtmt40Jk>_yOH)sORUo{SGpZ5^rpU-n?2=pyB4T6bqC{O}Qd zv%*)0WHW85=*#G94%IoRtO-uDv zJuBHo8>;WfAX*E(9n_hj(8LGz^+L64`uRoJ?VaGd;PLNZGGj-pxP8_+n>7~ba6(uLlIh!p=j~dx#9ts+9s4w z!Sm^p)%N+-618W1IZ^zqZ}zI_+?J#1+@GPaQ||V=!!EV$sRtz0$L8 z)uj4gT?@*ZHc)7^;b@L9L23I})k7{*9f0~Vey&*6V!E&T@%?25wda4Sv#)pYZPgiT zax}-Ie#Aj}jr?%vqqb?acmK$(e;#qrebf%|$1AFB4_{M1-1WDI+_E+5J5c(k>^r_( zf1eLREtGv{Io(a|AiuMQ+7{F+^7CG#`f4$7R!A$3X8ZPxIlYy`Q03dK@NO&J1@@?X zw;JaDdAt-wHj=HOWKX4Bm zs~%GM?Sk*$)q`&2>s6Eu!EUJXmHm2&+Pyn;kBdI@JN>|adcuF$+m}w_KW#wOm3_UA z>Ua$l-8!RTj1@y+|AUi_>FWmDfjeprp{>d2cwhM)IrpwREc z(R3qk4dR3S(Y-yjZvICr(G%2;_D8F!ZHqkKAFZgWWf*pO>ajIVbQ!VA5l$%ixdU{U z&{OqR^yk}x^mw(y_&opfn&dt|&z0({HekR&={#SOj7s_>NsXjT@<4J;az>Ja{__8N z$udc@2A?Ax&qEDha(OdMp z0MA8_MfXIvMERog=$8SGiw=qQh_;J1iPoXt23R6W5zQ6N6wMGtqF)E_7flj*h{lK< zMZ?kW16Yd8M7>0YqAnsS`h|d&BC&`oVhBG9-=W_Kcqx1)d?YLs-Vk0!zY>ruJSsdO z+$G#9+=zZBV1;mzaDi~PFhLkC3=>Wf`UpLQuEJ61w*q9s{=#0u?!rz&ZDDKlYXMv# zUCoOmIN3Q?OZ(Ay_U*70eOD3!(%;0=ZypoFI;zGlApEvFBKGEI55RJvf~?+MHG#5r<$m zvTNBD>=O1v_HFhR_F47`_CfYe_GWeldpSFmJ%=67j$#L~`MA-a20k0 zeFeB2Tm~)$mw=1GMPNFZ2Bv~3;6iW#I3G*~=YeyC46*a_4FJAxfRDX0tTfZCuI*dA;LwguaOnqX_N71$DN0X7Glff}F$6oVpA2ns+x zmP6kHW`XO$4DdH_B{%}?2J%2ImQCl7(i-Um$O4%l1EhmArL-pS6ZRwhEyz|%Ys7w_ zeZQt&xg0zLza z!KdI8@G)2fJ^~+t55W82J@77A2o`{Mz}w(0@FsWzybfLiuY&pD74R~63A_ki0MCPY z;5qOtcm_NT{t5m8=7Oidli&&PICu;^3LXJ-z{B7n@E~{q+z;*p_kw%C-QX^8Cw4pS zckDLWcI+0~RxlfzMcWK+!mg)n1UFzaXqn(THo6<Y?8RI)(Hd?m?B<|&!0WR8;AN|Kb!Qi6UTfINqp1d~p8 zO2lOXF5__-hs#)8&cJ01E~9Zd9hcK^8HLM8Tt?tB9G79Z48>&#E`xCygv&r&PQ~RE zTn6CMAD4c(oQz93E`4$7gUd;{^v0zZE4D4fxEzPevAA@{r5i5C;L;VB zF1U2Y-;<7a^Tj8=L zE?eNTIWC*wQiHrSMYJ7Cwkg@7BumM9B^l)UA}3rL<5Gf4F`>+na3Wk}8cB=&=|17-hnV=&>jHSonwVA$mNKe2jikZYX*@fqcxH z#@me^dytR0G_Dvu9#1|-xlkl}JdS+KTE;3ukH?aanL12M^w^zzj8c#7d^B+cB_ooL zl6R6fl9%Wg03S&TB{w9OCFdl$=r;hV{GZDIsr;YH|4}}khJ(fkdMcO#E(8~V^TA|r z9yk}A1I`AMz*(RIoCzj^31B=J2gZUkz!)$ZoDNO{qrgZo0t^Skz)&y*3Vi6;HmC)* z2it*d!8V{K*cxmFwgg*%&B11%1}Fi=pa>L#0+5fT^8YtvJJZpz=z-i@IH7CybBhB1>ha&3`9FhJN4^=r5aN=_V)Ez=g2`n_Hjt$_n=C~; z$x^tVECol%l7EUUd1uLzdxz7K}NHio|bf>>g{080Ue=IIUclpnYbJ0EiJ>sqC4*x3gB6NR$rZ^hi-S-!Jp?5SK z#kT0qzM0q<-Pf0j+n~Gp=uHcBPydam4BgQ$6kS92^K(Tx=x+X2(Ry?*e~~B|-N}y@ zg`)fTULtpN7vEMSL-+8FMf&ItejAYnx_?g-)}g!iWx~hk-u*Se}R|4UEnCN703i;0%L)`Kq_b>&=7D1G=3fb4Zn>4 zm|w`h#y`)`<>&DC@VD~U^H=c~@ss&8`O*APzCYiK@6LDR+wx_6GrlojpD*RN;cM`@ zd>XHg_l8%-d(123UE`hSv^kqi+IVrnY?ITD9@kg#dGI5@@#oBo*B=W zr_Yn}+VC`ZTpo>E$9=;s<38pVa<6gEb91>l+&$c_-1Xd5+(q1E?o4hpHgxox-_TrQW!spGuilyM$&3OUy}=Q+8Y9L^rjR?d3PD$XKKGG`_y zniIW7o0Yu*=ww*@f(D?DOngb`ETJDEL`9nB79`?I~+?rcZ4EnCJmV;i&e*-~~JwgwyhP6km&ydlbn$3!7kVk@zpSVb%%lF_$OqKQ!SlcQdQJK;#!5;DS!FedZ~Dba?|Ah-mLRmXb6Dq}rn z6|%0e&a-k^IjlXbt*rIvJ1mP>$*h^IXjUl8pXJ4JXF0NLSu&Ox%b2CllCs*cG+0~~ zjakQh!z^PyW)?E9G0!t|nK{fo%&pAz%vH=q=sPeonbFKpra#k*>CSXy+A?L5I@Pye z$Sxker#28A0GfmSK{L=4><9J*`+&W{UZ4qR4E6+#z#gC>*d6Q!;?x8kA5Kk>I5k1y z)C7rB6C_SekT^9#;?x9*QxhakO^`S>VKg#vN`k~G2@09rzA+6k|1$Pg47_-11SN;pa{e%30jU*5+qJZkUUt<1vwxaBtRC(1Q{S5 zq$y=IfuFD+n7@H5!DZlLFb!M?T7fvFV|-vVDyJDAu?>t5*eXUnSO>lb-+{GY4Ok7n z#lB(U)PeDa(V(1WRADO_Z?F}N*VtE#3ha7DIfzpd#(D-$Nsu@tLE@AIsod^*22M%P zG)_s7I3+>ilmv-W5+qJZkVUY4oRXkvoRT0P!2J8*J@77A2o`{Mz}w(0@FsWzybfLi zaY};rmk-ldK%A1G`IlfCrzB_^rzA+6k|1$Pf;uqzp>!ByC0j1}N=>|(}Ja0%9$u?S2D zmF>%#kqXl(pt8MLGZw(~d@vcD2hIiOfV06Qa2BWlXM%}f0vHd*fwABWFb0eUr-ReL zC@>O?0K>sBFcb^{gTWv$5S$850RupP&<~sp%0XYy2b=_YgI=H~I1!uxdVu4>ao||c z9drZ7fUckm=nReqoj^y>0UQO|gCoHa;Be3mv;}RzVc1TLq2Lg3FgOU5fmWa;XaNoc z2Y}{af6xpx1^a=0!9HMbuoq|o8iPGSBd`Z(2zCd%fd-&H*cI#ocE&OpdSFMe11JS` zK^;&V)B@Xs?ZCER8&DH$4YmSXf-S)2U^7qylz?JT1PVa`$Om~K7vz9!kN{a&I%7SU z0saQA1ebw}L2IxR$OIW!^jqzu^!M~8@Dtbwegr>&4PZT3hkeI*1=5t#-(zd(@37VM z8thwoHFgWV3VZ`rg0H~}P`TYL^m3Sf3BCZ!z*6uzSOPu+i@~Sh6Yw!u1U>>Ef)Bv^ z;63mzSO^w?cfi}=E$}9I1H2Ai1FwSl;1%#PcnQ1+UI5R7dEhzlEO-Vy4gLxK0p^0I zz?0w!@Hlu3JPIBGbHKykA@Cr00NfAm1NVY^z}?_3a3{C}{GAky=PjuGpUVHK{Qtkp z|Iz1baC}4J_=d#s4T<9$62~_rj&DdD-;g-IA#r>|;`oNd@ePUN8xqGiB#v)L9N&;Q zz9DgZL*n>`#PJP@;~NskHzba4NF3jgIKClqd_&^+hQ#p=iQ^j*$2TO7Z%7>9kT|{} zaePDK_=d#s4T<9$62~_rj&DdD-;g-IA#r>|;`oNd@ePUN8xqGiB#v)L9N&;Qz9DgZ zL*n>`#PJQeos4tHZQxcg8{7hJ1~-9O;6`u*xE{;|*MS+}T5t{c8@L)=1+D~FfXlH< zSWB^sSxc~KtVLitb|EVjOu;6z7J&1yb6E4hx!C@!*!D(O=7zsvz;b0gT3Wk8eU=SDxP6elc0iZwV2Tlg%pfBhHP6E9_FVGX52u=V! z!13TXa4hH!x`AUrSI`A?21kQVpd;u2jsoq$k>Ch$IA{mjf;QkV&>9>H4gm*)gFqQ* z1zLg@;6QKyXpU{eG6PM)eqdj)57-;*1)6}yU{BBp>;W2r-NA040jLjl1-pQq!A_ta z*b(diN#((~W#D2k4O|E&gLAl{(_uxBh4YL+o z&a4LCf>q!fuo8R?R)DXtFIkV3GHbwc>x0`($0Nw#_gSWt&;0^FPcn!P?=7U$j%ityOB6tBj59WdAz_Z{P z@HF@*_y?E^o&ryTC&1(2G4LpO1k3>sgNML_-~n(yxDVV5?g4j$yTF~`4)AwyJGc$p z3TA^_z|G($FbmuWZUEPVnczAw16&KP!LDMi#;#A9e?y;(75!*mij3sitJ!9*|tj0fYuSa1dy14e_>!D(O= z7zsvz;b0gT3Wk8eU=SDxP6elc0iZwV2Tlg%pfBhHP6E9_FVGX52u=V!!13TXa4hH! zx`AUrSI`A?21kQVpd;u2jsoq$k>Ch$IA{mjf;QkV&>9>H4gm*)gFqQ68W%sJ@_#D- zr}F>*F8@d0wZZWXiQ^j*$2TO7Z%7>9kT|{}aePDK_=d#s4T<9$62~_rj&DdD-;g-I zA#r>|;`oNd@ePUN8xqGiB#v)L9N&;Qz9DgZL*n>`#PJP@;~NskHzba4NF3jgIKClq zd_&^+hQ#p=iQ^j*$2TO7Z%7>9kT|{}aePDK_=d#s4T<9$62~_rj&DdD-;g-IA#r>| z;`oNd@ePUN8xqGiq!k(Gkd~kYI1n5FnuGm8Gtd<52lfT~fW5(9pb2OU_5_WvLbf5; z9qa}gfcjupunX83>;&q89l;Ku6x0QEKy6S9Y!9{r+k$OCO|UiC3Tz3s0Gor&Kn+j= zia`-6QO*{Ce2@onK@P|U36KRcK?X<%X-bJE@Dtbwegr>&4PZT32fhd2fwf=_SPi}f ztH3v4CHNYw0AFEWu#>?#;4Cl^j00o99-vSuQ4YSumJu(oMMNq194rB!fyLlc@Co=B z`-pu@DNzO%VILBYu=j}v*n7l%Y#MPFECdU{JK$|lx!pA4CQRP|uY=dXt6)BO1-uMi z0xyCW!1G`pcn&-Zo&isTe}aF2x!@`ABzOWm4ju!Kf=9p{@Gy7?JO~~D_k;Vuz2F{j zH@FMj3GM)Y2e*UUz^!05xCPt{ZUVEwjo=1wJ(vlu12e$2;2Q8Za5cCJTnVlKmxIf| zrQi~9F}Mg!$EFae*o8z2Hknue&d1In=7DpuvxwPX5;l=gfHSeZi3BhnRJO0)L@Z3t z0F~{nHxUif)4^$A6c`CcfZ<>m7z&1f!C(*=2u=m3fB~RC=m$;)<)AO<15N_HK`+qr z|6%XGgQck6H-Y!*)7>-EJu`jDIp>^n&N*kvIp>TZNs@?wf`EZ20+N#?$yq>>5Dbv<9tCtI?{o3av~l(TcPJElO-s>|v;-|q zi_xOA2rUc~ngwYAnxE#Qd1)S+o93c9X%3p5W}{hY7Mhu6q8Vuhnx3YkX=xgonx>*D zX$qR0CZkDd5}KGM@-&{B31}R3X_Q82m^##^AsVCsYEhFKR8pM^s`(mk=|A)h{hR(p zU(;9gC4E8vq|fOe^cnpfJ~lt1Q|V+nfsUi2X(5`>*LX^wz~79=@SgFA{z`wLKhuZw z0liOuf_Kf!zQ%9#9=v1Rg}02`@TPIgQ@t-97~lWL_y6(z|9{s1tLJVY=ak4fC2~%Q zoKqs_l*l1GpsAo{#@b`i1@T#hKzuwuv6^h(c`%Bzk zq!;LUdXD}G&jgS72CmYx@O0n|JgKH>ykF;sz)3hxJqgp>j|&{9$LRO;DD}VYxWHlV z57C440Nqdb(Y`sOK-bfCbS+&&SJPE=CH;o3 zpv&np`ZZljzoJX%V!DVfqzmYL`X&8>eojB5^XObUhkiQr`Z1kJXVB>s?-$g6 z_i5aJNT<>d=oC7cPNEa(1o}Q5561+?!qI9{%lrRpWZ*qI3JwpvOGm&VYM#sc=Y|A^ z!hz~ptloZLU=Yj}7(n|||MitE(3kr@)PKEY3-sc?C+$JI({8jY?Ls@#PP8NKK-<%H zv@LBzThmswC4Gmspv`GB+LSh-jcFs=kT#(8X+2t()}gg&En1V-pw($LT9sCzm1!kf zkyfDPX*pV!mZ7C-DO!@2pv7r1T9g){g=ryLkQSi%X+D~l=ApT1E}E0(pxL1n$V#)& z%rq0tNHfs%G#yP#)6mp36-`M~(Bw23O-hr{#5562NE6UF>e48U&@gqVO+z$D1D@8s zfJqH1sZIsee66?iANq#=P5+{==_~q@zMy~7=kyQyjQ&oa(kJvW{f$1NztUgm&-5XE zK=0F^;2o8K-gUSW7)?jg;dBTcNV8GPS6xB;eCsZ}t-g}CzhvE_H|Y&}onE6?=@ohz zUQ~ZVdtc|Gb(>y-7p#l$oOK@lXq|(Tt+VtDJxx#1lhpsZldTinAE(FY_w*<|LJ!kJ z^dLPz_tSlJFWp0T(_M5Y{f_RS+vzsCm2RQm(#>=e-AFgk^>iIwOV`lVbQN7mzo9GW za=MIuO_$QI=n}e^E}{$R0y>|5Nxz_<)6eKUI+xC&pVHZM7X5^NOlQ&=bUOWrPNN^v zsq_On1x~ak!3owxIL>;Xj)$YIv2+X^X}w2B!Qs}sbOg*}4WmP;|N6>f4fgh`zlg;5 z|MC5QeEyMiBM(D^Lde0;z|bK4{eGbVv_I?}>P!2;9-&^eC+r&PPP@TWp)RyD?L<4$ z4zxXON88div^8x-The!E3)-AEqfKcO+L$(?4QT^fpVp&wX&qXd)}l3O4O*R6qg81Y zTA5a&6=?-po|dC!X&G9YmZBwT30j;MqeW>ET9_811!)1ApXQ@^X&#!J=At=i4w{{2 zqgiPdnwe&z8EFQZo~EN|X&Rc^Gx$1`lBS@^X)>CWCZUOGBASpUpmEftQ5vCP>QI}8 zXpjb|MNMi@Np&iy<{NxV|DkW_-}EndDfo)Mq%Y{7^f~>5KBK?Wr}PPZOn;+~=&$q_ z`ZIke#%5AE{}o~I|^vEXrdH26I{5AOXy;{h%TfH=zRJm{epf@Kcn;LTsnt-N@vqq^b`6qok?fV>GUHyjeba{(hukq zI+;$Q6X^u{J{?cT(Xn(49Zlb(qu_|(yKs1L1RN3^Mu)=ztB`@`PBzO)a_ z5bQ;JQvdapA=sV!Zq$FhWe9fRzBBDaJJJrcJ#9zZ(l)d;ZADwscW4XRoHnCPX%pI* zHlht_16rTfqjhN=TAS9QHE9i6omQh&X%$+TR-zSY1zMh#qh)CsTAG%kC20v-oED=+ zX%SkO7NP}d0h*uYqj_l_nw#dLIcW}>oo1t1X%?E9W}+FP7EDjm(X=!TO-)nLlr#lR zPLt83Gzm>i6VZe;0ga=sr+Oc}CcgiV@Bicb|NpH2SKp5V`KCm^DUokV|O5~gpIj2O__xh`V0M;KBN!mefkr(9J_h&NN9hrIm>!}B=>fW*?t^=rZ+-3W>0Y?o-UD~qyWn^BPT0@hLATRw zbSvFL{jb~4-pu_bx{+?6>*+eWmad_z=_-T|^hs1#~|B zl72xyr=QVzbS|AkKc%zjEcyxkn9ig#=yduKokl;TQ|SkE3Y|k}|V4d>}q$X-C!rX3+)Ws z+Z|~Kn8a>J+fx7amBen%eJkp}-jdkwaNmM9r_E?n+JrWyjc7yKfYzt=XkA)|)~2;+ zONm_yyr^RSdT7(v+g=j%qfaa(9XkMC! z=BBx5PMU*er`c#$nuTVjnP^6ufu^VFXj+i@v6>=u7&7{z;$1v!Q47clwk* zp^xcr^b!4){z8AI59tGXpZ-Me(Yy2xy-jb?oAd^~POs6c^a{O9FVTzi0zFU9(I4Sy zyC3aMd(f`56KzkE_=f(VXXzPuDs&ni4xOYw&=d4HJx0H$N9hrG$o|eZbc!B^2SbP8 z{?GxqFSH*H3GJnO=x(};?xg(rPAQR7O5~IhIi*A{DUnOcgWl_3iF{HbpOpJh-yLZ~ThivV32jL0QT(|W-W|rD z4@%^m5;+&%6-Lg5H;0jLO5~do`KCm^DUokV`%eeVzCT)0zsE&lmV;WczMT}4+?|Lb-Nui$<;T}HpAOX*j130+JV(S>vY zoln1{U(nC#XLKH&OXtu}>1;ZSenLN{GwBREoqj~8(GTfV`T?CnC(}uEBAr0rr{n23 zI+l*1qv?Be6dg(5r6cHYI*bmbL+D^Shz_I!Xn)#|_N9GjZ`zCYq&;YN+KqOlU1(?6 zA>0wR4|jlV!tH2V*fQLjwt~&W@6Z;oNw^tp3Pa(>v=Q}RU!iaV?(0+k^%e@(<-QKB zO>5Dbv<9tCtI?{o3av~l(TcPJElO-s>|v;-|qi_xOA2rWzt(So!9%}?{u zyfhEZO>@zlGzZO2v(c3BMhj-_MhX!;%Zq5IUF+q66sw z+Mo8LeQ6)soA#nTVK=8c?CNxbot!STGi>j4q#a-zryXqzTRN?2E9g4!&=%BxeYs9E z?weBo_2xQ_xo<=p(gw6Xtw-z9Iiz!J`2Ih>|Bvtg|FiyI zeOF85n-ck^M7}AJZ%X8w68WY?zA2G!O5~do`KCm^DUokVW zJXCw+VYF!!IT&pcMGi(AMv;F?+`&^V57ZFU>=9 z(_Azs%|Wx%Y&0v)LNn7$G$YMG)6;Y`ElopH(^NDiO+k~>WHc#FLKD+OG$BnuQqq8H}aPL1FuH@rhn1b^c8(cU(i44bNUB;Mt`SI=@a^x z{zf0sU+FLOXZnynp!exd^d7xS@6g-y7QIPt(ChRX+!47#FVjo(BE3M*({uDkdX}D{ zr|BtrlKwzX(Bt$N{hl7BN9bXCh#sT|=zhA7?xlO^Zn}%^q~F19(dM)XZAk0U+O&pm z*+eUCi<0cWGh_@S4Y;sm628Oo5)JoCbEJq zr_1QqbSd?}Zkxyw?ibTVbRk_p=hH9g7xZ)b8J$Px(mC`~I-Aa-pU{u#Oge*3rytR2 z^g}w8en6+t$#fE(NGH(u>3BMhj-_MhX!;%Zq5IUF+q66sw+Mo8L zeQ6)soA#nTX%E_+cB5Ts7uuP2q8(`m+Mc$fZDFfOYuGZ<3O0|tLtDTmk!G|hY#3=w z8^L;!2DCnuk-D@F^2vxAeMWz$Pw5l-nEpl|(O>B=^k@2zKA`vMPxKzWOYgwF;al`3 zy+N;gs`usZ+3rg*N0!&zC+-9HrhCynvsf(ObBIlIIIVEyViJVg+=ak4fC2~%QoKqs_l*l=e-AFgk^>iIv9ao#ypjBxlTAr5ib@At- zyV^a){aU&Pu5wqyCGJZ44P8N((`EE)x|Dte7st)@byv|PaFM$hE^rsZ`R)SP(EXBr zK|iOT(RtMWx((eq+Qr`Z1kJXVB^NBRY+KNT<>d=oC7cPNEa(1o}Q5Psh=* zbPOF$-=m}GNct`vL5I^}bSND{2h%}xARR#a(|)us?L&LhUbH9eLA%p#v@7jGJJU|I zBke%j({{8iZ9`krR2X*1fCHldAaBUs;U0PDH+VQsfAtpjVgwP;OP)vZpe z!AfowTG=!D!mUUvP`tiWy&U&tDPC`?UYh$-v?MJ-i_>DXC@n$@(?YZ$EkN_rd^9i3 zLvzzyG$+kLv(s!eE6qYP(@ZoY%|O%BbTlnZLsQdKG$lY(>ppxoTP|Y{`mi|NE(7)+l^fi4&U&8azKk0M&2Yp6=r%&k< z`k4MkAJJdwFZ5^nkUpUI=}+_?y-V-V+w>N_NpH~W^cuZNuh7f%61_+-z|GNf^hbJ@ zo}s7dDSDFrKu^%)^cek~9;HX*-0pwsC` zbQ=ATPNg5vDReTOL?_Y-^nE&>j-zAg7){fHs!vFy?*D6J!kXs3{QvtC0CgGS|BAl? z@mC=J3dCQ5_$v^91>&zj{1u450{_on0k0p3n-_=vCT?yV`kT1gap-H5=xdbdYn13~ zl;~@e=xdbdYn13~l;~@e=xdbdYn13~l;~@e=xdbdYn13~l;~@e=xdbdYn13~l;~@e z=xdbdYm{T~`>7o_n!ZOz(UJ6BI)VI=s-Gv_NV=5U)qQEroCuS+Jknd z-Dp?Zg?6T$Xh+(Cwx{iATiS-Urmbj8`VMVDo6}~rDQ!X<(?+x*Z9wbOdbBRB18c_B zf;Hl5!m4rAX*F0Wt_rOT%g0ru6=0dTaR(IKQnVy3L5tI3v?wh?3)4chAT2=i z(|j~9%|mn3Tr?-mL9^3rG%L+QGt*2oBh5h5({wZ~O+!=DR5T?`L6g&DG$~C&6MOyt zwa`+{{mXssK6W3vcie041^2Xj%suGtcDK13-Bs>VcfLDEodlTTj&t922f4l7E^b?O z7NCJ!)2-x|c8j=q-E8VKKngdZ>$pbrP4v&`V|5i#idoBDy&G zd308ET6AJ`v^o>eKiVVOG1^L<2d*2frcMQvh!%|Iie`?ci6)J@>Rf<`ypBAJ{2I9z zxgNQoP6ix}?2GJ(Y>KRod>vV!&IZhkd=MEQ85tQI=@aRyP6xDzG>p`WRF0I16piFl z=L0fCQbrO*!Vxq4Pxys8A@DGKJA5VlWB5e)keW^2s^*2i2`>qM5uP2M7M>6u6&@V! z9qt@%9c~h?6Rr|26D|_Y6V4J&6HXG2giYs-^W1sl+;gru=bRJHL1(A4*;(x@b-r|F zJJXyA&M0TF)7$Cnw04>}b(|_r8K;Po$I0TPagsO@$F$$r&+SL{J^Pw{&OTuuw0GK@ z?bY^D`%8PaJ@0Q~JBb~!&Cr|B^U$Nvz0kGL zxzLHw!O+go=Fn>OpDzBd_$v^91>&zj{1u450`XVi|HUidzgg3+(W~?dy-Y9Bi}V6L zPtVaG=~;S)p7xBs&`!~l^apx^9;e6X_w*<|LJ!kJ^dLRp8F{1ar~Bw$x`*zjyXa2( z9o<2<(`|Gs-2$cdE!|8v(T#KiT~F81wR8<#O;^#C^c&A`NLxXd(`EE)x|DuJm(ay@ z5nV_Z(E0RBPsi21pr6yv=sY@?&Y_>u*>o2DgnmqC(ixt15^XyDh)$y)(y8jpFpUys=d^-Ac@M-VU&Zn(U8=uxbtvswe zEq&hcY2nk{r063S*Wg09iQ4hwR~#&)bOe9Q_ZKUPZgia zK9zhbdKe`t_>}i4=Tp|Fj8AEwQa&YpO86A_Ddtnur-+9vSJ65}Ixlb~m zq&`V}68j|bN$8WnC(g(9iJDS~1`(`btPWNiYY1x)YXGZ-)x>IGl~{E{ickY@{y#%t ztmgjh{;5s?{OsO!ud6A5GpheT;O=z4b=RsnfJN?S?kDbt?gaNecc_{K=KHI$9)}H=0dN1*C{3j5<*x@+R_UKGo2~U zSZBD?-|6nOcUm|NoElCAr-W0$$>C&hQaA~mkR$9@_EYMrVJEOdwg|lnJqHO&{uhe>EB*?^UxD~5 z5Pt>!SFV8fo;Plu`ZL&buD}QWakT|L@Q-~<1-@&X_D z$CVNMz+d0`R=B;AUaMjlzH%goG=E4wnl7mkA9n)!xFdw0~f8 z?G0?I{SBLFf5C>@YgkWv1#4?BVGZqtr|L2PgjcoaaEJB>+@?K)TeaWeTJ0%ZqdkGE zwa0L!_8a_0dj#8Pzrt49FR-QdGieu%&hYx_<9(|EBHp_Vx*FFFdC0f!}Mp;a+VQ+@qoQw|8sbVZT$`0l(9>!+zQ} z*hkw6duv-@55M=fyK0-U@1$*l?X`_CiMGKr^oO<{p4HI%hfZs2u|K7)frqu#@Q}6& z9@JLC{n|HhpWpk3hG@&NAEYgV1GTSVKW!=Ot$hW1Xz2YzUA4v7r_vUA248Cn;U#SW zyr9j8=QZ^H!4rP(A3Uyoj`}g}Gk8>+2ajlT;Ye)`9HF824-VI6V?RWj1qW)Mz<%1t zu(vi7X3)_42Q|O<58Ti`^6G(W+BA4o`w*VhrouDY2k^9p-al|sn~ePr+9Wtmn+V5f z6X0m=eK=A>?;jZM_x^z)+E~;FYUup~+0gq3EcE`?J@o$89rXUzZS?-uCG`H*MfCpG z1@!*bIrRS4kLdla$>{y9iRk^U3F!T;ap?W6(dhlHk?8%c;eEZ{%F2V@-wLDmH-ACz zZ$3otZ$3cpZ{9%fZ(c|5Z(c+1Z(c#~Z(c_4Z+?W{-~150zd04Xzd0Gbzc~TDzc~)Q zzd0JczgY;qznKudzws2kzwsEozwsM-f8!o{f8#ECf8!2%f8!Q4e`6MUf8%5H z{>Du7{>De>{f(*hyx*TO8NI(T0lmLb0=>VH9KFAMf!<#}NAEBHK<_VqLGLerM(-~l zqW72g(fiAv(EH2J(EH1|=>6p!^!{=ddVe_+y}$ejy}z7_-d~nO?=RD$_t)Q`_t*bI z@2|f`@2@{a@2@{W@2@{b@2@{X@2~%g-d|sc-d~@O-e3PRpZELIKSS@Y&q436&qD97 z&qVL9S3&QuXGZU@OTYIQe($fL_ZKhF`-?x(`-|u3{lzo%{^ED^{^D!&{^Be2{$dGw zf3Xm~zxWcpzxd4W{pb9*-an_``-lAAKj8QN7J7f}4SIj=Z}k4!U+Dd{SLpqw6s zeWp79bM7H`i|Y61xgV(e{Jq@PZe7*i=XW!>30$G>=wFE*R(F^Tw3O=B z6Gh%eeo=Sl_ea*Mo_tbdV5F_O2VX3bDdL9z3f~W(Rd?D~hUbLGhx>+GhO31Oh0}!{ z=Y?~}IqB?hmN}m|qn(~kGpCZ1S9QvU>XEOh?s&86i>IlMxU=eo%cw3miRynJsm}MH z>UqCZ-R@x3=hjgjZkAAj;2YJ|o>TqoYSqb3P(5sG)x8!`eQQK@toKx}x>I$jvjbxT zy#p-*RRZ|~sREYu%(`wJwKiD`tPiZgRy(VfRn*F0h0Pb{ZS#b=Rb7zyzv8bz{1u45 z0{_>qKoTRHt`F$kukV0CZM*ax(7XL0cab(Jv8syyR*fzj9b1g79vWK=Z|oaebhdSk zE!w-=#uh^do5dD`$Lhrv1E;IS7Oe~AVvFXrVzEWzPQKWpe3&h^s6S2@TNKZe#TK={ z+=NE9(0{*lfy71@UGFovciWDA`wZ$lY|ubAgEld?m{@y1wiwXHB~m|K{QIXzi@z6J z%QZ5#mUcu;EmChtY^~y;*jkYRF|}})KC!hHy<%&%d&Jb7QC(td{W`_gI(CSu*|XZj z)+V)zt&MmmrWRV+G`2Rsacpg7!v z*2<-htrbieQ`0*pjjc6J99yfAFs3F(L}P2c!?Cq?c1%s1X~ovwHxe3I#D9CO4%P9R z<}1H#pZ2{wt5^So*kWRBd~7kGjm59>@1GtmJ}S1B>)qH|+Tk&^NWHlj<>*gmFa&uSf8o76J4HljsLEwr*pY;As{*xJkn zF}2{nIQVmzXI`BApQ!(UxD~5@PF_M=;~{S+*xWB{oOy*_W-!6o|b(; zJ^z1|diwt?%>S=;m#GPW&)wPXboB&)@$M)!1JK{?>2_An0BGShQd0oc+zM_f^%Q`7 zZVojEkj71>&I>rM8GWlJ0iH!4MenO80bGfmQ?md^qx+*f)w2NBM^~z8fCbTc(NEOV z047Dps(FCH(Z12{>RdtVXfrhtP%By`T24I?pinfAI%SX{nkt$^Jrf`lm1-*BMdYbE zb8s(mBXUX21)PW+j_g&>1=y@kAgqWiiF_HEqn->fH8MfX1`Jo{5qhg<1GJB{RMP?V zA~hnF)YAcqsqY?;Ba$(aDv~%7iCE#c>gfQF)meobYC7O#_^>*$uqC`UyiCmo%n45q zPYRC?4^^iZx`x|^n}zGCGYsXz#lrc*+0;pfWMMZPaNer(43C}r&JE{+I@NI4+3jp` z)~d4&3!FL5bZ3$};V{(c>vVP6s&fwYoN7)vryYUk~yvuu;1D*?8o+f`-XkN zK4~AeciUU+we~W5fj!5bZcnmD+e7WXc2~Qt-OR3MSF_96#q4}`Hanf2%y#WS=xyjl z=yB+N=tk&5=w#?{Xm@B!Xl-a&XhCRBXnJT;Xmn_3sBfrisBNfOs9vaAs9dO6C|@XB zC|xL7$PER8Z-Xy_kAwGvH-Z;}CxeHByMtSTYlF*z3xack(}R4M3EZZHsd8+Z|T9Jn925x5XI88{r+9oQ0B8(0=t5SSB~9+(su z9T*zu8|WHn8)z1&7pN8}7bq6U7swV!7f2Rx0|D!;^}>2=-M4O77p#-kVQaUw#ae4E zQ>QcLSktXZ)@W;})z|83wY8dA^{i@EIjfkJ&&p<{vyxe^6)@kba~qG%`{oVvf_c(B zZ0lGnwg{0pqRl!gy@l zH*OdgjFZM;W4E!zSZgdZ78rAk>Bb~uv@z7^Yjic*8qJJ)Mm3|HQOw9^WHZtk$qd&B z$hY!^d@S$F8}foYDG$rta*JFmm&pZkj+`zh$o@;+Vi`L-D=9X+v>TtiabfB5>MJ92PijC=LmnHWUX1P8*5? z0;dhdeu2}5VxPciL$O!jw4vA|aN1Dp7C3Dvb_tv|6gve@8;bA57x;B{2%I()+XYS= zifsa?4aHW0(}rS;z-dGAt-xtRv031>q1YsF+E8p1IBh642%I()>jh35igf~~4aHh9 z9ly>RfzyU!wZLgZu}a{yp;#$!+E9EWaN1C;5IAipmJ6IV6w3ro8;Yd@rwzqofzyU! zk-%v~u|VLop_nhm;n(>>;IyImT#Uhfo)``1iud5BVicS$M#5R*UHFL@0Y4VQ;Y=|M z&JaW4bTI^eBnHE2Vi5dL41`m~0QiCE52uKJaI)wNCy73AqUa4Lh+go0(G!jrJ>WRe z9gY><;26;rjuu_ud!jQOB|5>8q9c4)bbup7dpKOQgTq8yI8?NOLquyhShRwJL`yhO zyaNY_7O=l)4*Q8_u&-ze`-mp6w`dG|iAJ!eXb5|V2C%!R54(wau&byGyNEilv#1R_ ziCVCus0llW8nC^n4%>-pu&t;H+lVT#wWthRiAu1gs0iN?6<`Zd9yS-{U^7t`HWg)H z6HyvA7NuY#Q4%&3C13+l9M%`bU_DV3))hry9Z?w87KLCfNA666jU{#S9 zRuOq%Wsw_J61iYSkrP%BIbeB_9hMW>U|EqBmJwNCX^|P05}9C0kr9^grwv4Lf7(D4 z^QR3&QGeP%6!E7GL}7p0Kos()4MahI+CUWWrwv4Yf7(Fg^QR3&UVqv^W?_|pc$_NNU*(4RICrax^U z41d}{NPpTu=>D{U5dO4*(EMow?X5p;p#9@d8)&cnX#?$rKW(7>=}#MI&;4lw?GJz2 zKzrs-8)(1#(+1j8f7(EM;!hiBkNs%_?KgkgKzrm*8)(1!(+1iv{cGaIY(60E?2HItR+CaPHPa9|#{b>X3fNX;HqcJ^(+1jcf7(Dh=1&`F-}}=B+EIVnKs(}38)%39 zX#?$$KW(5L^rsE91OBvuw%?yN(DwP$2HIYK+CbamPa9~v{b>Vjmp^Tw?ewP&wD0_B z18s*tZJ=%Urwz1i{-}j1 zZJj@Dpsn?%4YW1>w1KwTpEl4|`qKv5a(~)DTjoz2XiNQR1MMq++CW?EPa9~9{AmMi zfj@1a&G)Abv@iT=1MPEv+CclvpEl6u`O^m4Tz}d?o8wO#XrKDi2HI?Y+CZD-Pa9~T z_|pd3$Nsc|Hq)Or&}R742HJFg+CclrpEl5@`O^m4hyJvIHr1at&_3{|4YVo#w1GC+ zpEl4YVcI~Oh-m|D0;UbL_c3jtjmNZsHV)GU+E`2*Xk&(X6V#)Jdh2^bymi!IZyh+t^GI;@|!4(;o$L;85@;NIRksF$}6?CGrodU$L9?%vw3o45Au>aBgc zcx&&@-rB2^xAyGltvxz;Yxnlv+O3_pc5UmeUD|kS=hoiZsg<{OZ0W5X-tpGfTznnzz=e>aDe_cx$c7-deMgx7MiWt<@`dYqj#;TD6?FRw?VPmCJZ* zrPAJ7v6Qz~DCw=`OL%L!;@(=en75WG>aC@Vcx$P`-deJdx0WdAt;Gv?Yq9*^S~Q=x z7Rl?ah4Xl8q1@hDFqgL$$my;5a(HXr?B1Fuo44l9>aDr5cx%qg-kKwmw`R}it=Teo zYu5DMnkAjLW=`v^nbLS`#?;=LA(gkLPwB1cQh00HaD4gcx%eU-kL)7 z;&@}|PKpj!KgOHG$hq(>^<%t245xOgs~_V{qTM01R{a=n6x9j%r0U0bvuM4xMyVg; z4Wn7XxT}7QH;rs52|KY9LNeE&Z>zW|Mxz> zitqp9`+x8MgZTdcB|cAy@Bc4&*CD?DkMIBS|3Q5Jue!MS{y)C||KIHY6M3HkBv0tA z$r5;L(l~ET;(BXhJpV5dp8uB+&;Lt+=l{jo-rtR4{NKUz|7?u^hcNyh#Q1*zu8b3-#ilS5-d!$SQ+-9qg`%|rD=)kEb&#Y6c+*+c0=$wP6WU`PwT3_b}y2;K}{ z44w)e3GNAQ4Xz6=4=xPO4bBKo4vq;93-$|k3$_b357rM>50(!W59SYM52g<$55@(9 zK`rny@Fegca5Hc*a4K*luqUuJur9DXurM$;Fe5NIFeWf8&@a#}&@RwCP(M&TP(Dyx zJx3vXAblWtATAILXx2;XiS@v`XdnswcJ{0&9!D&ldUnG+>lh~0B}Q6z9(@*QjU_iAt^^n+>n&-O5Bi?BP4D}%Ha|>B-O8u8n$5ByLE`{t`DN)xSG#NXotvHzZ{ri5rr#x5N!e*-PSv zr0gkiLsIsTxFIRKOWcrDzjNG>lwBomNXjk}HzZ|ei5rr#lf(^4*-_$#r0gJZLsGVv zxFM;o8*WI-wh}iaWgCeblCrhL4N2KbX25l9DbvGuWIEVFriIOA8rV#xhD~KE*hHp; zjb#ehNG693Wir@6CWZB75?D_rhIM5kSVty=wPgZWOUA*P(uFl-6jqlJSWSjuRq4Pg z(uS2~2v(9oSWyOG1!=+Z(u8HD0n11UOG_P=k^+{L8Z4o|g~j!MU@`p-EUNzvi|BvB z!uo4iNPh(j>S#!00sRH``Sm|xKK(h&tN#J>=+9to{dbs4e+qNzPhbxHG0d+22D9mp zU{?KCm_`2uX4Zd(ne>M+qy7M9(C@?a`cE*Oeh;S6@50pj9hgeL4O8m3U<&;vOs?O6 z$@J?mseTP6(XYb9`W2W+zYG)VmtX?@B8=0~kVscQk9|}>2P68AFsz@2j(!H(`e_)_ zPr;ym5(e}iprxOHrhXh6`Y|Z=@1d?Ag+f09HT|%s{#HK(|IrV^zx4y~mA)Uo)c3&` z`d;{_z6U6_qP zeIvZ1Z-BS;_3)Oy4&Kz)!W;S;cwJu&uj#AcRedGAqJIM~>nq?TeL1|SFM}8Kui<%p zDLkit1%K3+z_a>dct&3YPwNZeDSZJvsn3T$=wHGU`WNuH{y99Re+IwT=fR`;TzEvE z0}tz;!bAFOcu=1O59pu3{rbmnpFR`r)n~vx`gFKk{|N5Vr@@{2hwwXnD%_!e0JrN? z;5L0S+^SE4Tl9(WTYUoDtiKO8>Eq!>eH`4NkA>^?F>swe8m`sfgKP9raJ4=XuF~Iy zEAI(c8kw zdK)-NZw)8vt>6T`C466h2aeZUz;SwWI96{4$LLMrXuS!1Pj3uI>5bq>y&-&8ZvaQ= z_2F>69vr6Eg+ui^aEM+T4%TbIL3&L%P_F?8=+$9=y&CMNSA~7`DzJ}U8TQsI!Crbr z*i)|nd+6n1cfB0!rk90X^)j%FUK)1ROTkWhN!U>@0Xyi$VSBw8Y^N86ZS^9sjb0eG z)(gQ_dO_GyF96@s^TQT;KGJ&kF15 zSzui~GpwU$g0=OGu$G1kk9JvFSXr-GIAl(3?n0#?wI!}5AE zSWZt0%j!vB89gy9ttWz|^n|dao&c86<6v>!g~jwJEUHIf5j_kG>kcfW+pwS>f(7&- z%&!MvKHY+Obra^%4VYV(Fqe9?qW9-SPF=tpx(2h0w=kRd2WAy-U>5N=%q0GT8O3Xu zLA-+L#Y>n@ynt!NpD>Mh4pWOiU@GwprWC)!6yhmNE}p<-;xSAreuGKGBbZqH3KNN6 zU_$XTOduY@IPn0w;y#RupI}7XgJE$OI^qtr#cdc8w_s4*gaL5_TH-o1#WiS%t56qL zJjGja8U7)i7YJf|juon#MQKFjhcm zEQh+W3<~3Gs2NK=)x5}8@E>Cdd}Azze;bS7U&ccC+E@Tz8S~*w<4gF$_yYcEd=8%* zpTR$jdGMJr7yfR{flrK2;Ui-<{MDERe=$CRKN}yzhsI3!z?cE=8`I%W#z*kBF%8}@ zK7`kesqmWd0laEVfme*l@Uk%pUNR=ai^c?a!FV5@H^#$r#yI$+F&3UR#=tYiXn5Lq z51ukc!IQ>F_=E8-JYkH0$Bp6em@y1~Zw!S;jUn)eF&G{;2EjwdKzPs?01p`b;eMkZ z+-LNKdyPJDkI@_MHhRHbMo+lY=mEboy2BktH@MyC3bz?u;8vqE++uWs-x?j^W}^e# zWVDAHjdpN@(H5>Z+Q4;2Yq-{E1=koY;cDX@xXNe&R~pUXH%2qK!e|PY8%^LcqcQy2 zXatuU4dGWt1GvPf4;LHt;3A_gTxisR3yj)uzEKN)Y1D*Y7&YMMMs@g^Q4P*Bs=~QO z6*$MJ3_mq0!P!Pd_=!;g&M?Zu=|(v?%_s{$G|IpajM8w5Q3_5nO2Uao3HZKI9F8}N z!Er`WIMyfv#~6j-XrmB(&nO5-83o`-BR_oC$OlIldEsy)4;*IXhC_{9aEOr;4mNVY zK}L2s(8vY{7+GO|BMaOt6oU5%xARz+Ogr*waV{dl+e9cOwn#W~7E)ja0CU zkrH+`Qov3|a@f&G20Iu@VS6J9Y-c2fZH+{*jgb(xHWI*AMjUKuxbPh#3R@Tv*xU%i zW`+Zs8a8ZVgkWPM2pbsz*wCMZ9{{#qp_l?P!|9)JV=*Yd$ zmV01G?uJ3R3kKv)Xvy!ODR)3aZiiBCgSy-bh1>!)`K_m#v)l~-k(=Nfxe@*?H^9H- zdiYwdgRkUT_)@Ncf6CSHnOp^bmn-2@`3-y`SHQ<|Is8p7gOB9b@K?DMK9FC*dvXc9 zD;L8%auK{O7s6X|0lX>a!yEETcwK%0ugTBhRrwjbBIm)&axT0i=fI2dQ+PqnhUeuh zcusx-f0Q4?vvMXpBWJ+VaymREKY}OaH28!35T1}z;c@u^JSL~W@8x88R8E3NJra zSIDk#x$FX$$hlLzd-*o~xX8mp<$E>Foa!j3_ zK#r+rARx!A-z?;q^~gewS-)DyG3yr#IcEKAA;+wT7II9Tyg-hrCnzAtte-68n041e zj#;-X$HU&Q)fPqW9r!r$T8~&3pr+;u#jWcaSJ(S9kY;Q*7p{2OnnCqLG7 zcB=~9W>tn;tx9lSQ% zRSd4Qio$QKB5;LO7%sO8!DUuK__b94F17N*udIAFw(`J5R&Kb^$^{o#IpKUO z2mI2?4!^Ln!OyL%@G~n5oM&Z*bFEBpj+GI9YGr`4t@Ln~l@5MlrG+0`Y2ZvNHJo9k zg43;(@FOb)oMt75A6m)aR4Xa`z)AwASc%~zD-nF(N(jeW3E)^O4vw*0_?{JoqpS#g z*9yZCmIH@bHXLe&;1DYa2U`I+$g}v_w$I@VL^DXRU{sViO zZ(tAeZ`j@Z3wASK!>;Bl*u{JaJDV?HC-YC((R>a&n18_b<}=vN{2jJ6pTaig6WH2( z3|pDM!ItJD_>TE2Y+?Qao0~twX68fK)O-M&nD=30^C#HIyayYacVPqb4yEM^{sMa?6yhC6oUr6-;ifgvrcrU{Z4hOkysF ziOpp&k@+=DXfA~b%&%aaxdgiAVi+|S!HBsKI_3gs{}1-=I=GH2S{MA0bQ(6x%*@Qp z%*@Qp%*@Qp%#N9vnPPSfF*Dn-W5;>j-_hXBzxvOojg#Q{Ycy zGW=mof`1zm;df&K{AP@Ye;MQ8S7R*vVvK>GjnVLvF$#V(M#2xq2>9L@4&NEW;9FxT zd}9oOuZ_X*r7;LTGX}z^#sK)l=no$o{oo^`FMMe9fqxpk;RB-={KM!8Zy7z{O`|)! zVRVDnjjr&T(FI;LI>XV)a4%}zdhI@@# zaF0}te-U5x0kvk?t;GNQtcMikh=2#4(r z1=|@CwlxH7V{q8oV6c@zVM~L+7KRO*8y0M4n6Rl~z$QizI*(6)jf_y(&{(^Ob|G+xIpRjiD2dowR8`cbdhc$xVVD;c%uv+jdtQ!0Rs{}v8%E3>tQt%_J z82kV$1mDB*!FRA+@GUGGd;`k_U&GSDSFlv@B`g_y0ZRm*!{WhbuvqXZEE;^`YJNBS z`|E#y{qL{;|7+`id!3CrX0Np|$LuvW=9qH_2IiRaBt^_Id!>yzX0Na@$L!_K8I{hz zhxRfXbIe|9V~*KNY|Js|{te7A=Q)g+WA;MlS&Q!XFR(Gk?D;n4m_5(N9JA-zm}Aa4 zn3!YE-5;1^&QlyQ$LyIF=9oRh!W^@wTbN_^Gz)Xgo@!x^*;6dcF?+IwIc86?FvskP z7Uq~e!NMG~$6J_V_Bac3%pPlDj@e@@%rSekg*j%Aa(0)vzweP2=9oRg!W^@QTbN_^ zFbi|c9%^BZ*+VSMF?+CuIc5*CFvskH7Uq~ez``7}`&pP{c5mx3UcZ<12==fZ!tU0e zu&ea|cCqfmPS!ox(Yg!UTYtcI)*aZ^x((Y{w_t1QCTwNhfGw@-u!VIEHn*gY~Slu m*0D~*+SVyp%Q^{bS|?x)>o}}# z9fQ@Zqp+%V1Xi&Q!^+knSjjpFD_RF&1#3SnZ|#HSti7(au{OiP)+SiU`VAJeHo^kd2AJPk5A#{;U|wr2%ww&AxvkYO zm$eG!v{u3#)(V*2S`M>W%V1V(Da>LmftjtvFq5?iX0#T<4Auge-kJ~7S@U38Yc5P< z&4H<{*)Wwg3#PPY!W7mFnB1BUlUdVXQfn$qY)yfQtjRE;H3=rLCc^mE1Q^d6593s{KjBSm9v8>TBrZo!2utvh@)(9BQ8V;jc!(bF^C=9oTKxGYv(i#MXH4t)Z0AyBw zNUeU5Sbd>w^?{bv8=6)xXjnaA(CPuhtnM(>>IOrsuCC6s=>mUQo#8)LC-~Fq2!B`| z;NMnz_}yv;zgcbJUsfCV)oKmDSgqh^t0nwowSXV3=J12n48FIT!gp2^_||F+-&l>{ zYpWrAWi@~=t@`kVRS!P5>cVGM9r)C$4WC%G;A5*Md}P&t53TC(PpcYyU{!_ptSa!1 zRT03;dQGVyk?b!SFJMel2sa>w@SftR!Ml)Dgn<}#o=kI7(8Va zg(s~d@Pt(u9=8g?V^%?U)G7dvSoz^$D<3>$<%I{WJn(>(8}7Gq!F^UvxYx=7_gLBC zZYvwyWo3mstt@bdl^JfgGQn+DM!40=0Jm7_;btox++?MNzgcPEMk_VkV5NfVt(0(` zl>)A{lEXDtGPv4G3RhW4;7ThoTwx`G%dLcPnUw%8wc^7iRy?@aiVGK6ao|ELHe6uE zg7dAIaGn(d&b6Y$IaV|{+lmTjSyA9jD;&R_Hi^0hjg_A4-Ct5a~ zU|DdyWx{cm0moWFIK~QtqpeUl$_jxatpFTh{tJhjzu++QA2`JP2?v@#-~jV)*w6e9 z`3Y(ixU^DYEY-&D&P0WX|vH2%#WIlim&HJ!{c@Ne%@4|ZK zAF!@@2i7rf!`kL8Sj)T#YnnG;4f8szZeD}c%&V}f`8%v)UV)X(%dnDp305>O!V2aE zSl&Dj%bDk3S@SF`W1fMf&C{@yc?y;^Pr?%B30T}b4vU$`U{UiZEMgvkh0VjTka-9e zG!Mc8<^h=B+z<1a`(R#kFU(`^fw|4yFqgRt<}`Q09Oe#~-P{hdncHAib1TeZZh@K2 z%`lU>31&2ZgBi??Ful0}rZd;WwB|aP##{?in`>Z7b2Ut1u7b(Ul`xsP0wy(=!zAW1 znAltj6PZh3LUS=pU@n63&4npueW3P>wka!@2~&=YwLg7h+vM< zh6HnrHXxW|v_8QcqxA^p7_Cb%$7mgbIYw&}%rRPvV2;t61apkmAedvcI>8*H)d=Po ztx7P*XcdAvMk^D{F2035-G- z!*J3FD$)>2(f|rlA97L;GEx^(QU?-J8``86v`9^8k{Zw;)nSlSgJGm93?)@y2&oJM zq>`(1?np)Wi&TLBkn->+DF=U$vhZ(G27V`{;WttW{zXc{ucQS0LW;vrq!|22ioy@1 z2z*Zp!*`?*d`k+#H>3c3P4dH6Bp-Z9^1>G+4}4B?!)GKHd`fb{CnN`aOtQmABpZB4 zvcf+}7WjZH#Sk$CVJi3<;tIPegO4G)r7@BoPk_mdcK zABhh4l4x)bi3)dPP_D49!{s3p&@8K-_9h_;ug){6oaJu~(PP1RZsrE}a#eM-N+t1-7 z`x%^QKZO(QCvd#|7>=_a!Ljy3IL7`Hj6gIPuz^3+L*u*{r8`}qABl`esXzzy& z?0vAly%*NA_rSXLZdk|O1#81wW^t|vxMp^)2)Jf)t_Zkhbgl@vW^k?uxTbfm2)L$mt_ZlMb*>1w zrg5$axTbcl2)L$lt_ZlMbgl@vrf{wZxF&b52)HJ5t_Zj$b*>1wCULF^xF&Y42)HJ4 zt_Zj$bgl@vCb0X%_|BC9_w%@RUl_;k17kZ^3f!;9vU|apc25|??g69Q-C;Dl8;ojq zg;DG-Fx>78mE8$SyCW2K2gvRAklF1ZwcA2sw}H0Z8d`QMXxc5IVYh%myEzQAo54`K zDGafjz<}M@)j6-L5&UI0g#XwL;7_|g{9)IF-|f2an_UO~W!HvZ?OO1QT@!w`Yrs!- zb@HE(c%OW#MzX418vnhEMHM@QGa# zKDJB1M|N@e&@Kl5w2Q(Ab`f~rE)4J4h2UMgApFBF0PooO;cYt~ylLl!*X%s-s+}AD zZs&qm?40njodaI7v%`yaHh96#3eVVC;7L0(JYi>o$L);pn4JM0wbR2Rb~>iH`#IEZ+2|B(T?Rx8>^j`3&=bBsr2m}5K&!yMz`40DVthB?M1!yMy+VUBUmFvmD!m}8tW%rR~= z%rWOgO3X2CGR!e%rW+hVUDqX80HxJ z$uP&*4~99${$`kC>^s98bDpAsIp&;(i8;o;GR!gdg<+1d&kS>nePWnn>?6Y*V;>mi znDY!y%rWOIP0TU&mSK*uHw<%(y=ItW>=nZtV=o!z7<<8Dy7SCAfinht%A&(3EE;^w zqQXZk3Vg`I;h#*w2Ta2IOu&1L!@G>ZKNy8~7=gE$4R0|E-ee}c!3=nv1>rRo2CuSE z_&W=MS6BdErvJiA^cTEH|A80iPk5gGfamDn@GSif&(Lr1H2n*nqF><&`UM`PpWzYu z2_B{&;UW3~9;EN#0s0Q^r*Gjt`UdWzui;a2(rZlTZNX8H_nqEF#(^a6KYx(yDXTVa2?1@@zxVPCol_MyMQ-gG1EMK{2nbUo}r*TL>|E$l|u zz^-&P>_S(;&U7X0L|4F$bUExmm%;XQDQriVz&3O-Y)Kcv7IYzOMi;=QbUtiM=fOsF zE^I*O!1{DHtV?IXI&>ziO=rMbbULg_r@OOeerfbUds` z$H5A8EG$pQz;bjnEK5hhGIS&?O-H~|bT}+YhrtqbC@fBgz+!YTEJ_E#B6J`uOb5V1 zv_C9J`@sUVFU(K-z%j!HE{sp>z<9Jaj7w|5IJ72=O>4kdv^tDQtHBtwDvVC6z-Y8G zj7lrPD6}FBrxl>2<)NVEAg5&^qh%nar6HlEpiN6ciUEe;J@3ZzfLqDma0~ekHe+M5u8UJ!nx#6IEOrdv&nrpi`;`V$z3>u`~jzvJ8&Ag4X2V@a0at;nBXW=k%1`Z{s;Sh2P z4kjnzAaVi@B*)Q0xvJZAAdto=S2X-a9 zVHdIsb|O1rd$I$zBimscvJJK-TVYGG1-2lYVKcJHmB(YH{Pn-T{`c4a|F!kM_?=^p zi7Om)OkC!eW8xCW91|Bg=9sv^F~`JtjyWdIam+DsmSc{IGaPeFoaUHg;uOam6DK+5 zm^i^P$HZ}tIVO&A%rSA4V~&X<9CJ(@=9pvR5XT%72RY`LIKVN-#D0!BCiZd6F|mhZ zj)|Qdb4={um}6o)#~c&eIOdqx$}z{p7LGY4Hgn7|v58}jiS-JjWB39%n$L%$_&hk0&xIrS95|fMhQs(QIF!$XL--6hm`{g;_%t|> zPlW^c6xg3nhW+>?*q2X)efR{}n~#UR_&C^;kA*$>7}%YUhTZrm*p-ijUHAytnGc7Z z_%PU!4}~505ZIm%hVA$u*p?53ZTJA#n)ipTct6;Z_k}HZAK0AthRt{{*p&B#O?VI3 zn0JSbcsJO9cZGF%7g&dPhP8MnSd({z)p-Y4jkkwYcsp2`w}lmX8(4w2hUIxHSdO=Z zWqAu&hBt?$c{5mwH-#m66Ig;bhQ)a!Sd2G>MR@~Qgx80Kc|BN&*M$Xn9aw#FyQF#U!g{Oz%JRMX# zEtEVB6g)NLJQZX-C8RtBBs@8^c`|76q|oF^purQvAWsBCc|sV%6TkqE@5=w>@!&5W z7yiTJz@I!e{J~?vzj;jfoyUORcy#y|j|RW;sPGGq0zdO`_=zj{kxTf23;3RM_>MF9 zmQ(nK6Zo3j@D;b?^#!xxQv~GOW8@dgq?tk*>Sjt9fJ$mQMiB|f%Dm6IFB8I zbJ;;ShaG^k*?u^S?SnJfUO0p8fz#P;IF0RsQ`t^9h3$Zo*>*UIZG#ipRycuef#ca` zIF4klijey}3z3oEcbusrJx%duXtEb9r& zupY29>kdn?Zm=Zl3QMpqusG`si?L3yDC-D|unw>=YYz*tcCa983k$F|Fh6Sz^RZSi zFKY?&uof^kYYuadc z)3I7GEvpIBuo^Hms}57KYA_|M3RAEuFgdFXld(!LDXR#RunI6SD-RQ~axft)3lp$1 zuHq_xK-IU9S^oKd{`r6JoqMbwIQKbUa_%+u&;R>>b^f0kDKW>?2#GnShD*#bHB4fT zsi6{cObwBkV`{L(98-fN=9n5NF~`&Zi8-eFOUyCVPhyU#z7lgx^^urks<*@(Q@teS znCdAp$5aoAIi|Wx%rVtXVveb<5_3#-k(gtuv&0-zoh0U%>L@YCR0oMUrrJx)G1X3D zj;Xd1b4<07m}9E7#2iztB<7fEDKW=X3yC?VnoGpCmW5y`SrC?x1z>TR9~PDQU=f)Y7M6KnA(=zj=9XDtE}0qTl$l@-nGt4}8DKV<9%hy4U>2DcW|nDSCYc&$ zl&N3_nG&X#DPTI89Hy1YU>cberj|)yDw!Cjl!;&pnGhzI31BiAA10OYU=kS@CYEtv zA{iSdl(Ap}8572rFU>q40#+FfFEEx`CN(Ey`38PB^qe%{&05QPOO1z#cH@xtb)tLO1M<4fJ?-3xL7QMi^Nj6P%ME9#9}yK zEQ0gILO53}fOEurI9tqvv&39DQ_O)g#B4ZS%!1RzOgL4{fK$YDI9W`Clf+awQA~jo z#AG;LOoHRYL^xJVfMdjXI9iN@qr_M^QjCEk#ArBNjDo|&NH|oCfJ4M^I9LpWgTzoc zPz-?s#9-K841)c{K-gCdfPF-N*jx02y+mKwQ}ls7L~qz#^n%?)PuNxTfL%m)*jaRg zokUmIQFMVFL}%Dubb{?fN7z<$fNey3*jluMtwdYcQnZ0BL~Gbww1UkqU*MP*n*RD#7tMOaK!fJH@lSVWYAg+*CdNR)vEMQK<-l!Ez1 zNtjQRfO$o6m`4Mwm=wfJsGqm_($5iA7qNNTh)Y zMQWHpq=NB9N*GV1fN@1~7)K<7u|-lCOC*6YMPe93B!baJLKsaXfKf$!7)8W`;UX?n zA`X-yHWVTjJq zOL&jJfOq+Gc!xiOxA{|ei$8%k`D1v4KZ4i!L+AW*TaA)^-0S}VI}lLc)O+<@Jy18* zC3Q+2QoGb9wOTD!v(;oZS`AjcRcF;&HCDA%WmQ@gR=HJXm0BfMu~oRTRX~1|@8xs( zK;D#>mXqaZIav0Von>p;Sk{)6WocPh=9ZadYMEHZmf_Nt0r5?| z7th55aZ_9pr^F$#OKcLW#bPmAOctZXV9{H27Oh2NQCn0NrA1+pTVxigMPdFoQFVe7R7W^ob%5hkdpK6LgJV=%I9j!Vqf~1+Qni93R7*HqwSdD^b2wBr zgF_V7G2w$1)-mCO6xK1}0~OXW;R6)bG2#6c)-mDz6xK1}eHGR*;e8a=G2y*c9lTyI zRU7tHwO|ib6LwcMU^i7Ac2(727gZH@R#jjpRT*|vm0$-|5w=$qU^`VFwpHa|8&wvz zR%Ku-RT{QbrCciOn4)Obxe3eg>_7L1BG==czuO+ zOn5zobxe3&g>_7L9ffsFcx{yjuUAXuhBZ|#SVQH6)m08yO=X8wRW?{fWrdYh7FbDT zh80yNSV3ik>+8dy}NhDB5=SV*OW1yu@|UnPh6R5F-XVI32m zM`0Zko?BrZ6P`<99TT2YVI32mLtz~go?T%b6P`_B9TT2aVI32mMa9AEWmd6aCKU^2 zR54)&6$7SM(P26j4W?C5VHy<$rdHuFl~ORJk}!o5FuCF|nPM=hqA-afFtM^>B4xpZ z%7h7&0pqJ6jHkk2T!nQ^cpQawOn7XCbxe3HiFHhPOo??&cnpbkOn7vObxe3PiFHhP zREc#=cog{^uNN-AK_&l!QhtR(et}$mhD?5fRDOg+et@=o4=wo)n({3)xUX}ad?{Xi!BKN|}au2*Hcf$*E7d$U_!gF#5JS(@uGjba|Ew{o`atk~uH^UQh z6Fe?|gU94XcvNnHN91~VSgwPI&VlRXY`9j=f@|bVxLVGDtK@XJQci;_ zI8RQ1bLDtAM~;KD|MYpZV5*Rit}Qxu>e{_g z%T8T8H0d5Hq6Slu|LfNnUcPgSu5CIs4>hUt@cIj(wf?)Ol}Uwwk6UsxfM~8l?KF z9?o3>?Nm$EMAcWdR8>{Mxi_GgDyZ_P>?)H=t5P_32*gt{RTRaQr9$OD&V2%(oySLYR-I}eKT{5+>~??GCgoF{bdK8VUWH=QT1|Lr^-{VjXW z9)A@Sn9XI=*+e#)4Q2gVPu7{WWzAVbR-0927k z?MBQLb=KxSwTDw#V2*?S6K5yMx`*Ze-WCtJr1jVs?Hzhn>+*Whb`d z*img}2c6UFzd9$%KX*=fzwMm7e$F`!{eW{q`6lPo@TJa4-qW4awMRN9UiWrRp>FG( z3|-$jt+}Fef^uQ!RO779NyEvV(|=<+C-T}>i21kq$$V`-G4Gi-%uD8J^N6|E+-7bt zSDK5=+2#~;tU1i=Z}u=dnyt*nW*xJtSpVH`Ae8Jmr@#xi5RF~gW>j4}oreT=R~JEOVLz^G|dGD;gojJ!rRBfXKr zNNB_|!VO}C27d%U2j2vr2JZ)N1}_KC1dj&y1-Az`23G|a2j>K*2FC@52L}Xu20H~? z2b%=z2CD_j2TKGC26F{72h#+T2IB>z2Sv~f``3B4;fJu7fq)t2yyX}2Uh_`#R`W*l zTJuWtQu9LdT=PuxRP#jhSo28pQ1hqef#$yEp60IR56vCTZOtvsP0bC>bow~%Yc*>$t2L`MD>W-L%Qee1OEpV0i#3Zh z3pEQg^ELA{b2W1`vo*6cGc_|b(>2pHQ#DgGlQokx6Ezbw<2BM$mqcx*6BQ+y5 z!!^S+Lp4J*gEfOR12qFQ{WbkGeKmbFy*0fwJvBWv-8J1bT{T@aoi&{_9W@;^?KSN* zZ8dE)tu?JQEj2AP%{9$5O*Kt4jWvxl4K)ok^)>Z0bv1P~wKcUgH8nLf)iu>LRW(&K zl{J+#6*Uz!V z;9t!z%|Du-njc0O2@U*>`W^Kf>R+f|QNN&mM*W2P5%mM=d(?NRZ&BZ%zD9k8`V#d8 z>T}d*s83O!pgu-@g!&NmPt*sf_fhYm-bMWb^$zN7)LW=GQE#ALN4)2E2x)I zFQHyUy?}Zi^&IM1)HA53QBR?sL_L9e9Q7FLQPd--hfxoq9z;EWx*v5P>R!}6sJl^j zq3%T8fw~=a8|qfnEvTDOH=+K9x)F5)>Uz|5sB2Nzpsq$;g}M@T1?qCtWvEM0m!K|2 zU4*(2bph&p)Oo0LQRkq}MxBK^6Lkjabku36Q&Fd&PDY)CIuUgO>Uh*~sAEybppHfz zg*p;-1nO|qVW>ky|!l>Ft(YZt3Zk9&YLGmTqq8 z>Xt5U>FkzHZt3Wj4sL1hmUeDw>y|cdY3-I)ZfWV37H(Xs&MY3!CpZfWS2 z25zbEmU?cf>y|ohsqL0pZmH>(8g8lXmTGRP>Xs^QsqB_YZmH;&3T`RymU3<>>y|QZ zDeab0ZYk-O5^gE(mSS!x>XssIDeRU)ZYk)N0&dCgmV9o>>y|uj$?cY0ZprDE9B#?( zmTYdx>Xs~S$?TR)ZprAD3~ov9mUM1O>y|WbN$r+YZb|8u6mCiGmSk>8>XsyKN$i$H zZb|5t1a67%mUwQ7>y|ifiS3qHZi(rZ7;cH~mS}E?>Xs;O33rQfi*$={3wH~13v~-| zi|rQ6Ev8!xw*;NyEcPS*SLMBX|NpzN5AOc|ztl&6{qL{;{q?`U{&(Lgc!~e}3VES< zu6d?;s(GS$ta+q)sQFX#KyzPnPjgrEhvtsvw&s@Rrsjs`y5^eZs^)ji70qSMCCx?6 z1?UDa}dE3C(fMG0joU5zS%EAwVE}W)tXhBm6{cr<(g%hrJ5z0#hOK$g_;GL`I>o} zxtcke*_v6JnVK1z>6&SpshTO8$(l);iJA$T@tSd(v6?ZO(V9`3k(v>j;hJHZp_(C@ z!J0vuftmrD{+fQ8zM4Lo-kM&To|=gNsCL(%b<=d!bkTIybkcOxbkMZdw9~ZJw9&NI zw9>THw9quyG}AQIG|@EHG}1KGG|<%7)YH_})X~(|)Y8<{)X-GdRMS+|RMAw{RMJ$` zRM3>yl+%>el+l#dl+u*cl+YB{6w?&d6wwsc6w(yb6wu_?C;q|v0-q|&6+q|hYSB-13-B+(?+B+?|*B+$gy#M8vp z#L>jo#L~pn#Lz_7MAJmoMA3w6ltyZVhHIFHYKX?xSQ=AfXo8wBO{gYB69E4DAL~GW z{qL{;oh6#T{`c4a?o$2#xAlKWnEUeH{{NtQ98lk#)&DCe_#b`;{0{gX@H^ml!0&+H z0lx!&2mB8B9q>Egcfjw!|B@Zh@3el_eA0Z>e9-&6}9)owK2&b6%5h&QNl8@IM!SireCMXZQYbaX{>J&IDK^mWp{|x|rbX+8^Yc z3eZKg70sMI`!z&GQOY?NAdj+H~{Jk;5r|B=0P zcIV$_SDd~1N7x=`XZ~8Y)Y+Fmm5p_F<@aS>ojv(YoE`a!J*m)UdeDfVc4kloYSS>N2QXIHb!*oB;3_37*+b}U=i zVb*tNKm8Nyj&<2OY3;YRIy>nXSu?D0))1?g)xp_AU&pFsm9X+zS*%pf?)hjIG5tz3)+iR@gsbpTb^*-4DAK zb~fyA*v_yGVavkigiQ(?5!NrPOIWM024U60%7hgP%Mq3?EJ;|bFcBa@=fZbrNV$-H zUh37^cC>h7g(Uh3+lE?(;FrA}Vz=%o%`YVW0XUTW*5HePD&rB+^Q z>7^E4YVM_GUTW&4CSGdnrAA(A=%of;s_&(GUaIS*I$o;nrCMI9>7^Q8s_vy~UaIP) zDqgDWrAl6^=%or?Dj$*3hLrVE884OgQYkN$^il~g757pxFBSDt5ib??QXwxD^ilyY z<@ZuPFXi=89xvtgQZ6s$^imElW%p7xFJ<*o7B6M?QYJ5D^il>drT0?0h~(^k^HORr zrSeirFQxEOaxW$GQc^D^@ls+hCGt{2FD39&d@sfGQd}>^@ltFr#qv^2FU9aubT38o zQdBQR@lv>#l$WHJgqOINn3rfovO-pRX_=Q6duf4}=6Y$Cm&$o5t(S7pbF*1CJ4DTbuyU6f1GQ5cluOq{&$nY{Uyoe0X zBg3=E@H8?!i42b;!=uRXFf#lZ86HH2`;p;ZWVjm{{)h~BBE#*-a4Rz0j0`s-!}Z8; zEizn<48KQ)E0N)HWVjR=E=Gn6k>Pw~I2RetMuszy;dEp;6&X%Oh7*zDcw{&h8IDGV zBaz{7WH=NV4n~Fpkzs#i*cTc0Mut6+VRvNM6&ZF$h8>Y%dt}%a8Ma1-Esr|Mi4jhSaD0U0A{-mxmBFHSsiYHE}etHL)}?H8C{NHPJLtHBmI-_%}fW)cv;{(tMQ z|NZs9zy9~v|L%Jr{`x;{h`;_%8{)73oxASz>Ndo=_0CKF`u}H$zy1#mtPl9>f2Rrl z`aj^W|DE?Y{PlkofBm1?U;qENUH?1#{*54U-ir(diH+y~cV*)P>Z^LK9y-_lFQ{W` zkJ_YGs`<`U|IuoobIrf4YOHFi@~WtFy+5N$uHve2Wy+t|EWiFRRMZvY^Z^(>mAnV@fVV#W(R*Ja(?=UlJ$8KCwlt77Lwg`D2~u z@AVYzoqO+VJ6G|Gi@YMU^US^YBC4?YFaFti*4_ht-FeR55x&cL#@=#1*Ll9)Navb; zSKgX8be^kMmKS!Osh8flS|6J`k2iF#)xT#?o#*IXVW*sD=xuYZ(=TQ-oj;`i;dj9A zfZqYX1AYhm4)`7LJK%TVzflJw-dg+=@;M@X4EYq1R)&1=(t9tx^U_-{z46j(FTL{8 zOE10f(sM68^U_lAaWDdFiZ|&Uopxmri-A07UdFiN^j(F*? zmkxR9pqCDKX}_2Dd1i?mo|CnH!p4U(grWB z_tH8qt@Y9xFRk{{s))2aWJN?;7P34dEe=`gr6mz*LC7L6EsRKWL*{#FUPPJ|GRI4^ zBT~7LnO>UVrRiRp=B24#n&PF&UYg{kiC&uErSV=G=cTb;8snwWUK-`4kzN|%rQu#0 z=B1%t8seqFUK-@3fnFNmrT*^uf1&Djz`6E+`@hjB^nWXU2mB8B9q>Egcfjv}-vPe^ zeh2&x_#N;&;CI09!2eVSqFJF~pEgcfjv}-vPe^eh2&x_#OCPqyvWYzCft={QoG*+5i7T zeRfX&d;0(I{J&|=xA-4^2mB8B9q>Egcfjv}-vPe^eh2&x_#N;&;CI09!2bswFvCJa z)DmWdg%Y{S{X}eV4oeLU;ai=fQ$s`8Zs!2j&=7jiIZrh-gdBHHR}BrZ&pKzVhK4u~ zY`UO30NhK7W`bB<~a4GH<;9Mu{c68J&gGXRBq z{XdF)8cEgcfjv} z-vPe^eh2&x_#N;&;CJA^c?Wn{f{;!ewSZdRPYi-OXEcZ_=hc zzF)Gi1jPCIS=^r=@#C{({Leq0Mf~_I89Vo}B3?ZUel3ak@y=(FKi=@Z!Ts@J5kEdl zM(4+?h##LNW4Rg-_>!CWTR?gD^>!_-#ob&!b0p}fpOe(cX z;ye>Dn&QfkzvN%?y?h}b$~(@J0nf`5@}S%$H_NqhnVc_YIL`+hB?rqsva4(-o682W zrmW;VC9sIhE3?V;GKEYiW65wyoM#385TC^x@l@OwH^pUfMjUmX7`R<*6syEyF-J@l z49lPQV~x?7ebi)U;d4MaGoLfi2uQ_@eBMU zKg4(QEqtBxB*6uICZEJd^TE6~@9aEHuraUAEA!IM^8|DA%se$u>^xC0oZCFWzB$hn ze9j)Qo9vSFRKY`R7u&>EJI@uI%_g(aY%uH1IzO>&8xB8SK>vdK9MaIy2Z&j0W`;CI09fZqYX1AYhoH|c;C z78)9!FK`#;4*UUg1@6F{f!i=g;1o9HL8cY+o3R4Guhp7TrV9LN{ zS5+i%2^I`ogarZ@VE({)m@jY+<_(;Mc>-r(mcVJ4IdBSQ3Y>%)11DgHz;T#9aLiSf z3LJ&S14m%7z+qT4a0nI&9E61f2VkMVewZt;59SQ)g*gIyVD`Xnm@TjiW)19g6%_(I zVA;TSSSGLymJV!%r2<=E$-rhb@~2R6cdfekRPbC#F;x_JWYVD7+LS6(f! z238KNhLr-VV8y^nSRt?imJck4VY}1T3|M;8khyE1ZKiAff=xLU^*-nmUIEi+JoBdK&`f*W*Zl)OKZ@v73k0sv~L00H3w~*fi_J+lO~{XW6-D( zXxI=mXaMThcQJd_1KsO_ZgoJ{+Mr7<(77h)R0FiC4q8?NEvka%RY0@KplKx+V?agF zw*u%>9`r5;dX)t|%YYuGL5EVHeM!)+1ZZ0vv?&H!7j+2^D*^@=27?NLfd#>U0-%3> z&@Uh8mKSu*1G?k}opXUsIYGx9E@5M`gOS<5h^%0E7BDO`7@7$T$q4#n0KL&IIfl;CEHiN)Ocl{qtO>)=&pPgp_ zK39*_UFRKu%j&E;t`4f*YOC`ez$&#w%~La+Z}UI=4)`7LJK%S~?||O{zXN^;{0{gX z@H^ml!0&+Hf&Y33a5A4lm8b%hqcT*AN>FW73)Mt5P=l!dan2uczTGAv(NLqJMnMgC z*Z)EF!oC0hi+bk-|HJQq-vPe^eh2&x_#N;&;CI09fZqYX1AYhm4)`7TU$+D9BmbfW S-5NE_tx^8Rm{^z@IRyZ%eOJT) diff --git a/sparta/test/SyncPort/SyncPort_test.cpp b/sparta/test/SyncPort/SyncPort_test.cpp index 48c1c5eae8..92523e5cfc 100644 --- a/sparta/test/SyncPort/SyncPort_test.cpp +++ b/sparta/test/SyncPort/SyncPort_test.cpp @@ -13,6 +13,7 @@ #include "sparta/simulation/ParameterSet.hpp" #include "sparta/events/Event.hpp" #include "sparta/log/Tap.hpp" +#include "sparta/collection/PipelineCollector.hpp" #include #include @@ -20,9 +21,6 @@ TEST_INIT -// Pipeout generation does not work with this test -#define PIPEOUT_GEN - // Be verbose -- very verbose // #define MAKE_NOISE @@ -164,7 +162,6 @@ class TestSystem std::unique_ptr master_tn; std::unique_ptr slave_tn; std::unique_ptr pc; - }; ////////////////////////////////////////////////////////////////////// @@ -179,10 +176,8 @@ Unit::Unit(sparta::TreeNode* node, const Unit::ParameterSet*) : ev_set(node), ev_do_work(&ev_set, "unit_do_work_event", CREATE_SPARTA_HANDLER(Unit, doWork)) { -#ifdef PIPEOUT_GEN out_cmd.enableCollection(node); in_cmd.enableCollection(node); -#endif in_cmd.registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Unit, cmd_callback, DataType)); in_data.registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Unit, data_callback, char)); @@ -394,19 +389,14 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) slave_unit->in_cmd.precedes(slave_unit->in_data); master_unit->in_cmd.precedes(master_unit->in_data); -#ifdef PIPEOUT_GEN - pc.reset(new sparta::collection::PipelineCollector("testPipe", 1000000, root_clk.get(), &rtn)); -#endif - + pc.reset(new sparta::collection::PipelineCollector("testPipe", {}, 10, &rtn, nullptr)); sched.finalize(); // Align the scheduler to the rising edge of both clocks while(!(master_clk->isPosedge() && slave_clk->isPosedge())) { sched.run(1, true, false); // exacting_run = true, measure time = false } -#ifdef PIPEOUT_GEN - pc->startCollection(&rtn); -#endif + pc->startCollecting(); //sparta::log::Tap scheduler_tap(sparta::Scheduler::getScheduler(), "debug", "sched_cmds.out"); master_unit->schedule_commands(slave_frequency_mhz); @@ -418,9 +408,7 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) TestSystem::~TestSystem() { -#ifdef PIPEOUT_GEN - pc->destroy(); -#endif + pc->stopCollecting(); rtn.enterTeardown(); sched.restartAt(0); } diff --git a/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp b/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp index ef553cbe19..d5e402b171 100644 --- a/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp +++ b/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp @@ -16,8 +16,6 @@ #include "sparta/utils/SpartaTester.hpp" #include "sparta/utils/Utils.hpp" -#include "sparta/pipeViewer/TransactionDatabaseInterface.hpp" - /*! * \file TransactionDatabaseAPI_main.cpp * \brief Test for reading from pipeout transaction database From 5909dd60be8de0023050bb431c6c9353da6dce02 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 3 Feb 2025 14:02:20 -0600 Subject: [PATCH 02/49] SimDB / v3 integration --- sparta/example/CoreModel/src/Execute.cpp | 3 +- sparta/example/CoreModel/src/Execute.hpp | 4 +- sparta/example/CoreModel/src/LSU.hpp | 2 +- sparta/sparta/app/AppTriggers.hpp | 8 +- .../sparta/collection/CollectableTreeNode.hpp | 84 +++++++++++++------ sparta/sparta/collection/CollectionPoints.hpp | 8 +- .../sparta/collection/PipelineCollector.hpp | 6 +- sparta/sparta/ports/DataPort.hpp | 14 ++-- sparta/sparta/ports/SyncPort.hpp | 21 ++--- sparta/src/CommandLineSimulator.cpp | 5 +- sparta/test/Array/Array_test.cpp | 2 +- sparta/test/Buffer/Buffer_test.cpp | 2 +- sparta/test/Pipe/Pipe_test.cpp | 2 +- sparta/test/Pipeline/Pipeline_test.cpp | 2 +- sparta/test/Queue/Queue_test.cpp | 2 +- sparta/test/SyncPort/SyncPort_test.cpp | 2 +- 16 files changed, 104 insertions(+), 63 deletions(-) diff --git a/sparta/example/CoreModel/src/Execute.cpp b/sparta/example/CoreModel/src/Execute.cpp index 64f24576c6..b1445593ee 100644 --- a/sparta/example/CoreModel/src/Execute.cpp +++ b/sparta/example/CoreModel/src/Execute.cpp @@ -84,7 +84,7 @@ namespace core_example ex_inst.setStatus(ExampleInst::Status::SCHEDULED); const uint32_t exe_time = ignore_inst_execute_time_ ? execute_time_ : ex_inst.getExecuteTime(); - // TODO cnyce collected_inst_.collectWithDuration(ex_inst, exe_time); + collected_inst_.collectWithDuration(ex_inst, exe_time); if(SPARTA_EXPECT_FALSE(info_logger_)) { info_logger_ << "Executing: " << ex_inst << " for " << exe_time + getClock()->currentCycle(); @@ -152,7 +152,6 @@ namespace core_example if(complete_inst_.getNumOutstandingEvents() == 0) { unit_busy_ = false; - // TODO cnyce collected_inst_.closeRecord(); } } diff --git a/sparta/example/CoreModel/src/Execute.hpp b/sparta/example/CoreModel/src/Execute.hpp index 4f3ab7e6bf..e34d1eef09 100644 --- a/sparta/example/CoreModel/src/Execute.hpp +++ b/sparta/example/CoreModel/src/Execute.hpp @@ -89,8 +89,8 @@ namespace core_example IterableCollectorType ready_queue_collector_{ getContainer(), "scheduler_queue", &ready_queue_, scheduler_size_}; - sparta::collection::Collectable collected_inst_{ - getContainer(), "collected_inst", nullptr}; + sparta::collection::ManualCollectable collected_inst_{ + getContainer(), "collected_inst"}; // Events used to issue and complete the instruction sparta::UniqueEvent<> issue_inst_{&unit_event_set_, getName() + "_issue_inst", diff --git a/sparta/example/CoreModel/src/LSU.hpp b/sparta/example/CoreModel/src/LSU.hpp index da5ec9ab06..cfec718586 100644 --- a/sparta/example/CoreModel/src/LSU.hpp +++ b/sparta/example/CoreModel/src/LSU.hpp @@ -166,7 +166,7 @@ namespace core_example ExampleInstPtr cache_pending_inst_ptr_ = nullptr; // Collection - sparta::collection::Collectable cache_busy_collectable_; + sparta::collection::AutoCollectable cache_busy_collectable_; // NOTE: // Depending on which kind of cache (e.g. blocking vs. non-blocking) is being used diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index 414f51d38f..caf72b9c4f 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -30,12 +30,12 @@ class PipelineTrigger : public trigger::Triggerable { public: PipelineTrigger(const std::string& simdb_filename, - const std::set& enabled_nodes, - const size_t heartbeat, sparta::RootTreeNode * rtn, - const sparta::CounterBase * insts_retired_counter) + const size_t heartbeat = 10, + const std::set& enabled_nodes = {}, + const sparta::CounterBase * insts_retired_counter = nullptr) { - pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, enabled_nodes, heartbeat, rtn, insts_retired_counter)); + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, rtn, heartbeat, enabled_nodes, insts_retired_counter)); } void go() override diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 687e5ebae3..874fdb7ca0 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -18,68 +18,98 @@ class CollectableTreeNode : public TreeNode } virtual void addCollectionPoint(CollectionPoints & collection_points) = 0; + + bool isCollected() const { return false; } }; -template -class Collectable : public CollectableTreeNode +template +class ManualCollectable : public CollectableTreeNode { public: - using value_type = typename MetaStruct::remove_any_pointer::type; - - Collectable(TreeNode* parent, const std::string& name, const CollectableT* collectable, const std::string& desc = "Collectable ") + ManualCollectable(TreeNode* parent, const std::string& name, const std::string& desc = "ManualCollectable ") : CollectableTreeNode(parent, name, desc) - , collectable_(collectable) { } void addCollectionPoint(CollectionPoints & collection_points) override { - auto is_pod = std::integral_constant::value>(); - addCollectionPoint_(collection_points, is_pod); + (void)collection_points; } -private: - void addCollectionPoint_(CollectionPoints & collection_points, std::true_type) + void collect(const CollectedT & dat) { - if (collectable_) { - collection_points.addStat(getLocation(), getClock(), collectable_); - } + (void)dat; } - void addCollectionPoint_(CollectionPoints & collection_points, std::false_type) + void collectWithDuration(const CollectedT & dat, size_t dur) { - sparta_assert(!collectable_); - //TODO cnyce collection_points.add + (void)dat; + (void)dur; } - const CollectableT* collectable_; +private: }; -template <> -class Collectable : public CollectableTreeNode +template +class DelayedCollectable : public CollectableTreeNode { public: - using value_type = int32_t; + DelayedCollectable(TreeNode* parent, const std::string& name, const std::string& desc = "DelayedCollectable ") + : CollectableTreeNode(parent, name, desc) + { + } + + void addCollectionPoint(CollectionPoints & collection_points) override + { + (void)collection_points; + } + + void collect(const CollectedT & dat, uint64_t delay) + { + (void)dat; + (void)delay; + } + + void collectWithDuration(const CollectedT & dat, uint64_t delay, size_t dur) + { + (void)dat; + (void)delay; + (void)dur; + } + +private: +}; - Collectable(TreeNode* parent, const std::string& name, const bool* collectable, const std::string& desc = "Collectable ") +template +class AutoCollectable : public CollectableTreeNode +{ +public: + AutoCollectable(TreeNode* parent, const std::string& name, const CollectedT* back_ptr, const std::string& desc = "AutoCollectable ") : CollectableTreeNode(parent, name, desc) - , collectable_(collectable) + , back_ptr_(back_ptr) { } void addCollectionPoint(CollectionPoints & collection_points) override { - std::function get_bool_as_int = [this]() { return getBoolAsInt_(); }; - collection_points.addStat(getLocation(), getClock(), get_bool_as_int, simdb::Format::boolalpha); + auto flag = std::integral_constant::value>{}; + addCollectionPoint_(collection_points, flag); } private: - value_type getBoolAsInt_() const + void addCollectionPoint_(CollectionPoints & collection_points, std::true_type) + { + using value_type = int32_t; + auto getter = std::function([this]() { return *back_ptr_ ? 1 : 0; }); + collection_points.addStat(getLocation(), getClock(), getter); + } + + void addCollectionPoint_(CollectionPoints & collection_points, std::false_type) { - return *collectable_ ? 1 : 0; + collection_points.addStat(getLocation(), getClock(), back_ptr_); } - const bool* collectable_; + const CollectedT* back_ptr_; }; template diff --git a/sparta/sparta/collection/CollectionPoints.hpp b/sparta/sparta/collection/CollectionPoints.hpp index 37cd925667..1bf1ad83f9 100644 --- a/sparta/sparta/collection/CollectionPoints.hpp +++ b/sparta/sparta/collection/CollectionPoints.hpp @@ -81,11 +81,17 @@ class CollectionPoints typename std::enable_if::value, void>::type addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) { - // TODO cnyce (void)location; (void)clk; (void)container; (void)capacity; + + // TODO cnyce + // 1 - Array_test (Subprocess aborted) + // 7 - Buffer_test (Subprocess aborted) + // 99 - Pipe_test (Subprocess aborted) + //103 - Pipeline_test (Subprocess aborted) + //107 - Queue_test (Subprocess aborted) } void createCollections(simdb::Collections* collections) diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index a1e44fe505..7171bd8740 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -32,10 +32,10 @@ namespace collection { public: PipelineCollector(const std::string& simdb_filename, - const std::set& enabled_nodes, - const size_t heartbeat, sparta::RootTreeNode * rtn, - const sparta::CounterBase * insts_retired_counter) + const size_t heartbeat = 10, + const std::set& enabled_nodes = {}, + const sparta::CounterBase * insts_retired_counter = nullptr) : db_mgr_(simdb_filename, true) , filename_(simdb_filename) , root_(rtn) diff --git a/sparta/sparta/ports/DataPort.hpp b/sparta/sparta/ports/DataPort.hpp index 56bd02e802..208d4ebfe8 100644 --- a/sparta/sparta/ports/DataPort.hpp +++ b/sparta/sparta/ports/DataPort.hpp @@ -18,6 +18,7 @@ #include "sparta/events/PayloadEvent.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/Scheduleable.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -287,8 +288,7 @@ namespace sparta class DataInPort final : public InPort, public DataContainer { // Pipeline collection type - // TODO cnyce - // typedef collection::Collectable CollectorType; + typedef collection::ManualCollectable CollectorType; public: @@ -445,7 +445,7 @@ namespace sparta * \param node The TreeNode to add the collector */ void enableCollection(TreeNode* node) override { - (void) node; + collector_ = std::make_unique(node, getName() + "_collector"); } private: @@ -530,8 +530,7 @@ namespace sparta const Clock * receiver_clock_ = nullptr; //! Pipeline collection - //! TODO cnyce - //! std::unique_ptr collector_; + std::unique_ptr collector_; //! Data receiving point void receivePortData_(const DataT & dat) @@ -540,6 +539,11 @@ namespace sparta if(SPARTA_EXPECT_TRUE(explicit_consumer_handler_)) { explicit_consumer_handler_((const void*)&dat); } + if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { + if(SPARTA_EXPECT_FALSE(collector_->isCollected())) { + collector_->collect(dat); + } + } } //! This port's additional delay for receiving the data diff --git a/sparta/sparta/ports/SyncPort.hpp b/sparta/sparta/ports/SyncPort.hpp index 29064b9af9..09cf9435af 100644 --- a/sparta/sparta/ports/SyncPort.hpp +++ b/sparta/sparta/ports/SyncPort.hpp @@ -75,6 +75,7 @@ #include "sparta/ports/Port.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/EventSet.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -110,8 +111,7 @@ namespace sparta template class SyncOutPort final : public OutPort { - // TODO cnyce - // typedef collection::DelayedCollectable CollectorType; + typedef collection::DelayedCollectable CollectorType; public: //! A typedef for the type of data this port passes. @@ -303,6 +303,10 @@ namespace sparta uint64_t sched_delay_ticks = sync_in_port_->send_(dat, clk_, send_delay_cycles, allow_slide, is_fwd_progress); + if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { + collector_->collectWithDuration(dat, send_delay_cycles, 1); + } + sparta_assert(send_cycle > prev_data_send_cycle_ || prev_data_send_cycle_ == PREV_DATA_SEND_CYCLE_INIT, //init tick 0 getLocation() << ": trying to send at cycle " @@ -371,7 +375,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - (void) node; + collector_ = std::make_unique(node, getName() + "_collector"); } private: @@ -383,8 +387,7 @@ namespace sparta SyncInPort * sync_in_port_ = nullptr; /// Pipeline collection - /// TODO cnyce - /// std::unique_ptr collector_; + std::unique_ptr collector_; /// Last cycle any data was sent Clock::Cycle PREV_DATA_SEND_CYCLE_INIT = 0xffffffffffffffff; //init tick 0 @@ -406,8 +409,7 @@ namespace sparta template class SyncInPort final : public InPort, public DataContainer { - //! TODO cnyce - //! typedef collection::Collectable CollectorType; + typedef collection::ManualCollectable CollectorType; public: //! Expected typedef for DataT @@ -591,7 +593,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - (void) node; + collector_ = std::make_unique(node, getName() + "_collector"); } //! Set the ready state for the port before simulation begins @@ -1052,8 +1054,7 @@ namespace sparta Scheduler::Tick set_ready_tick_ = 0; //! Pipeline collection - //! TODO cnyce - //! std::unique_ptr collector_; + std::unique_ptr collector_; /// loggers sparta::log::MessageSource info_logger_; diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index cc8c3e496e..9b3fcfd90f 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -2035,9 +2035,9 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) { pipeline_collection_triggerable_.reset(new PipelineTrigger( sim_config_.pipeline_collection_file_prefix, - pipeline_enabled_node_names_, - std::atoi(pipeline_heartbeat_.c_str()), sim->getRoot(), + std::atoi(pipeline_heartbeat_.c_str()), + pipeline_enabled_node_names_, sim->findSemanticCounter(Simulation::CSEM_INSTRUCTIONS))); } @@ -2439,3 +2439,4 @@ bool CommandLineSimulator::openALFAndFindPipelineNodes_(const std::string & alf_ } // namespace app } // namespace sparta + diff --git a/sparta/test/Array/Array_test.cpp b/sparta/test/Array/Array_test.cpp index 9d43a6204e..31e68d57b8 100644 --- a/sparta/test/Array/Array_test.cpp +++ b/sparta/test/Array/Array_test.cpp @@ -134,7 +134,7 @@ int main() root_node.enterConfiguring(); root_node.enterFinalized(); - sparta::collection::PipelineCollector pc("test_collection_", {}, 10, &root_node, nullptr); + sparta::collection::PipelineCollector pc("test_collection_", &root_node); sched.finalize(); pc.startCollecting(); diff --git a/sparta/test/Buffer/Buffer_test.cpp b/sparta/test/Buffer/Buffer_test.cpp index 51b5377b04..556ccb806f 100644 --- a/sparta/test/Buffer/Buffer_test.cpp +++ b/sparta/test/Buffer/Buffer_test.cpp @@ -103,7 +103,7 @@ void generalTest() // Get info messages from the scheduler node and send them to this file sparta::log::Tap t2(root_clk.get()->getScheduler(), "debug", "scheduler.log.debug"); - sparta::collection::PipelineCollector pc("testBuffer", {}, 10, &rtn, nullptr); + sparta::collection::PipelineCollector pc("testBuffer", &rtn); sched.finalize(); pc.startCollecting(); diff --git a/sparta/test/Pipe/Pipe_test.cpp b/sparta/test/Pipe/Pipe_test.cpp index b9f2c7a17f..5efd218f71 100644 --- a/sparta/test/Pipe/Pipe_test.cpp +++ b/sparta/test/Pipe/Pipe_test.cpp @@ -78,7 +78,7 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testPipe", {}, 10, &rtn, nullptr); + sparta::collection::PipelineCollector pc("testPipe", &rtn); sched.finalize(); EXPECT_THROW(pipe2.resize(5)); diff --git a/sparta/test/Pipeline/Pipeline_test.cpp b/sparta/test/Pipeline/Pipeline_test.cpp index e69d6066a5..654f59139a 100644 --- a/sparta/test/Pipeline/Pipeline_test.cpp +++ b/sparta/test/Pipeline/Pipeline_test.cpp @@ -555,7 +555,7 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("examplePipeline1", {}, 10, &rtn, nullptr); + sparta::collection::PipelineCollector pc("examplePipeline1", &rtn); //////////////////////////////////////////////////////////////////////////////// // Pipeline stage handling event precedence setup diff --git a/sparta/test/Queue/Queue_test.cpp b/sparta/test/Queue/Queue_test.cpp index 3d0a4d5d5a..6981b0c22a 100644 --- a/sparta/test/Queue/Queue_test.cpp +++ b/sparta/test/Queue/Queue_test.cpp @@ -68,7 +68,7 @@ int main() rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testPipe", {}, 10, &rtn, nullptr); + sparta::collection::PipelineCollector pc("testPipe", &rtn); sched.finalize(); pc.startCollecting(); diff --git a/sparta/test/SyncPort/SyncPort_test.cpp b/sparta/test/SyncPort/SyncPort_test.cpp index 92523e5cfc..4f0075cfe8 100644 --- a/sparta/test/SyncPort/SyncPort_test.cpp +++ b/sparta/test/SyncPort/SyncPort_test.cpp @@ -389,7 +389,7 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) slave_unit->in_cmd.precedes(slave_unit->in_data); master_unit->in_cmd.precedes(master_unit->in_data); - pc.reset(new sparta::collection::PipelineCollector("testPipe", {}, 10, &rtn, nullptr)); + pc.reset(new sparta::collection::PipelineCollector("testPipe", &rtn)); sched.finalize(); // Align the scheduler to the rising edge of both clocks From d76c0bd4ccb946ed4de1a6a9ed4ac8655209009d Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 5 Feb 2025 13:36:17 -0600 Subject: [PATCH 03/49] Fixes for Sparta v3 integration --- sparta/example/CoreModel/src/Execute.hpp | 2 +- sparta/example/CoreModel/src/Fetch.hpp | 5 + sparta/example/CoreModel/src/LSU.cpp | 2 - sparta/example/CoreModel/src/LSU.hpp | 3 +- .../CoreModel/src/LoadStoreInstInfo.hpp | 1 - .../sparta/collection/CollectableTreeNode.hpp | 133 +++++++++++++----- sparta/sparta/collection/CollectionPoints.hpp | 66 ++++++++- .../sparta/collection/PipelineCollector.hpp | 8 ++ sparta/sparta/ports/DataPort.hpp | 2 +- sparta/sparta/ports/SyncPort.hpp | 19 ++- sparta/sparta/resources/Pipe.hpp | 3 +- .../CircularBuffer/CircularBuffer_test.cpp | 2 + sparta/test/Pipe/Pipe_test.cpp | 5 +- 13 files changed, 198 insertions(+), 53 deletions(-) diff --git a/sparta/example/CoreModel/src/Execute.hpp b/sparta/example/CoreModel/src/Execute.hpp index e34d1eef09..5bb3d5f528 100644 --- a/sparta/example/CoreModel/src/Execute.hpp +++ b/sparta/example/CoreModel/src/Execute.hpp @@ -90,7 +90,7 @@ namespace core_example getContainer(), "scheduler_queue", &ready_queue_, scheduler_size_}; sparta::collection::ManualCollectable collected_inst_{ - getContainer(), "collected_inst"}; + getContainer(), getContainer()->getName()}; // Events used to issue and complete the instruction sparta::UniqueEvent<> issue_inst_{&unit_event_set_, getName() + "_issue_inst", diff --git a/sparta/example/CoreModel/src/Fetch.hpp b/sparta/example/CoreModel/src/Fetch.hpp index 4393a7bb56..e8a7e38e0b 100644 --- a/sparta/example/CoreModel/src/Fetch.hpp +++ b/sparta/example/CoreModel/src/Fetch.hpp @@ -14,6 +14,7 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/TreeNode.hpp" #include "sparta/simulation/ParameterSet.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" #include "CoreTypes.hpp" @@ -101,6 +102,10 @@ namespace core_example // instructions or a perfect IPC set std::unique_ptr> fetch_inst_event_; + // A pipeline collector + sparta::collection::AutoCollectable next_pc_{ + getContainer(), "next_pc", &vaddr_}; + //////////////////////////////////////////////////////////////////////////////// // Callbacks diff --git a/sparta/example/CoreModel/src/LSU.cpp b/sparta/example/CoreModel/src/LSU.cpp index a2e485f2b3..c47327b038 100644 --- a/sparta/example/CoreModel/src/LSU.cpp +++ b/sparta/example/CoreModel/src/LSU.cpp @@ -16,10 +16,8 @@ namespace core_example load_store_info_allocator(50, 30), ldst_inst_queue_("lsu_inst_queue", p->ldst_inst_queue_size, getClock()), ldst_inst_queue_size_(p->ldst_inst_queue_size), - tlb_always_hit_(p->tlb_always_hit), dl1_always_hit_(p->dl1_always_hit), - cache_busy_collectable_(getContainer(), "dcache_busy", &cache_busy_), issue_latency_(p->issue_latency), mmu_latency_(p->mmu_latency), cache_latency_(p->cache_latency), diff --git a/sparta/example/CoreModel/src/LSU.hpp b/sparta/example/CoreModel/src/LSU.hpp index cfec718586..0700552016 100644 --- a/sparta/example/CoreModel/src/LSU.hpp +++ b/sparta/example/CoreModel/src/LSU.hpp @@ -166,7 +166,8 @@ namespace core_example ExampleInstPtr cache_pending_inst_ptr_ = nullptr; // Collection - sparta::collection::AutoCollectable cache_busy_collectable_; + sparta::collection::AutoCollectable cache_busy_collectable_{ + getContainer(), "dcache_busy", &cache_busy_}; // NOTE: // Depending on which kind of cache (e.g. blocking vs. non-blocking) is being used diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index 112944f2d0..fc9b4b84d8 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -212,7 +212,6 @@ inline void defineStructSchema(StructSchema& sc schema.addField("rank"); schema.addField("state"); schema.setAutoColorizeColumn("DID"); - //schema.addFlattenField("mem_access_info_ptr"); } template <> diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 874fdb7ca0..cf7018818c 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -2,12 +2,16 @@ #include "sparta/simulation/TreeNode.hpp" #include "sparta/collection/CollectionPoints.hpp" +#include "sparta/events/PayloadEvent.hpp" namespace sparta { namespace collection { +inline void colbydebug() {} + class CollectionPoints; +// Base class for all collectable classes. class CollectableTreeNode : public TreeNode { public: @@ -19,9 +23,20 @@ class CollectableTreeNode : public TreeNode virtual void addCollectionPoint(CollectionPoints & collection_points) = 0; - bool isCollected() const { return false; } + bool isCollected() const { return collected_; } + + void enable() { collected_ = true; } + + // Note there is no way to disable collection on a per-collectable basis. You can + // only call PipelineCollector::stopCollecting() to stop all collection for all + // collectables. + +private: + bool collected_ = false; }; +// Manually-collected means that the user must explicitly call collect() or collectWithDuration() +// and cannot give a backpointer to this class' constructor for automatic every-cycle collection. template class ManualCollectable : public CollectableTreeNode { @@ -29,57 +44,114 @@ class ManualCollectable : public CollectableTreeNode ManualCollectable(TreeNode* parent, const std::string& name, const std::string& desc = "ManualCollectable ") : CollectableTreeNode(parent, name, desc) { + sparta_assert(getClock()); } void addCollectionPoint(CollectionPoints & collection_points) override { - (void)collection_points; + collector_ = collection_points.addManualCollectable(getLocation(), getClock()); } + using CollectableTreeNode::isCollected; + void collect(const CollectedT & dat) { - (void)dat; + if (isCollected()) { + colbydebug(); + collector_->collect(dat); + } } - void collectWithDuration(const CollectedT & dat, size_t dur) + void collectWithDuration(const CollectedT & dat, const Clock::Cycle dur) { - (void)dat; - (void)dur; + if (isCollected()) { + colbydebug(); + const auto ticks = getClock()->getTick(dur); + collector_->collectWithDuration(dat, ticks); + } } private: + simdb::AnyCollection* collector_ = nullptr; }; +// Similar to the ManualCollectable class, you cannot use a DelayedCollectable +// for any kind of automatic collection. You must explicitly call collect() or +// collectWithDuration() to collect data. template -class DelayedCollectable : public CollectableTreeNode +class DelayedCollectable : public ManualCollectable { -public: - DelayedCollectable(TreeNode* parent, const std::string& name, const std::string& desc = "DelayedCollectable ") - : CollectableTreeNode(parent, name, desc) + struct DurationData { - } + DurationData() = default; - void addCollectionPoint(CollectionPoints & collection_points) override + DurationData(const CollectedT & dat, + sparta::Clock::Cycle duration) + : data(dat) + , duration(duration) + {} + + CollectedT data; + sparta::Clock::Cycle duration; + }; + +public: + DelayedCollectable(TreeNode* parent, const std::string& name, sparta::EventSet* ev_set, const std::string& desc = "DelayedCollectable ") + : ManualCollectable(parent, name, desc) + , ev_collect_(ev_set, name + "_event", CREATE_SPARTA_HANDLER_WITH_DATA(ManualCollectable, collect, CollectedT)) + , ev_collect_duration_(ev_set, name + "_duration_event", CREATE_SPARTA_HANDLER_WITH_DATA(DelayedCollectable, collectWithDuration_, DurationData)) { - (void)collection_points; } + using ManualCollectable::isCollected; + void collect(const CollectedT & dat, uint64_t delay) { - (void)dat; - (void)delay; + if (isCollected()) { + colbydebug(); + if (delay == 0) { + ManualCollectable::collect(dat); + } else { + ev_collect_.schedule(dat, delay); + } + } } - void collectWithDuration(const CollectedT & dat, uint64_t delay, size_t dur) + void collectWithDuration(const CollectedT & dat, uint64_t delay, const Clock::Cycle dur) { - (void)dat; - (void)delay; - (void)dur; + if (isCollected()) { + colbydebug(); + if (delay == 0) { + ManualCollectable::collectWithDuration(dat, dur); + } else { + ev_collect_duration_.preparePayload({dat, dur})->schedule(delay); + } + } } private: + // Called from collectWithDuration where the data needs to be + // delivered at a given delayed time, but only for a short + // duration. + void collectWithDuration_(const DurationData & dur_dat) + { + colbydebug(); + ManualCollectable::collectWithDuration(dur_dat.data, dur_dat.duration); + } + + // For those folks that want collection to appear in the future + sparta::PayloadEvent ev_collect_; + + // For those folks that want collection to appear in the future with a duration + sparta::PayloadEvent ev_collect_duration_; }; +// Auto-collectable means that the user can give a backpointer to this class' constructor +// and we will collect your data every cycle automatically. There is no way to automatically +// collect at any other time than "every cycle". +// +// This class is to be used for non-iterable types like uint64_t, struct, bool, etc. +// You should use the IterableCollector class for iterable types like std::vector, sparta::Array, etc. template class AutoCollectable : public CollectableTreeNode { @@ -88,27 +160,22 @@ class AutoCollectable : public CollectableTreeNode : CollectableTreeNode(parent, name, desc) , back_ptr_(back_ptr) { + static_assert(std::is_integral::value || std::is_floating_point::value, + "AutoCollectable only supports integral and floating-point types!"); } void addCollectionPoint(CollectionPoints & collection_points) override { - auto flag = std::integral_constant::value>{}; - addCollectionPoint_(collection_points, flag); + if constexpr (std::is_same::value) { + using value_type = int32_t; + auto getter = std::function([this]() { return *back_ptr_ ? 1 : 0; }); + collection_points.addStat(getLocation(), getClock(), getter); + } else { + collection_points.addStat(getLocation(), getClock(), back_ptr_); + } } private: - void addCollectionPoint_(CollectionPoints & collection_points, std::true_type) - { - using value_type = int32_t; - auto getter = std::function([this]() { return *back_ptr_ ? 1 : 0; }); - collection_points.addStat(getLocation(), getClock(), getter); - } - - void addCollectionPoint_(CollectionPoints & collection_points, std::false_type) - { - collection_points.addStat(getLocation(), getClock(), back_ptr_); - } - const CollectedT* back_ptr_; }; diff --git a/sparta/sparta/collection/CollectionPoints.hpp b/sparta/sparta/collection/CollectionPoints.hpp index 1bf1ad83f9..3557b51d33 100644 --- a/sparta/sparta/collection/CollectionPoints.hpp +++ b/sparta/sparta/collection/CollectionPoints.hpp @@ -6,6 +6,7 @@ #include "simdb/collection/Structs.hpp" #include "simdb/collection/IterableStructs.hpp" #include "simdb/collection/TimeseriesCollector.hpp" +#include "simdb/collection/Any.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/utils/MetaStructs.hpp" @@ -81,17 +82,24 @@ class CollectionPoints typename std::enable_if::value, void>::type addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) { - (void)location; + // TODO cnyce (only affects unit tests e.g. Array, Buffer, etc.) + std::cout << "Container type " << demangled_type() << " is not a pointer type. Skipping collection. " + << "This occurred at " << location << std::endl; + (void)clk; (void)container; (void)capacity; + } - // TODO cnyce - // 1 - Array_test (Subprocess aborted) - // 7 - Buffer_test (Subprocess aborted) - // 99 - Pipe_test (Subprocess aborted) - //103 - Pipeline_test (Subprocess aborted) - //107 - Queue_test (Subprocess aborted) + template + simdb::AnyCollection* addManualCollectable(const std::string& location, const Clock* clk) + { + auto& instantiator = instantiators_["ManualCollectable"]; + if (!instantiator) { + instantiator.reset(new ManualCollectableInstantiator()); + } + + return dynamic_cast(instantiator.get())->addManualCollectable(location, clk); } void createCollections(simdb::Collections* collections) @@ -134,6 +142,50 @@ class CollectionPoints virtual void createCollections(simdb::Collections* collections, const std::string& collection_prefix) = 0; }; + class ManualCollectableInstantiator : public CollectableInstantiator + { + public: + template + simdb::AnyCollection* addManualCollectable(const std::string& location, const Clock* clk) + { + clocks_.emplace_back(location, clk); + + using CollectionT = simdb::AnyCollection; + std::unique_ptr collection(new CollectionT(location)); + auto ret = collection.get(); + collections_.emplace_back(std::move(collection)); + return ret; + } + + private: + void getClockPeriods(std::unordered_map& clk_periods) const override + { + for (const auto& tup : clocks_) { + const auto clk = std::get<1>(tup); + clk_periods[clk->getName()] = clk->getPeriod(); + } + } + + void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override + { + for (const auto& tup : clocks_) { + const auto location = std::get<0>(tup); + const auto clk = std::get<1>(tup); + clk_names_by_location[location] = clk->getName(); + } + } + + void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override + { + for (auto& collection : collections_) { + collections->addCollection(std::move(collection)); + } + } + + std::vector> clocks_; + std::vector> collections_; + }; + template class StatInstantiator : public CollectableInstantiator { diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 7171bd8740..6add231325 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -70,6 +70,10 @@ namespace collection } void startCollecting() { + for (auto c : collectables_) { + c->enable(); + } + // Schedule collect event in the next cycle in case // this is called in an unavailable phase. ev_collect_->schedule(sparta::Clock::Cycle(1)); @@ -113,6 +117,7 @@ namespace collection if (auto c = dynamic_cast(node)) { if (enabled_nodes.empty() || enabled_nodes.count(c->getLocation())) { c->addCollectionPoint(collectables); + collectables_.push_back(c); } } for (auto & child : node->getChildren()) { @@ -180,6 +185,9 @@ namespace collection //! Fastest clock across all collectable tree nodes const Clock * fastest_clk_ = nullptr; + + //! All collectables. + std::vector collectables_; }; }// namespace collection diff --git a/sparta/sparta/ports/DataPort.hpp b/sparta/sparta/ports/DataPort.hpp index 208d4ebfe8..368ace8db8 100644 --- a/sparta/sparta/ports/DataPort.hpp +++ b/sparta/sparta/ports/DataPort.hpp @@ -445,7 +445,7 @@ namespace sparta * \param node The TreeNode to add the collector */ void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, getName() + "_collector"); + collector_ = std::make_unique(node, Port::name_, "Data being received on this DataInPort"); } private: diff --git a/sparta/sparta/ports/SyncPort.hpp b/sparta/sparta/ports/SyncPort.hpp index 09cf9435af..2a9a68eaad 100644 --- a/sparta/sparta/ports/SyncPort.hpp +++ b/sparta/sparta/ports/SyncPort.hpp @@ -130,7 +130,8 @@ namespace sparta SyncOutPort(TreeNode * portset, const std::string & name, const Clock * clk, bool presume_zero_delay = true) : OutPort(portset, name, presume_zero_delay), - clk_(clk), info_logger_(this, "pinfo", getLocation() + "_info") + clk_(clk), info_logger_(this, "pinfo", getLocation() + "_info"), + sync_port_events_(this) { sparta_assert(name.length() != 0, "You cannot have an unnamed port."); sparta_assert(clk_ != 0, "Clock ptr cannot be null in port: " << name); @@ -374,8 +375,9 @@ namespace sparta SyncOutPort & operator=(const SyncOutPort &) = delete; //! Enable pipeline collection - void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, getName() + "_collector"); + void enableCollection(TreeNode* node) override + { + collector_ = std::make_unique(node, Port::name_, &sync_port_events_, "Data being sent out on this SyncOutPort"); } private: @@ -398,6 +400,9 @@ namespace sparta //! The bound SyncIn ports std::vector *> bound_in_ports_; + + //! Event Set for this port + sparta::EventSet sync_port_events_; }; @@ -593,7 +598,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, getName() + "_collector"); + collector_ = std::make_unique(node, Port::name_, "Data being recirculated on this SyncInPort"); } //! Set the ready state for the port before simulation begins @@ -808,6 +813,12 @@ namespace sparta explicit_consumer_handler_((const void*)&dat); } + // Show the data that has arrived on this OutPort that + // the receiver now sees + if(SPARTA_EXPECT_FALSE(collector_ != nullptr)) { + collector_->collectWithDuration(dat, 1); + } + if (SPARTA_EXPECT_FALSE(info_logger_)) { info_logger_ << "DELIVERING @" << receiver_clock_->currentCycle() << "(" << num_in_flight_ << ") " diff --git a/sparta/sparta/resources/Pipe.hpp b/sparta/sparta/resources/Pipe.hpp index d8fce5155c..b6340ea851 100644 --- a/sparta/sparta/resources/Pipe.hpp +++ b/sparta/sparta/resources/Pipe.hpp @@ -223,6 +223,7 @@ class Pipe * finialization nor after enabling pipeline collection. */ void resize(uint32_t new_size) { + sparta_assert(collector_ == nullptr); //sparta_assert(!getClock()->isFinalized()); initPipe_(new_size); } @@ -524,7 +525,7 @@ class Pipe * \brief Check if pipe is collecting */ bool isCollected() const { - return false; + return collector_ && collector_->isCollected(); } private: diff --git a/sparta/test/CircularBuffer/CircularBuffer_test.cpp b/sparta/test/CircularBuffer/CircularBuffer_test.cpp index 86fbf30878..8a2cbcaef0 100644 --- a/sparta/test/CircularBuffer/CircularBuffer_test.cpp +++ b/sparta/test/CircularBuffer/CircularBuffer_test.cpp @@ -11,6 +11,7 @@ #include "sparta/kernel/Scheduler.hpp" #include "sparta/utils/SpartaTester.hpp" #include "sparta/report/Report.hpp" +#include "sparta/collection/PipelineCollector.hpp" TEST_INIT @@ -462,6 +463,7 @@ void testCollection() rtn.enterConfiguring(); rtn.enterFinalized(); + sparta::collection::PipelineCollector pc("testCircBuffer", &rtn); sched.finalize(); for(uint32_t i = 0; i < BUF_SIZE/2; ++i) { diff --git a/sparta/test/Pipe/Pipe_test.cpp b/sparta/test/Pipe/Pipe_test.cpp index 5efd218f71..95f92c7d76 100644 --- a/sparta/test/Pipe/Pipe_test.cpp +++ b/sparta/test/Pipe/Pipe_test.cpp @@ -81,8 +81,9 @@ int main () sparta::collection::PipelineCollector pc("testPipe", &rtn); sched.finalize(); - EXPECT_THROW(pipe2.resize(5)); - EXPECT_EQUAL(pipe2.capacity(), 10); // Make sure it really didn't get resized + //TODO cnyce: when pipe2.enableCollection() is fixed, put back these two checks. + //EXPECT_THROW(pipe2.resize(5)); + //EXPECT_EQUAL(pipe2.capacity(), 10); // Make sure it really didn't get resized pc.startCollecting(); // Check initials From 37b3eabeebbb996cac465372bc0352de24d54c99 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 5 Feb 2025 15:24:48 -0600 Subject: [PATCH 04/49] Fixes for Sparta v3 integration --- sparta/CMakeLists.txt | 4 ++-- sparta/cmake/sparta-config.cmake | 7 +------ sparta/sparta/app/AppTriggers.hpp | 2 +- sparta/sparta/collection/CollectableTreeNode.hpp | 8 +------- sparta/sparta/resources/Buffer.hpp | 2 ++ 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index ebd42f7698..084a61b82f 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -151,7 +151,7 @@ if(DEFINED SPARTA_CXX_FLAGS_DEBUG AND SPARTA_CXX_FLAGS_DEBUG) set(CMAKE_CXX_FLAGS_DEBUG "${SPARTA_CXX_FLAGS_DEBUG}") message(STATUS "Using Sparta custom debug flags: ${CMAKE_CXX_FLAGS_DEBUG}") else() - set(CMAKE_CXX_FLAGS_DEBUG "-U_FORTIFY_SOURCE -O0 -g3") + set(CMAKE_CXX_FLAGS_DEBUG "-U_FORTIFY_SOURCE -O0 -g3") message(STATUS "Using Sparta default debug flags: ${CMAKE_CXX_FLAGS_DEBUG}") endif() # @@ -207,7 +207,7 @@ add_library (sparta ${SourceCppFiles}) # Add local includes target_include_directories (sparta PUBLIC "./") -target_include_directories (sparta PUBLIC "./simdb/include") +target_include_directories (sparta PUBLIC "simdb/include") set (SPARTA_STATIC_LIBS ${PROJECT_BINARY_DIR}/libsparta.a) set (SPARTA_CMAKE_MACRO_PATH ${SPARTA_BASE}/cmake) diff --git a/sparta/cmake/sparta-config.cmake b/sparta/cmake/sparta-config.cmake index e2b234d283..03cb638d3c 100644 --- a/sparta/cmake/sparta-config.cmake +++ b/sparta/cmake/sparta-config.cmake @@ -74,14 +74,9 @@ find_package(ZLIB REQUIRED) include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) message (STATUS "Using zlib ${ZLIB_VERSION_STRING}") -# Find HDF5. -find_package (HDF5 1.10 REQUIRED COMPONENTS CXX) -include_directories (SYSTEM ${HDF5_INCLUDE_DIRS}) -message (STATUS "Using HDF5 ${HDF5_VERSION}") - # Populate the Sparta_LIBS variable with the required libraries for # basic Sparta linking -set (Sparta_LIBS sparta HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread +set (Sparta_LIBS sparta sqlite3 yaml-cpp ZLIB::ZLIB pthread Boost::date_time Boost::iostreams Boost::serialization Boost::timer Boost::program_options) # On Linux we need to link against rt as well diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index caf72b9c4f..fbbc2efd01 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -16,8 +16,8 @@ #include "sparta/trigger/Triggerable.hpp" #include "sparta/pevents/PeventTrigger.hpp" #include "sparta/pevents/PeventController.hpp" -#include "sparta/log/Tap.hpp" #include "sparta/collection/PipelineCollector.hpp" +#include "sparta/log/Tap.hpp" namespace sparta { namespace app { diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index cf7018818c..c638394c49 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -7,8 +7,6 @@ namespace sparta { namespace collection { -inline void colbydebug() {} - class CollectionPoints; // Base class for all collectable classes. @@ -57,7 +55,6 @@ class ManualCollectable : public CollectableTreeNode void collect(const CollectedT & dat) { if (isCollected()) { - colbydebug(); collector_->collect(dat); } } @@ -65,7 +62,6 @@ class ManualCollectable : public CollectableTreeNode void collectWithDuration(const CollectedT & dat, const Clock::Cycle dur) { if (isCollected()) { - colbydebug(); const auto ticks = getClock()->getTick(dur); collector_->collectWithDuration(dat, ticks); } @@ -108,10 +104,10 @@ class DelayedCollectable : public ManualCollectable void collect(const CollectedT & dat, uint64_t delay) { if (isCollected()) { - colbydebug(); if (delay == 0) { ManualCollectable::collect(dat); } else { + sparta_assert(false); ev_collect_.schedule(dat, delay); } } @@ -120,7 +116,6 @@ class DelayedCollectable : public ManualCollectable void collectWithDuration(const CollectedT & dat, uint64_t delay, const Clock::Cycle dur) { if (isCollected()) { - colbydebug(); if (delay == 0) { ManualCollectable::collectWithDuration(dat, dur); } else { @@ -135,7 +130,6 @@ class DelayedCollectable : public ManualCollectable // duration. void collectWithDuration_(const DurationData & dur_dat) { - colbydebug(); ManualCollectable::collectWithDuration(dur_dat.data, dur_dat.duration); } diff --git a/sparta/sparta/resources/Buffer.hpp b/sparta/sparta/resources/Buffer.hpp index dbb8e80085..c2ee543846 100644 --- a/sparta/sparta/resources/Buffer.hpp +++ b/sparta/sparta/resources/Buffer.hpp @@ -1083,6 +1083,7 @@ namespace sparta num_valid_(rval.num_valid_), validator_(new DataPointerValidator(*this)), utilization_(std::move(rval.utilization_)), + collector_(std::move(rval.collector_)), is_infinite_mode_(rval.is_infinite_mode_), resize_delta_(std::move(rval.resize_delta_)), address_map_(std::move(rval.address_map_)){ @@ -1093,6 +1094,7 @@ namespace sparta rval.first_position_ = nullptr; rval.num_valid_ = 0; rval.utilization_ = nullptr; + rval.collector_ = nullptr; validator_->validator_ = std::move(rval.validator_->validator_); } } From da8a8ba26ba9d7a2a8528092ac4f9427ab03daa1 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Thu, 6 Feb 2025 11:05:17 -0600 Subject: [PATCH 05/49] SimDB / v3 integration --- sparta/example/CoreModel/src/Execute.hpp | 2 +- .../sparta/collection/CollectableTreeNode.hpp | 33 +++++++++++++++---- sparta/sparta/ports/DataPort.hpp | 2 +- sparta/sparta/ports/SyncPort.hpp | 2 +- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/sparta/example/CoreModel/src/Execute.hpp b/sparta/example/CoreModel/src/Execute.hpp index 5bb3d5f528..c49de846f3 100644 --- a/sparta/example/CoreModel/src/Execute.hpp +++ b/sparta/example/CoreModel/src/Execute.hpp @@ -90,7 +90,7 @@ namespace core_example getContainer(), "scheduler_queue", &ready_queue_, scheduler_size_}; sparta::collection::ManualCollectable collected_inst_{ - getContainer(), getContainer()->getName()}; + getContainer(), getContainer()->getName(), getEventSet()}; // Events used to issue and complete the instruction sparta::UniqueEvent<> issue_inst_{&unit_event_set_, getName() + "_issue_inst", diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index c638394c49..b4462fdda1 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -3,6 +3,8 @@ #include "sparta/simulation/TreeNode.hpp" #include "sparta/collection/CollectionPoints.hpp" #include "sparta/events/PayloadEvent.hpp" +#include "sparta/events/PhasedUniqueEvent.hpp" +#include "sparta/events/EventSet.hpp" namespace sparta { namespace collection { @@ -39,8 +41,9 @@ template class ManualCollectable : public CollectableTreeNode { public: - ManualCollectable(TreeNode* parent, const std::string& name, const std::string& desc = "ManualCollectable ") + ManualCollectable(TreeNode* parent, const std::string& name, sparta::EventSet* ev_set, const std::string& desc = "ManualCollectable ") : CollectableTreeNode(parent, name, desc) + , ev_end_duration_(getEventSet_(parent, ev_set), name + "_end_duration_event", SchedulingPhase::Collection, CREATE_SPARTA_HANDLER(ManualCollectable, endDuration_)) { sparta_assert(getClock()); } @@ -55,20 +58,39 @@ class ManualCollectable : public CollectableTreeNode void collect(const CollectedT & dat) { if (isCollected()) { - collector_->collect(dat); + collector_->collectOnce(dat); } } void collectWithDuration(const CollectedT & dat, const Clock::Cycle dur) { if (isCollected()) { - const auto ticks = getClock()->getTick(dur); - collector_->collectWithDuration(dat, ticks); + collector_->beginZOH(dat); + ev_end_duration_.schedule(getClock()->getTick(dur)); } } private: + EventSet* getEventSet_(TreeNode* parent, EventSet* ev_set) + { + if (ev_set) { + return ev_set; + } else { + ev_set_.reset(new EventSet(parent)); + return ev_set_.get(); + } + } + + void endDuration_() + { + if (isCollected()) { + collector_->endZOH(); + } + } + simdb::AnyCollection* collector_ = nullptr; + sparta::PhasedUniqueEvent ev_end_duration_; + std::unique_ptr ev_set_; }; // Similar to the ManualCollectable class, you cannot use a DelayedCollectable @@ -93,7 +115,7 @@ class DelayedCollectable : public ManualCollectable public: DelayedCollectable(TreeNode* parent, const std::string& name, sparta::EventSet* ev_set, const std::string& desc = "DelayedCollectable ") - : ManualCollectable(parent, name, desc) + : ManualCollectable(parent, name, ev_set, desc) , ev_collect_(ev_set, name + "_event", CREATE_SPARTA_HANDLER_WITH_DATA(ManualCollectable, collect, CollectedT)) , ev_collect_duration_(ev_set, name + "_duration_event", CREATE_SPARTA_HANDLER_WITH_DATA(DelayedCollectable, collectWithDuration_, DurationData)) { @@ -107,7 +129,6 @@ class DelayedCollectable : public ManualCollectable if (delay == 0) { ManualCollectable::collect(dat); } else { - sparta_assert(false); ev_collect_.schedule(dat, delay); } } diff --git a/sparta/sparta/ports/DataPort.hpp b/sparta/sparta/ports/DataPort.hpp index 368ace8db8..ce96943557 100644 --- a/sparta/sparta/ports/DataPort.hpp +++ b/sparta/sparta/ports/DataPort.hpp @@ -445,7 +445,7 @@ namespace sparta * \param node The TreeNode to add the collector */ void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, Port::name_, "Data being received on this DataInPort"); + collector_ = std::make_unique(node, Port::name_, &data_in_port_events_, "Data being received on this DataInPort"); } private: diff --git a/sparta/sparta/ports/SyncPort.hpp b/sparta/sparta/ports/SyncPort.hpp index 2a9a68eaad..c89bb5ced4 100644 --- a/sparta/sparta/ports/SyncPort.hpp +++ b/sparta/sparta/ports/SyncPort.hpp @@ -598,7 +598,7 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, Port::name_, "Data being recirculated on this SyncInPort"); + collector_ = std::make_unique(node, Port::name_, &sync_port_events_, "Data being recirculated on this SyncInPort"); } //! Set the ready state for the port before simulation begins From a0fe18e823f84ce58bb26c1d6bf7534d786fff4e Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Thu, 6 Feb 2025 15:12:38 -0600 Subject: [PATCH 06/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/app/AppTriggers.hpp | 9 +++++--- sparta/sparta/collection/CollectionPoints.hpp | 16 +------------- .../sparta/collection/PipelineCollector.hpp | 21 +------------------ sparta/sparta/report/format/JSON_detail.hpp | 2 -- sparta/src/CommandLineSimulator.cpp | 3 +-- 6 files changed, 10 insertions(+), 43 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index cb3c59e0ac..b980116477 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit cb3c59e0ac26bb8346f67ccbb215897e650d4e94 +Subproject commit b980116477e6440d4d6d58a07bfdebf8cebbd9de diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index fbbc2efd01..525d4a8e2a 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -32,16 +32,19 @@ class PipelineTrigger : public trigger::Triggerable PipelineTrigger(const std::string& simdb_filename, sparta::RootTreeNode * rtn, const size_t heartbeat = 10, - const std::set& enabled_nodes = {}, - const sparta::CounterBase * insts_retired_counter = nullptr) + const std::set& enabled_nodes = {}) { - pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, rtn, heartbeat, enabled_nodes, insts_retired_counter)); + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, rtn, heartbeat, enabled_nodes)); } void go() override { sparta_assert(!triggered_, "Why has pipeline trigger been triggered?"); triggered_ = true; + + std::cout << "Pipeline collection started, output to database file '" + << pipeline_collector_->getFilePath() << "'" << std::endl; + pipeline_collector_->startCollecting(); } diff --git a/sparta/sparta/collection/CollectionPoints.hpp b/sparta/sparta/collection/CollectionPoints.hpp index 3557b51d33..a00a7bbfe0 100644 --- a/sparta/sparta/collection/CollectionPoints.hpp +++ b/sparta/sparta/collection/CollectionPoints.hpp @@ -66,8 +66,7 @@ class CollectionPoints } template - typename std::enable_if::value, void>::type - addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) + void addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) { auto demangled = demangled_type() + (Sparse ? "Sparse" : "Contig"); auto& instantiator = instantiators_[demangled]; @@ -78,19 +77,6 @@ class CollectionPoints dynamic_cast*>(instantiator.get())->addContainer(location, clk, container, capacity); } - template - typename std::enable_if::value, void>::type - addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) - { - // TODO cnyce (only affects unit tests e.g. Array, Buffer, etc.) - std::cout << "Container type " << demangled_type() << " is not a pointer type. Skipping collection. " - << "This occurred at " << location << std::endl; - - (void)clk; - (void)container; - (void)capacity; - } - template simdb::AnyCollection* addManualCollectable(const std::string& location, const Clock* clk) { diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 6add231325..bba099b3b5 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -34,13 +34,11 @@ namespace collection PipelineCollector(const std::string& simdb_filename, sparta::RootTreeNode * rtn, const size_t heartbeat = 10, - const std::set& enabled_nodes = {}, - const sparta::CounterBase * insts_retired_counter = nullptr) + const std::set& enabled_nodes = {}) : db_mgr_(simdb_filename, true) , filename_(simdb_filename) , root_(rtn) , ev_set_(nullptr) - , insts_retired_counter_(insts_retired_counter) { // Note that we only care about the collection data and have // no need for any other tables, aside from the tables that the @@ -50,12 +48,6 @@ namespace collection std::function func_ptr = std::bind(&PipelineCollector::getCollectionTick_, this); db_mgr_.getCollectionMgr()->useTimestampsFrom(func_ptr); - - if (insts_retired_counter_) { - std::function ipc_func = std::bind(&PipelineCollector::getIPC_, this); - db_mgr_.getCollectionMgr()->enableArgosIPC(ipc_func); - } - db_mgr_.getCollectionMgr()->setHeartbeat(heartbeat); createCollections_(enabled_nodes); } @@ -154,14 +146,6 @@ namespace collection ev_collect_->schedule(); } - double getIPC_() const - { - auto tick = scheduler_->getCurrentTick(); - auto cycle = fastest_clk_->getCycle(tick); - auto num_retired = insts_retired_counter_->get(); - return static_cast(num_retired) / static_cast(cycle); - } - //! The SimDB database simdb::DatabaseManager db_mgr_; @@ -180,9 +164,6 @@ namespace collection //! The simulation scheduler const sparta::Scheduler * scheduler_; - //! The "total instruction retired" counter - const sparta::CounterBase * insts_retired_counter_; - //! Fastest clock across all collectable tree nodes const Clock * fastest_clk_ = nullptr; diff --git a/sparta/sparta/report/format/JSON_detail.hpp b/sparta/sparta/report/format/JSON_detail.hpp index cd33642a6a..9cf4880253 100644 --- a/sparta/sparta/report/format/JSON_detail.hpp +++ b/sparta/sparta/report/format/JSON_detail.hpp @@ -211,8 +211,6 @@ class JSON_detail : public BaseOstreamFormatter local_name = p_name + "." + flattenReportName(r->getName()); } - // TODO cnyce - // Go through all subreports for (const Report& sr : r->getSubreports()) { collectDictContents_(&sr, idx+1, local_name); diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index 9b3fcfd90f..08980e2732 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -2037,8 +2037,7 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) sim_config_.pipeline_collection_file_prefix, sim->getRoot(), std::atoi(pipeline_heartbeat_.c_str()), - pipeline_enabled_node_names_, - sim->findSemanticCounter(Simulation::CSEM_INSTRUCTIONS))); + pipeline_enabled_node_names_)); } // Finalize the pevent controller now that the tree is built. From 8633088e599d9a03066e6de7d2ec86774ec0c190 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 7 Feb 2025 10:53:29 -0600 Subject: [PATCH 07/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/app/AppTriggers.hpp | 41 ++++- .../sparta/collection/PipelineCollector.hpp | 143 +++++++++--------- sparta/sparta/resources/Pipe.hpp | 10 +- sparta/test/Pipe/Pipe_test.cpp | 2 +- sparta/test/Pipeline/Pipeline_test.cpp | 15 -- 6 files changed, 117 insertions(+), 96 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index b980116477..f80a4cb268 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit b980116477e6440d4d6d58a07bfdebf8cebbd9de +Subproject commit f80a4cb268e491c86f78f8d6de41a9f767068331 diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index 525d4a8e2a..04bd206672 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -33,8 +33,10 @@ class PipelineTrigger : public trigger::Triggerable sparta::RootTreeNode * rtn, const size_t heartbeat = 10, const std::set& enabled_nodes = {}) + : rtn_(rtn) + , enabled_nodes_(enabled_nodes) { - pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, rtn, heartbeat, enabled_nodes)); + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, heartbeat)); } void go() override @@ -45,18 +47,51 @@ class PipelineTrigger : public trigger::Triggerable std::cout << "Pipeline collection started, output to database file '" << pipeline_collector_->getFilePath() << "'" << std::endl; - pipeline_collector_->startCollecting(); + startCollection_(); } void stop() override { sparta_assert(triggered_, "Why stop an inactivated trigger?"); triggered_ = false; - pipeline_collector_->stopCollecting(); + stopCollection_(); } private: + void startCollection_() + { + if (enabled_nodes_.empty()) { + pipeline_collector_->enableCollection(rtn_); + } else { + for (const auto & node_name : enabled_nodes_) { + std::vector results; + rtn_->getSearchScope()->findChildren(node_name, results); + if (results.empty()) { + std::cerr << SPARTA_CURRENT_COLOR_RED + << "WARNING (Pipeline collection): Could not find node named: '" + << node_name + <<"' Collection will not occur on that node!" + << SPARTA_CURRENT_COLOR_NORMAL + << std::endl; + } + for (auto & tn : results) { + std::cout << "Collection enabled on node: '" << tn->getLocation() << "'" << std::endl; + pipeline_collector_->enableCollection(tn); + } + } + } + + pipeline_collector_->finalizeCollector(rtn_); + } + + void stopCollection_() + { + pipeline_collector_->disableCollection(); + } + std::unique_ptr pipeline_collector_; + sparta::RootTreeNode * rtn_; + std::set enabled_nodes_; }; /*! diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index bba099b3b5..1e1f94e70e 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -32,13 +32,9 @@ namespace collection { public: PipelineCollector(const std::string& simdb_filename, - sparta::RootTreeNode * rtn, - const size_t heartbeat = 10, - const std::set& enabled_nodes = {}) + const size_t heartbeat = 10) : db_mgr_(simdb_filename, true) , filename_(simdb_filename) - , root_(rtn) - , ev_set_(nullptr) { // Note that we only care about the collection data and have // no need for any other tables, aside from the tables that the @@ -49,7 +45,6 @@ namespace collection std::function func_ptr = std::bind(&PipelineCollector::getCollectionTick_, this); db_mgr_.getCollectionMgr()->useTimestampsFrom(func_ptr); db_mgr_.getCollectionMgr()->setHeartbeat(heartbeat); - createCollections_(enabled_nodes); } ~PipelineCollector() { @@ -61,90 +56,106 @@ namespace collection return filename_; } - void startCollecting() { - for (auto c : collectables_) { - c->enable(); + void enableCollection(TreeNode* starting_node) { + for (auto c : getCollectablesFrom_(starting_node)) { + if (!scheduler_) { + scheduler_ = c->getScheduler(); + } + auto &collectables = collectables_by_clk_[c->getClock()]; + if (!collectables) { + collectables.reset(new CollectablesByClock(c->getClock(), db_mgr_)); + } + collectables->addToAutoCollection(c); } + } - // Schedule collect event in the next cycle in case - // this is called in an unavailable phase. - ev_collect_->schedule(sparta::Clock::Cycle(1)); + void finalizeCollector(RootTreeNode* rtn) { + createCollections_(rtn); } - void stopCollecting() { + void disableCollection() { db_mgr_.onPipelineCollectorClosing(); db_mgr_.getConnection()->getTaskQueue()->stopThread(); db_mgr_.closeDatabase(); } private: + std::vector getCollectablesFrom_(TreeNode* starting_node) const + { + std::vector collectables; + std::function recursiveFindCollectables; + recursiveFindCollectables = [&recursiveFindCollectables, &collectables](TreeNode* node) { + if (auto c = dynamic_cast(node)) { + collectables.push_back(c); + } + for (auto & child : node->getChildren()) { + recursiveFindCollectables(child); + } + }; + recursiveFindCollectables(starting_node); + return collectables; + } + uint64_t getCollectionTick_() const { return scheduler_->getCurrentTick() - 1; } - void createCollections_(const std::set& enabled_nodes) { + void createCollections_(RootTreeNode* root) { CollectionPoints collectables; - recurseAddCollectables_(root_, collectables, enabled_nodes); + recurseAddCollectables_(root, collectables); collectables.createCollections(db_mgr_.getCollectionMgr()); db_mgr_.finalizeCollections(); - - recurseFindFastestCollectableClock_(root_, fastest_clk_, enabled_nodes); - sparta_assert(fastest_clk_, "No clock found! Are there any collectables?"); - - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(fastest_clk_)->getName() + "_auto_collection_event_collection", - CREATE_SPARTA_HANDLER(PipelineCollector, performCollection_), 1)); - - ev_collect_->setScheduleableClock(fastest_clk_); - ev_collect_->setScheduler(fastest_clk_->getScheduler()); - ev_collect_->setContinuing(false); - scheduler_ = fastest_clk_->getScheduler(); - sparta_assert(scheduler_, "Cannot run pipeline collection without a scheduler"); } void recurseAddCollectables_(sparta::TreeNode * node, - CollectionPoints & collectables, - const std::set& enabled_nodes) + CollectionPoints & collectables) { if (auto c = dynamic_cast(node)) { - if (enabled_nodes.empty() || enabled_nodes.count(c->getLocation())) { + if (c->isCollected()) { c->addCollectionPoint(collectables); - collectables_.push_back(c); } } for (auto & child : node->getChildren()) { - recurseAddCollectables_(child, collectables, enabled_nodes); + recurseAddCollectables_(child, collectables); } } - void recurseFindFastestCollectableClock_(const sparta::TreeNode * node, - const Clock *& fastest_clk, - const std::set& enabled_nodes) + class CollectablesByClock { - if (auto c = dynamic_cast(node)) { - if (!enabled_nodes.empty() && !enabled_nodes.count(c->getLocation())) { - return; - } - if (c->getClock() != nullptr) { - if (fastest_clk == nullptr) { - fastest_clk = c->getClock(); - } else { - if (c->getClock()->getPeriod() < fastest_clk->getPeriod()) { - fastest_clk = c->getClock(); - } - } - } + public: + CollectablesByClock(const Clock* clk, simdb::DatabaseManager &db_mgr) + : db_mgr_(db_mgr) + , ev_set_(nullptr) + { + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_collection", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection_), 1)); + ev_collect_->setScheduleableClock(clk); + ev_collect_->setScheduler(clk->getScheduler()); + ev_collect_->setContinuing(false); } - for (auto & child : node->getChildren()) { - recurseFindFastestCollectableClock_(child, fastest_clk, enabled_nodes); + void addToAutoCollection(CollectableTreeNode* ctn) { + ctn->enable(); + sparta_assert(clk_domain_ == ctn->getClock()->getName() || clk_domain_.empty()); + clk_domain_ = ctn->getClock()->getName(); + + // Schedule collect event in the next cycle in case + // this is called in an unavailable phase. + ev_collect_->schedule(sparta::Clock::Cycle(1)); } - } - void performCollection_() { - db_mgr_.getCollectionMgr()->collectAll(); - ev_collect_->schedule(); - } + private: + void performCollection_() { + db_mgr_.getCollectionMgr()->collectDomain(clk_domain_); + ev_collect_->schedule(); + } + + simdb::DatabaseManager &db_mgr_; + sparta::EventSet ev_set_; + std::unique_ptr> ev_collect_; + std::string clk_domain_; + }; //! The SimDB database simdb::DatabaseManager db_mgr_; @@ -152,23 +163,11 @@ namespace collection //! The SimDB filename e.g. "pipeline.db" std::string filename_; - //! The root tree node - sparta::RootTreeNode * root_ = nullptr; - - //! The event set for the performCollection_() callback event - EventSet ev_set_; - - //! The unique event for the performCollection_() callback - std::unique_ptr ev_collect_; - - //! The simulation scheduler - const sparta::Scheduler * scheduler_; - - //! Fastest clock across all collectable tree nodes - const Clock * fastest_clk_ = nullptr; + //! Scheduler + sparta::Scheduler * scheduler_ = nullptr; - //! All collectables. - std::vector collectables_; + //! Collectables by clock + std::unordered_map> collectables_by_clk_; }; }// namespace collection diff --git a/sparta/sparta/resources/Pipe.hpp b/sparta/sparta/resources/Pipe.hpp index b6340ea851..fb926813cf 100644 --- a/sparta/sparta/resources/Pipe.hpp +++ b/sparta/sparta/resources/Pipe.hpp @@ -515,10 +515,12 @@ class Pipe */ template void enableCollection(TreeNode * parent) { - static_assert(phase == SchedulingPhase::Collection, - "TODO cnyce: Only Collection phase is supported for Pipe"); - - collector_ = std::make_unique(parent, name_, this, capacity()); + if constexpr (phase == SchedulingPhase::Collection) { + collector_ = std::make_unique(parent, name_, this, capacity()); + } else { + std::cout << "ERROR: sparta::Pipe '" << (parent->getLocation() + "." + name_ + "' ") + << "only supports collection in SchedulingPhase::Collection" << std::endl; + } } /** diff --git a/sparta/test/Pipe/Pipe_test.cpp b/sparta/test/Pipe/Pipe_test.cpp index 95f92c7d76..34110944a0 100644 --- a/sparta/test/Pipe/Pipe_test.cpp +++ b/sparta/test/Pipe/Pipe_test.cpp @@ -69,7 +69,7 @@ int main () // sparta::log::categories::DEBUG, std::cout); pipe1.enableCollection(&rtn); - // TODO cnyce: pipe2.enableCollection(&rtn); + pipe2.enableCollection(&rtn); sparta::PayloadEvent ev(&es, "dummy_ev", diff --git a/sparta/test/Pipeline/Pipeline_test.cpp b/sparta/test/Pipeline/Pipeline_test.cpp index 654f59139a..e914ed1ab6 100644 --- a/sparta/test/Pipeline/Pipeline_test.cpp +++ b/sparta/test/Pipeline/Pipeline_test.cpp @@ -311,7 +311,6 @@ int main () sparta::PayloadEvent ev_flush_one (&es, "ev_flush_one", CREATE_SPARTA_HANDLER_WITH_DATA_WITH_OBJ(DummyClass2, &dummyObj2, flushOne, uint32_t)); -#if 0 // TODO cnyce EXPECT_FALSE(examplePipeline1.isCollected()); examplePipeline1.enableCollection(&rtn); EXPECT_FALSE(examplePipeline1.isCollected()); @@ -324,20 +323,6 @@ int main () examplePipeline8.enableCollection(&rtn); examplePipeline9.enableCollection(&rtn); stwr_pipe.enableCollection(&rtn); -#else - EXPECT_FALSE(examplePipeline1.isCollected()); - examplePipeline1.enableCollection(&rtn); - EXPECT_FALSE(examplePipeline1.isCollected()); - examplePipeline2.enableCollection(&rtn); - examplePipeline3.enableCollection(&rtn); - examplePipeline4.enableCollection(&rtn); - examplePipeline5.enableCollection(&rtn); - examplePipeline6.enableCollection(&rtn); - examplePipeline7.enableCollection(&rtn); - examplePipeline8.enableCollection(&rtn); - examplePipeline9.enableCollection(&rtn); - stwr_pipe.enableCollection(&rtn); -#endif //////////////////////////////////////////////////////////////////////////////// // Pipeline stage handler registration From dc337ca69d37300e20544f3ba1d3c3ed8f3fcb6f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 11:25:44 -0600 Subject: [PATCH 08/49] SimDB / v3 integration --- sparta/example/CoreModel/src/BIU.hpp | 1 + .../CoreModel/src/ExampleSimulation.cpp | 15 + sparta/example/CoreModel/src/Execute.cpp | 4 +- sparta/example/CoreModel/src/Execute.hpp | 17 +- sparta/example/CoreModel/src/Fetch.cpp | 3 +- sparta/example/CoreModel/src/Fetch.hpp | 5 +- sparta/example/CoreModel/src/LSU.hpp | 2 +- sparta/sparta/app/AppTriggers.hpp | 90 ++++- sparta/sparta/app/CommandLineSimulator.hpp | 7 +- sparta/sparta/collection/Collectable.hpp | 295 ++++++++++++++ .../sparta/collection/CollectableTreeNode.hpp | 316 +++++---------- .../sparta/collection/DelayedCollectable.hpp | 176 ++++++++ .../sparta/collection/IterableCollector.hpp | 287 +++++++++++++ .../sparta/collection/PipelineCollector.hpp | 376 +++++++++++++----- sparta/sparta/ports/DataPort.hpp | 9 +- sparta/sparta/ports/SyncPort.hpp | 24 +- sparta/sparta/resources/Array.hpp | 22 +- sparta/sparta/resources/Buffer.hpp | 12 +- sparta/sparta/resources/CircularBuffer.hpp | 2 +- sparta/sparta/resources/Pipe.hpp | 16 +- sparta/sparta/resources/Queue.hpp | 11 +- sparta/src/CommandLineSimulator.cpp | 43 +- sparta/test/Array/Array_test.cpp | 37 +- sparta/test/Buffer/Buffer_test.cpp | 19 +- .../CircularBuffer/CircularBuffer_test.cpp | 5 +- sparta/test/Pipe/Pipe_test.cpp | 22 +- sparta/test/Pipeline/Pipeline_test.cpp | 20 +- sparta/test/Port/Port_test.cpp | 2 + sparta/test/Queue/Queue_test.cpp | 20 +- sparta/test/SyncPort/SyncPort_test.cpp | 22 +- 30 files changed, 1431 insertions(+), 449 deletions(-) create mode 100644 sparta/sparta/collection/Collectable.hpp create mode 100644 sparta/sparta/collection/DelayedCollectable.hpp create mode 100644 sparta/sparta/collection/IterableCollector.hpp diff --git a/sparta/example/CoreModel/src/BIU.hpp b/sparta/example/CoreModel/src/BIU.hpp index fb71226f07..49ebb26678 100644 --- a/sparta/example/CoreModel/src/BIU.hpp +++ b/sparta/example/CoreModel/src/BIU.hpp @@ -9,6 +9,7 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" +#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "CoreTypes.hpp" diff --git a/sparta/example/CoreModel/src/ExampleSimulation.cpp b/sparta/example/CoreModel/src/ExampleSimulation.cpp index fb94b91598..f9a9fb5482 100644 --- a/sparta/example/CoreModel/src/ExampleSimulation.cpp +++ b/sparta/example/CoreModel/src/ExampleSimulation.cpp @@ -366,6 +366,21 @@ void ExampleSimulator::configureTree_() { validateTreeNodeExtensions_(); + // To get better coverage of the pipeline collector, we will use different clock + // domains for each core when collection is enabled. + auto pc = getSimulationConfiguration()->pipeline_collection_file_prefix != sparta::app::NoPipelineCollectionStr; + if (pc) { + auto root_clk = getClockManager().getRoot(); + for (uint32_t core_idx = 1; core_idx < num_cores_; ++core_idx) { + const std::string clk_name = "core" + std::to_string(core_idx) + "_clk"; + const auto numer = 1; + const auto denom = core_idx + 1; + auto core_clk = getClockManager().makeClock("core1_clk", root_clk, numer, denom); + auto core = getRoot()->getChild("cpu.core" + std::to_string(core_idx)); + core->setClock(core_clk.get()); + } + } + // In TREE_CONFIGURING phase // Configuration from command line is already applied diff --git a/sparta/example/CoreModel/src/Execute.cpp b/sparta/example/CoreModel/src/Execute.cpp index b1445593ee..ac512d43f5 100644 --- a/sparta/example/CoreModel/src/Execute.cpp +++ b/sparta/example/CoreModel/src/Execute.cpp @@ -13,7 +13,8 @@ namespace core_example ignore_inst_execute_time_(p->ignore_inst_execute_time), execute_time_(p->execute_time), scheduler_size_(p->scheduler_size), - in_order_issue_(p->in_order_issue) + in_order_issue_(p->in_order_issue), + collected_inst_(node, node->getName()) { in_execute_inst_. registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Execute, getInstsFromDispatch_, @@ -152,6 +153,7 @@ namespace core_example if(complete_inst_.getNumOutstandingEvents() == 0) { unit_busy_ = false; + collected_inst_.closeRecord(); } } diff --git a/sparta/example/CoreModel/src/Execute.hpp b/sparta/example/CoreModel/src/Execute.hpp index c49de846f3..d2c893b531 100644 --- a/sparta/example/CoreModel/src/Execute.hpp +++ b/sparta/example/CoreModel/src/Execute.hpp @@ -20,8 +20,8 @@ #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/Clock.hpp" #include "sparta/ports/Port.hpp" +#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" #include "CoreTypes.hpp" #include "FlushManager.hpp" @@ -82,15 +82,9 @@ namespace core_example const uint32_t execute_time_; const uint32_t scheduler_size_; const bool in_order_issue_; - - using IterableCollectorType = sparta::collection::IterableCollector>; - - // Collection - IterableCollectorType ready_queue_collector_{ - getContainer(), "scheduler_queue", &ready_queue_, scheduler_size_}; - - sparta::collection::ManualCollectable collected_inst_{ - getContainer(), getContainer()->getName(), getEventSet()}; + sparta::collection::IterableCollector> + ready_queue_collector_ {getContainer(), "scheduler_queue", + &ready_queue_, scheduler_size_}; // Events used to issue and complete the instruction sparta::UniqueEvent<> issue_inst_{&unit_event_set_, getName() + "_issue_inst", @@ -99,6 +93,9 @@ namespace core_example &unit_event_set_, getName() + "_complete_inst", CREATE_SPARTA_HANDLER_WITH_DATA(Execute, completeInst_, ExampleInstPtr)}; + // A pipeline collector + sparta::collection::Collectable collected_inst_; + // Counter sparta::Counter total_insts_issued_{ getStatisticSet(), "total_insts_issued", diff --git a/sparta/example/CoreModel/src/Fetch.cpp b/sparta/example/CoreModel/src/Fetch.cpp index f1b910bbbf..393303df12 100644 --- a/sparta/example/CoreModel/src/Fetch.cpp +++ b/sparta/example/CoreModel/src/Fetch.cpp @@ -104,7 +104,8 @@ namespace core_example Fetch::Fetch(sparta::TreeNode * node, const FetchParameterSet * p) : sparta::Unit(node), - num_insts_to_fetch_(p->num_to_fetch) + num_insts_to_fetch_(p->num_to_fetch), + next_pc_(node, "next_pc", &vaddr_) { in_fetch_queue_credits_. registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Fetch, receiveFetchQueueCredits_, uint32_t)); diff --git a/sparta/example/CoreModel/src/Fetch.hpp b/sparta/example/CoreModel/src/Fetch.hpp index e8a7e38e0b..9df40489cf 100644 --- a/sparta/example/CoreModel/src/Fetch.hpp +++ b/sparta/example/CoreModel/src/Fetch.hpp @@ -11,10 +11,10 @@ #include #include "sparta/ports/DataPort.hpp" #include "sparta/events/SingleCycleUniqueEvent.hpp" +#include "sparta/collection/Collectable.hpp" #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/TreeNode.hpp" #include "sparta/simulation/ParameterSet.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" #include "CoreTypes.hpp" @@ -103,8 +103,7 @@ namespace core_example std::unique_ptr> fetch_inst_event_; // A pipeline collector - sparta::collection::AutoCollectable next_pc_{ - getContainer(), "next_pc", &vaddr_}; + sparta::collection::Collectable next_pc_; //////////////////////////////////////////////////////////////////////////////// // Callbacks diff --git a/sparta/example/CoreModel/src/LSU.hpp b/sparta/example/CoreModel/src/LSU.hpp index 0700552016..9a3574e9e2 100644 --- a/sparta/example/CoreModel/src/LSU.hpp +++ b/sparta/example/CoreModel/src/LSU.hpp @@ -166,7 +166,7 @@ namespace core_example ExampleInstPtr cache_pending_inst_ptr_ = nullptr; // Collection - sparta::collection::AutoCollectable cache_busy_collectable_{ + sparta::collection::Collectable cache_busy_collectable_{ getContainer(), "dcache_busy", &cache_busy_}; // NOTE: diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index 04bd206672..3b0d15b76d 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -29,25 +29,31 @@ namespace sparta { class PipelineTrigger : public trigger::Triggerable { public: - PipelineTrigger(const std::string& simdb_filename, - sparta::RootTreeNode * rtn, - const size_t heartbeat = 10, - const std::set& enabled_nodes = {}) - : rtn_(rtn) - , enabled_nodes_(enabled_nodes) + PipelineTrigger(const std::string& pipeline_collection_path, + const std::set& pipeline_enabled_node_names, + uint64_t pipeline_heartbeat, + bool multiple_triggers, + sparta::RootTreeNode * rtn) : + pipeline_collection_path_(pipeline_collection_path), + pipeline_enabled_node_names_(pipeline_enabled_node_names), + multiple_triggers_(multiple_triggers), + root_(rtn) { - pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, heartbeat)); + auto simdb_filename = getCollectionPath_(); + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, pipeline_heartbeat, rtn)); } void go() override { sparta_assert(!triggered_, "Why has pipeline trigger been triggered?"); triggered_ = true; - std::cout << "Pipeline collection started, output to database file '" << pipeline_collector_->getFilePath() << "'" << std::endl; - startCollection_(); + + if(multiple_triggers_) { + std::cout << "#" << num_collections_ << " pipeline collection started" << std::endl; + } } void stop() override @@ -55,18 +61,27 @@ class PipelineTrigger : public trigger::Triggerable sparta_assert(triggered_, "Why stop an inactivated trigger?"); triggered_ = false; stopCollection_(); + + if(multiple_triggers_) { + std::cout << "#" << num_collections_ << " pipeline collection ended" << std::endl; + ++num_collections_; + pipeline_collector_->reactivate(getCollectionPath_()); + } } private: void startCollection_() { - if (enabled_nodes_.empty()) { - pipeline_collector_->enableCollection(rtn_); - } else { - for (const auto & node_name : enabled_nodes_) { + if(pipeline_enabled_node_names_.empty()) { + // Start collection at the root node + pipeline_collector_->startCollection(root_); + } + else { + // Find the nodes in the root and enable them + for(const auto & node_name : pipeline_enabled_node_names_) { std::vector results; - rtn_->getSearchScope()->findChildren(node_name, results); - if (results.empty()) { + root_->getSearchScope()->findChildren(node_name, results); + if(results.size() == 0) { std::cerr << SPARTA_CURRENT_COLOR_RED << "WARNING (Pipeline collection): Could not find node named: '" << node_name @@ -74,24 +89,55 @@ class PipelineTrigger : public trigger::Triggerable << SPARTA_CURRENT_COLOR_NORMAL << std::endl; } - for (auto & tn : results) { + for(auto & tn : results) { std::cout << "Collection enabled on node: '" << tn->getLocation() << "'" << std::endl; - pipeline_collector_->enableCollection(tn); + pipeline_collector_->startCollection(tn); } } } - - pipeline_collector_->finalizeCollector(rtn_); } void stopCollection_() { - pipeline_collector_->disableCollection(); + if(pipeline_enabled_node_names_.empty()) { + // Start collection at the root node + pipeline_collector_->stopCollection(root_); + } + else { + // Find the nodes in the root and enable them + for(const auto & node_name : pipeline_enabled_node_names_) { + std::vector results; + root_->getSearchScope()->findChildren(node_name, results); + for(auto & tn : results) { + (void)tn; + pipeline_collector_->stopCollection(tn); + } + } + } + pipeline_collector_->destroy(); + } + + std::string getCollectionPath_() const + { + if (num_collections_ == 0) { + return pipeline_collection_path_; + } + + auto p = pipeline_collection_path_; + auto dot = p.rfind(".db"); + sparta_assert(dot != std::string::npos, "Database filename must end in .db"); + + p = p.substr(0, dot); + p += "_" + std::to_string(num_collections_) + ".db"; + return p; } std::unique_ptr pipeline_collector_; - sparta::RootTreeNode * rtn_; - std::set enabled_nodes_; + const std::string pipeline_collection_path_; + const std::set pipeline_enabled_node_names_; + const bool multiple_triggers_; + sparta::RootTreeNode * root_ = nullptr; + uint32_t num_collections_ = 0; }; /*! diff --git a/sparta/sparta/app/CommandLineSimulator.hpp b/sparta/sparta/app/CommandLineSimulator.hpp index c38057aaa5..b7a9d57b91 100644 --- a/sparta/sparta/app/CommandLineSimulator.hpp +++ b/sparta/sparta/app/CommandLineSimulator.hpp @@ -55,8 +55,6 @@ class InformationWriter; namespace app { -const constexpr char DefaultHeartbeat[] = "0"; - /*! * \class CommandLineSimulator @@ -460,10 +458,9 @@ class CommandLineSimulator std::unique_ptr info_out_; /*! - * \brief Heartbeat period of pipeline collection file (before lexical cast - * or validation) + * \brief Heartbeat period of pipeline collection file. */ - std::string pipeline_heartbeat_ = DefaultHeartbeat; + uint64_t pipeline_heartbeat_ = 10; //! The names of the nodes to be enabled std::set pipeline_enabled_node_names_; diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp new file mode 100644 index 0000000000..4927977817 --- /dev/null +++ b/sparta/sparta/collection/Collectable.hpp @@ -0,0 +1,295 @@ +// -*- C++ -*- + +/** + * \file Collectable.hpp + * + * \brief Implementation of the Collectable class that allows + * a user to collect an object into an pipeViewer pipeline file + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "sparta/collection/PipelineCollector.hpp" +#include "sparta/events/PayloadEvent.hpp" +#include "sparta/events/EventSet.hpp" +#include "sparta/events/SchedulingPhases.hpp" +#include "sparta/utils/Utils.hpp" +#include "sparta/utils/MetaStructs.hpp" + +namespace sparta{ + namespace collection + { + + /** + * \class Collectable + * \brief Class used to either manually or auto-collect an Annotation String + * object in a pipeline database + * \tparam DataT The DataT of the collectable being collected + * \tparam collection_phase The phase collection will occur. If + * sparta::SchedulingPhase::Collection, + * collection is done prior to Tick. + * + * Auto-collection will occur only if a Collectable is + * constructed with a collected_object. If no object is + * provided, it assumes a manual collection and the + * scheduling phase is ignored. + */ + + template + class Collectable : public CollectableTreeNode + { + public: + /** + * \brief Construct the Collectable, no data object associated, part of a group + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param group the name of the group for this treenode + * \param index the index within the group + * \param desc A description for the interface + */ + Collectable(sparta::TreeNode* parent, + const std::string& name, + const std::string& group, + uint32_t index, + const std::string & desc = "Collectable ") : + CollectableTreeNode(sparta::notNull(parent), name, group, index, desc), + event_set_(this), + ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", + CREATE_SPARTA_HANDLER_WITH_DATA(Collectable, closeRecord, bool)) + { + } + + /** + * \brief Construct the Collectable + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param collected_object Pointer to the object to collect during the "COLLECT" phase + * \param desc A description for the interface + */ + Collectable(sparta::TreeNode* parent, + const std::string& name, + const DataT * collected_object, + const std::string & desc = "Collectable ") : + Collectable(parent, name, + TreeNode::GROUP_NAME_NONE, + TreeNode::GROUP_IDX_NONE, + desc) + { + collected_object_ = collected_object; + + //TODO cnyce: prev_annot_ / initialize() + } + + /** + * \brief Construct the Collectable, no data object associated + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param desc A description for the interface + */ + Collectable(sparta::TreeNode* parent, + const std::string& name, + const std::string & desc = "Collectable ") : + Collectable(parent, name, nullptr, desc) + { + // Can't auto collect without setting collected_object_ + setManualCollection(); + } + + //! Virtual destructor -- does nothing + virtual ~Collectable() {} + + //! Explicitly/manually collect a value for this collectable, ignoring + //! what the Collectable is currently pointing to. + //! Here we pass the actual object of the collectable type we are collecting. + template + MetaStruct::enable_if_t::value, void> + collect(const T & val) + { + //TODO cnyce + (void)val; + + if(SPARTA_EXPECT_FALSE(isCollected())) + { + //std::ostringstream ss; + //ss << val; + //if((ss.str() != prev_annot_) && !record_closed_) + //{ + // // Close the old record (if there is one) + // closeRecord(); + //} + + // Remember the new string for a new record and start + // a new record if not empty. + //prev_annot_ = ss.str(); + //if(!prev_annot_.empty() && record_closed_) { + // startNewRecord_(); + // record_closed_ = false; + //} + } + } + + //! Explicitly/manually collect a value for this collectable, ignoring + //! what the Collectable is currently pointing to. + //! Here we pass the shared pointer to the actual object of the collectable type we are collecting. + template + MetaStruct::enable_if_t::value, void> + collect(const T & val){ + // If pointer has become nullified, close the record + if(nullptr == val) { + closeRecord(); + return; + } + collect(*val); + } + + /*! + * \brief Explicitly collect a value for the given duration + * \param duration The amount of time in cycles the value is available + * + * Explicitly collect a value for this collectable for the + * given amount of time. + * + * \warn No checks are performed if a new value is collected + * within the previous duration! + */ + template + MetaStruct::enable_if_t::value, void> + collectWithDuration(const T & val, sparta::Clock::Cycle duration){ + if(SPARTA_EXPECT_FALSE(isCollected())) + { + if(duration != 0) { + ev_close_record_.preparePayload(false)->schedule(duration); + } + collect(val); + } + } + + /*! + * \brief Explicitly collect a value from a shared pointer for the given duration + * \param duration The amount of time in cycles the value is available + * + * Explicitly collect a value for this collectable passed as a shared pointer for the + * given amount of time. + * + * \warn No checks are performed if a new value is collected + * within the previous duration! + */ + template + MetaStruct::enable_if_t::value, void> + collectWithDuration(const T & val, sparta::Clock::Cycle duration){ + // If pointer has become nullified, close the record + if(nullptr == val) { + closeRecord(); + return; + } + collectWithDuration(*val, duration); + } + + //! Virtual method called by + //! CollectableTreeNode/PipelineCollector when a user of the + //! TreeNode requests this object to be collected. + void collect() override final { + // If pointer has become nullified, close the record + if(nullptr == collected_object_) { + closeRecord(); + return; + } + collect(*collected_object_); + } + + /*! + * \brief Calls collectWithDuration using the internal collected_object_ + * specified at construction. + * \pre Must have constructed wit ha non-null collected object + */ + void collectWithDuration(sparta::Clock::Cycle duration) { + // If pointer has become nullified, close the record + if(nullptr == collected_object_) { + closeRecord(); + return; + } + collectWithDuration(*collected_object_, duration); + } + + //! Force close a record. + void closeRecord(const bool & simulation_ending = false) override final + { + if(SPARTA_EXPECT_FALSE(isCollected())) + { + if(!record_closed_) { + writeRecord_(simulation_ending); + record_closed_ = true; + } + } + } + + //! \brief Do not perform any automatic collection + //! The SchedulingPhase is ignored + void setManualCollection() + { + auto_collect_ = false; + } + + protected: + + //! \brief Get a reference to the internal event set + //! \return Reference to event set -- used by DelayedCollectable + EventSet & getEventSet_() { + return event_set_; + } + + private: + bool writeRecord_(bool simulation_ending = false) + { + //TODO cnyce + (void)simulation_ending; + return true; + } + + void setCollecting_(bool collect, PipelineCollector * collector, simdb::DatabaseManager*) override final + { + // If the collected object is null, this Collectable + // object is to be explicitly collected + if(collected_object_ && auto_collect_) { + if(collect) { + // Add this Collectable to the PipelineCollector's + // list of objects requiring collection + collector->addToAutoCollection(this, collection_phase); + } + else { + // Remove this Collectable from the + // PipelineCollector's list of objects requiring + // collection + collector->removeFromAutoCollection(this); + } + } + + if(!collect && !record_closed_) { + // Force the record to be written + closeRecord(); + } + } + + // The annotation object to be collected + const DataT * collected_object_ = nullptr; + + // For those folks that want a value to automatically + // disappear in the future + sparta::EventSet event_set_; + sparta::PayloadEvent ev_close_record_; + + // Is this collectable currently closed? + bool record_closed_ = true; + + // Should we auto-collect? + bool auto_collect_ = true; + }; + +}//namespace collection +}//namespace sparta diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index b4462fdda1..0d8107f844 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -1,3 +1,13 @@ +// -*- C++ -*- + + +/** + *\file CollectableTreeNode.hpp + * + * + *\brief define a CollectableTreeNode type TreeNode. + */ + #pragma once #include "sparta/simulation/TreeNode.hpp" @@ -6,228 +16,116 @@ #include "sparta/events/PhasedUniqueEvent.hpp" #include "sparta/events/EventSet.hpp" -namespace sparta { -namespace collection { - -class CollectionPoints; - -// Base class for all collectable classes. -class CollectableTreeNode : public TreeNode -{ -public: - CollectableTreeNode(TreeNode* parent, const std::string& name, const std::string& desc = "CollectableTreeNode ") - : TreeNode(parent, name, desc) - { - markHidden(); - } - - virtual void addCollectionPoint(CollectionPoints & collection_points) = 0; - - bool isCollected() const { return collected_; } - - void enable() { collected_ = true; } - - // Note there is no way to disable collection on a per-collectable basis. You can - // only call PipelineCollector::stopCollecting() to stop all collection for all - // collectables. +namespace simdb { + class DatabaseManager; +} -private: - bool collected_ = false; -}; - -// Manually-collected means that the user must explicitly call collect() or collectWithDuration() -// and cannot give a backpointer to this class' constructor for automatic every-cycle collection. -template -class ManualCollectable : public CollectableTreeNode +namespace sparta{ +namespace collection { -public: - ManualCollectable(TreeNode* parent, const std::string& name, sparta::EventSet* ev_set, const std::string& desc = "ManualCollectable ") - : CollectableTreeNode(parent, name, desc) - , ev_end_duration_(getEventSet_(parent, ev_set), name + "_end_duration_event", SchedulingPhase::Collection, CREATE_SPARTA_HANDLER(ManualCollectable, endDuration_)) - { - sparta_assert(getClock()); - } - void addCollectionPoint(CollectionPoints & collection_points) override + /** + * \class CollectableTreeNode + * \brief An abstract type of TreeNode that + * has virtual calls to start collection on this node, + * and stop collection on this node. + */ + class CollectableTreeNode : public TreeNode { - collector_ = collection_points.addManualCollectable(getLocation(), getClock()); - } - - using CollectableTreeNode::isCollected; - - void collect(const CollectedT & dat) - { - if (isCollected()) { - collector_->collectOnce(dat); - } - } - - void collectWithDuration(const CollectedT & dat, const Clock::Cycle dur) - { - if (isCollected()) { - collector_->beginZOH(dat); - ev_end_duration_.schedule(getClock()->getTick(dur)); - } - } - -private: - EventSet* getEventSet_(TreeNode* parent, EventSet* ev_set) - { - if (ev_set) { - return ev_set; - } else { - ev_set_.reset(new EventSet(parent)); - return ev_set_.get(); - } - } - - void endDuration_() - { - if (isCollected()) { - collector_->endZOH(); + public: + /** + * \brief Construct. + * \param parent a pointer to the parent treenode + * \param name the name of this treenode + * \param group the name of the group for this treenode + * \param index the index within the group + * \param desc A description for this treenode. + */ + CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, + const std::string& group, uint32_t index, + const std::string& desc = "CollectableTreeNode ") : + sparta::TreeNode(parent, name, group, index, desc) + { + markHidden(); // Mark self as hidden from the default + // printouts (to reduce clutter) } - } - - simdb::AnyCollection* collector_ = nullptr; - sparta::PhasedUniqueEvent ev_end_duration_; - std::unique_ptr ev_set_; -}; - -// Similar to the ManualCollectable class, you cannot use a DelayedCollectable -// for any kind of automatic collection. You must explicitly call collect() or -// collectWithDuration() to collect data. -template -class DelayedCollectable : public ManualCollectable -{ - struct DurationData - { - DurationData() = default; - DurationData(const CollectedT & dat, - sparta::Clock::Cycle duration) - : data(dat) - , duration(duration) + /** + * \brief Construct. + * \param parent a pointer to the parent treenode + * \param name the name of this treenode + * \param desc Description of this CollectableTreeNode + */ + CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, + const std::string& desc= "CollectableTreeNode ") : + CollectableTreeNode(parent, name, sparta::TreeNode::GROUP_NAME_NONE, + sparta::TreeNode::GROUP_IDX_NONE, desc) {} - CollectedT data; - sparta::Clock::Cycle duration; - }; - -public: - DelayedCollectable(TreeNode* parent, const std::string& name, sparta::EventSet* ev_set, const std::string& desc = "DelayedCollectable ") - : ManualCollectable(parent, name, ev_set, desc) - , ev_collect_(ev_set, name + "_event", CREATE_SPARTA_HANDLER_WITH_DATA(ManualCollectable, collect, CollectedT)) - , ev_collect_duration_(ev_set, name + "_duration_event", CREATE_SPARTA_HANDLER_WITH_DATA(DelayedCollectable, collectWithDuration_, DurationData)) - { - } - - using ManualCollectable::isCollected; - - void collect(const CollectedT & dat, uint64_t delay) - { - if (isCollected()) { - if (delay == 0) { - ManualCollectable::collect(dat); - } else { - ev_collect_.schedule(dat, delay); - } - } - } - - void collectWithDuration(const CollectedT & dat, uint64_t delay, const Clock::Cycle dur) - { - if (isCollected()) { - if (delay == 0) { - ManualCollectable::collectWithDuration(dat, dur); - } else { - ev_collect_duration_.preparePayload({dat, dur})->schedule(delay); - } - } - } - -private: - // Called from collectWithDuration where the data needs to be - // delivered at a given delayed time, but only for a short - // duration. - void collectWithDuration_(const DurationData & dur_dat) - { - ManualCollectable::collectWithDuration(dur_dat.data, dur_dat.duration); - } - - // For those folks that want collection to appear in the future - sparta::PayloadEvent ev_collect_; - - // For those folks that want collection to appear in the future with a duration - sparta::PayloadEvent ev_collect_duration_; -}; - -// Auto-collectable means that the user can give a backpointer to this class' constructor -// and we will collect your data every cycle automatically. There is no way to automatically -// collect at any other time than "every cycle". -// -// This class is to be used for non-iterable types like uint64_t, struct, bool, etc. -// You should use the IterableCollector class for iterable types like std::vector, sparta::Array, etc. -template -class AutoCollectable : public CollectableTreeNode -{ -public: - AutoCollectable(TreeNode* parent, const std::string& name, const CollectedT* back_ptr, const std::string& desc = "AutoCollectable ") - : CollectableTreeNode(parent, name, desc) - , back_ptr_(back_ptr) - { - static_assert(std::is_integral::value || std::is_floating_point::value, - "AutoCollectable only supports integral and floating-point types!"); - } + //!Virtual destructor + virtual ~CollectableTreeNode() + {} - void addCollectionPoint(CollectionPoints & collection_points) override - { - if constexpr (std::is_same::value) { - using value_type = int32_t; - auto getter = std::function([this]() { return *back_ptr_ ? 1 : 0; }); - collection_points.addStat(getLocation(), getClock(), getter); - } else { - collection_points.addStat(getLocation(), getClock(), back_ptr_); + /** + * \brief Method that tells this treenode that is now running + * collection. + * \param collector The collector that is performing the collection + * + * This method should instantiate any necessary values for the + * treenode necessary to collection as well as flip the + * is_collecting boolean. + */ + void startCollecting(PipelineCollector* collector, simdb::DatabaseManager* db_mgr) { + is_collected_ = true; + setCollecting_(true, collector, db_mgr); } - } -private: - const CollectedT* back_ptr_; -}; - -template -class IterableCollector : public CollectableTreeNode -{ -public: - IterableCollector(TreeNode* parent, const std::string& name, const ContainerT* container, const size_t capacity, const std::string& desc = "IterableCollector ") - : CollectableTreeNode(parent, name, desc) - , container_(container) - , capacity_(capacity) - { - for (size_t i = 0; i < capacity_; ++i) { - bins_.emplace_back(new IterableCollectorBin(this, name + std::to_string(i))); + /** + * \brief Method that tells this treenode that is now not + * running collection. + */ + void stopCollecting(PipelineCollector* collector, simdb::DatabaseManager* db_mgr) { + setCollecting_(false, collector, db_mgr); + is_collected_ = false; } - } - void addCollectionPoint(CollectionPoints & collection_points) override - { - collection_points.addContainer(getLocation(), getClock(), container_, capacity_); - } + /** + * \brief Determine whether or not this node has collection + * turned on or off. + */ + bool isCollected() const { return is_collected_; } + + //! Pure virtual method used by deriving classes to be + //! notified when they can perform their collection + virtual void collect() = 0; + + //! Pure virtual method used by deriving classes to force + //! close a record. This is useful for simulation end where + //! each collectable is allowed its final say + //! \param simulation_ending If true, the simulation is + //! terminating and the "end cycle" is not really the + //! true end. Implementers of this method should a +1 + //! to their end cycle in their records to ensure it + //! does not get closed out. + virtual void closeRecord(const bool & simulation_ending = false) { (void) simulation_ending; } + + protected: + + /** + * \brief Indicate to sub-classes that collection has flipped + * \param collect true if collection is enabled; false otherwise + */ + virtual void setCollecting_(bool collect, PipelineCollector*, simdb::DatabaseManager*) { (void)collect; } + + private: + + /** + * \brief A value that represents whether or not this TreeNode is + * being collected by a collector + */ + bool is_collected_ = false; -private: - class IterableCollectorBin : public TreeNode - { - public: - IterableCollectorBin(TreeNode* parent, const std::string& name, const std::string& desc = "IterableCollectorBin ") - : TreeNode(parent, name, desc) - { - markHidden(); - } }; - const ContainerT* container_; - const size_t capacity_; - std::vector> bins_; -}; - -} // namespace collection -} // namespace sparta +} +} diff --git a/sparta/sparta/collection/DelayedCollectable.hpp b/sparta/sparta/collection/DelayedCollectable.hpp new file mode 100644 index 0000000000..159b885677 --- /dev/null +++ b/sparta/sparta/collection/DelayedCollectable.hpp @@ -0,0 +1,176 @@ +// -*- C++ -*- + +/** + * \file DelayedCollectable.hpp + * + * \brief Implementation of the DelayedCollectable class that allows + * a user to collect an object into an pipeViewer pipeline file + */ + +#pragma once + +#include +#include +#include "sparta/collection/Collectable.hpp" +#include "sparta/pipeViewer/transaction_structures.hpp" +#include "sparta/events/EventSet.hpp" +#include "sparta/events/PayloadEvent.hpp" + +namespace sparta { + +namespace collection +{ + /** + * \class DelayedCollectable + * \brief Class used to record data in pipeline collection, but + * recorded in a delayed fashion + * + * DelayedCollectable is useful for delivering a collected chunk + * of data to the PipelineCollector in the future. Such an + * example is SyncPort where data can be sent with a delay of N + * and the view should show this data only on cycle N for + * delivery. + */ + template + class DelayedCollectable : public Collectable + { + using CollectableTreeNode::isCollected; + + struct DurationData + { + DurationData() {} + + DurationData(const DataT & dat, + sparta::Clock::Cycle duration) : + data(dat), duration(duration) + {} + + DataT data; + sparta::Clock::Cycle duration; + }; + + public: + + // Promote the collect method from Collectable + using Collectable::collect; + + /** + * \brief Construct the DelayedCollectable, no data object associated, part of a group + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param group The group for which to create this object as a child sparta::TreeNode + * \param index The index within the group for this collectable + * \param desc A description for the interface + */ + DelayedCollectable(sparta::TreeNode* parent, + const std::string& name, + const std::string& group, + uint32_t index, + const std::string & desc = "DelayedCollectable ") : + Collectable(parent, name, group, index, desc) + { + } + + /** + * \brief Construct the DelayedCollectable + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param collected_object Pointer to the object to collect during the "COLLECT" phase + * \param desc A description for the interface + */ + DelayedCollectable(sparta::TreeNode* parent, + const std::string& name, + const DataT * collected_object, + const std::string & desc = "DelayedCollectable ") : + Collectable(parent, name, collected_object, desc) + {} + + /** + * \brief Construct the DelayedCollectable, no data object associated + * \param parent A pointer to a parent treenode. Must not be null + * \param name The name for which to create this object as a child sparta::TreeNode + * \param desc A description for the interface + */ + DelayedCollectable(sparta::TreeNode* parent, + const std::string& name, + const std::string & desc = "DelayedCollectable ") : + DelayedCollectable(parent, name, nullptr, desc) + { + // Can't auto collect without setting iterable_object_ + Collectable::setManualCollection(); + } + + /*! + * \brief Explicitly collect a value in the future + * \param val The value to collect in the future + * \param delay The delay before recording this value to file + * + * Explicitly collect a value for this collectable in the + * future, ignoring what the Collectable is currently pointing + * to. + */ + void collect(const DataT & val, sparta::Clock::Cycle delay) + { + if(SPARTA_EXPECT_FALSE(isCollected())) + { + if(delay != 0) { + ev_collect_.schedule(val, delay); + } + else { + Collectable::collect(val); + } + } + } + + /*! + * \brief Explicitly collect a value in the future + * \param val The value to collect in the future + * \param delay The delay before recording this value to file + * \param duration The amount of time in cycles the value is available + * + * Explicitly collect a value for this collectable in the + * future, ignoring what the Collectable is currently pointing + * to. + */ + void collectWithDuration(const DataT & val, + sparta::Clock::Cycle delay, + sparta::Clock::Cycle duration) + { + if(SPARTA_EXPECT_FALSE(isCollected())) + { + if(delay != 0) { + ev_collect_duration_.preparePayload({val, duration})->schedule(delay); + } + else { + Collectable::collectWithDuration(val, duration); + } + } + } + + private: + + // Called from collectWithDuration where the data needs to be + // delivered at a given delayed time, but only for a short + // duration. + void collectWithDuration_(const DurationData & dur_dat) { + Collectable::collectWithDuration(dur_dat.data, dur_dat.duration); + } + + // For those folks that want collection to appear in the + // future + sparta::PayloadEvent ev_collect_{ + &Collectable::getEventSet_(), "delayedpipelinecollectable_event", + CREATE_SPARTA_HANDLER_WITH_DATA(Collectable, collect, DataT)}; + + // For those folks that want collection to appear in the + // future with a duration + sparta::PayloadEvent ev_collect_duration_{ + &Collectable::getEventSet_(), "delayedpipelinecollectable_duration_event", + CREATE_SPARTA_HANDLER_WITH_DATA(DelayedCollectable, + collectWithDuration_, DurationData)}; + + + }; +}//namespace collection +}//namespace sparta + diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp new file mode 100644 index 0000000000..c23bf4a27c --- /dev/null +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -0,0 +1,287 @@ +// -*- C++ -*- + +/* + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include "sparta/utils/Utils.hpp" +#include "sparta/log/MessageSource.hpp" +#include "sparta/events/SchedulingPhases.hpp" +#include "sparta/collection/Collectable.hpp" +#include "sparta/collection/CollectableTreeNode.hpp" +#include "sparta/collection/PipelineCollector.hpp" + +namespace sparta { +namespace collection { + +template +class IterableCollector : public CollectableTreeNode +{ +public: + typedef typename IterableType::size_type size_type; + + /** + * \brief constructor + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param group Group this collector is part of + * \param index the index within the group + * \param desc Description of this node + * \param iterable Pointer to the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const std::string & group, + const uint32_t index, + const std::string & desc, + const IterableType * iterable, + const size_type expected_capacity) : + CollectableTreeNode(parent, name, group, index, desc), + iterable_object_(iterable), + expected_capacity_(expected_capacity), + event_set_(this), + ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", + CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)) + { + for (size_type i = 0; i < expected_capacity_; ++i) + { + std::stringstream name_str; + name_str << name << i; + positions_.emplace_back(new CollectableT(this, name_str.str(), group, i)); + } + } + + /** + * \brief constructor + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param group Group this collector is part of + * \param index the index within the group + * \param desc Description of this node + * \param iterable the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const std::string & group, + const uint32_t index, + const std::string & desc, + const IterableType & iterable, + const size_type expected_capacity) : + IterableCollector(parent, name, group, index, desc, &iterable, expected_capacity) + {} + + /** + * \brief constructor + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param desc Description of this node + * \param iterable Pointer to the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const std::string & desc, + const IterableType * iterable, + const size_type expected_capacity) : + IterableCollector(parent, name, name, 0, desc, iterable, expected_capacity) + {} + + /** + * \brief constructor + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param desc Description of this node + * \param iterable the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const std::string & desc, + const IterableType & iterable, + const size_type expected_capacity) : + IterableCollector(parent, name, name, 0, desc, &iterable, expected_capacity) + {} + + /** + * \brief constructor with no description + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param iterable Pointer to the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const IterableType * iterable, + const size_type expected_capacity) : + IterableCollector (parent, name, name + " Iterable Collector", + iterable, expected_capacity) + { + // Delegated constructor + } + + /** + * \brief constructor with no description + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param iterable the iterable object to collect + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const IterableType & iterable, + const size_type expected_capacity) : + IterableCollector (parent, name, name + " Iterable Collector", + &iterable, expected_capacity) + { + // Delegated constructor + } + + //! Collect the contents of the iterable object. This function + //! will walk starting from index 0 -> expected_capacity, clearing + //! out any records where the iterable object does not contain + //! data. + void collect(const IterableType * iterable_object) + { + // If pointer has become nullified, close the records + if(nullptr == iterable_object) { + closeRecord(); + } + else if (SPARTA_EXPECT_FALSE(isCollected())) + { + if(SPARTA_EXPECT_FALSE(iterable_object->size() > expected_capacity_)) + { + if(SPARTA_EXPECT_FALSE(warn_on_size_)) + { + sparta::log::MessageSource::getGlobalWarn() + << "WARNING! The collected object '" + << getLocation() << "' has grown beyond the " + << "expected capacity (given at construction) for collection. " + << "Expected " << expected_capacity_ << " but grew to " + << iterable_object->size() + << " This is your first and last warning."; + warn_on_size_ = false; + } + } + collectImpl_(iterable_object, std::integral_constant()); + } + } + + //! Collect the contents of the associated iterable object + void collect() override { + collect(iterable_object_); + } + + //! Force close all records for this iterable type. This will + //! close the record immediately and clear the field for the next + //! cycle + void closeRecord(const bool & simulation_ending = false) override { + for (size_type i = 0; i < positions_.size(); ++i) { + positions_[i]->closeRecord(simulation_ending); + } + } + + //! Schedule event to close all records for this iterable type. + void scheduleCloseRecord(sparta::Clock::Cycle cycle) { + if(SPARTA_EXPECT_FALSE(isCollected())) { + ev_close_record_.preparePayload(false)->schedule(cycle); + } + } + + //! \brief Do not perform any automatic collection + //! The SchedulingPhase is ignored + void setManualCollection() { + auto_collect_ = false; + } + + //! \brief Perform a collection, then close the records in the future + //! \param duration The time to close the records, 0 is not allowed + void collectWithDuration(sparta::Clock::Cycle duration) { + if(SPARTA_EXPECT_FALSE(isCollected())) { + collect(); + if(duration != 0) { + ev_close_record_.preparePayload(false)->schedule(duration); + } + } + } + + //! Reattach to a new iterable object (used for moves) + void reattach(const IterableType * obj) { + iterable_object_ = obj; + } + +private: + typedef Collectable::value_type> CollectableT; + // Standard walk of iterable types + void collectImpl_(const IterableType * iterable_object, std::false_type) + { + sparta_assert(nullptr != iterable_object, + "Can't collect iterable_object because it's a nullptr! How did we get here?"); + auto itr = iterable_object->begin(); + auto eitr = iterable_object->end(); + for (uint32_t i = 0; i < expected_capacity_; ++i) + { + if (itr != eitr) { + positions_[i]->collect(*itr); + ++itr; + } else { + positions_[i]->closeRecord(); + } + } + } + + // Full iteration walk, checking validity of the iterator. This + // is used for Pipe and Array where the iterator points to valid + // and not valid entries in the component + void collectImpl_(const IterableType * iterable_object, std::true_type) + { + sparta_assert(nullptr != iterable_object, + "Can't collect iterable_object because it's a nullptr! How did we get here?"); + uint32_t s = 0; + auto itr = iterable_object->begin(); + for (uint32_t i = 0; i < expected_capacity_; ++i, ++s) + { + if (itr.isValid()) { + positions_[i]->collect(*itr); + } else { + positions_[i]->closeRecord(); + } + ++itr; + } + } + + //! Virtual method called by CollectableTreeNode when collection + //! is enabled on the TreeNode + void setCollecting_(bool collect, PipelineCollector* collector, simdb::DatabaseManager* db_mgr) override { + if(collect && auto_collect_) { + // Add this Collectable to the PipelineCollector's + // list of objects requiring collection + collector->addToAutoCollection(this, collection_phase); + } + else { + closeRecord(); + } + } + + const IterableType * iterable_object_; + std::vector> positions_; + const size_type expected_capacity_ = 0; + bool auto_collect_ = true; + bool warn_on_size_ = true; + + // For those folks that want a value to automatically + // disappear in the future + sparta::EventSet event_set_; + sparta::PayloadEvent ev_close_record_; + +}; +} // namespace collection +} // namespace sparta diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 1e1f94e70e..ef4452c800 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -30,144 +30,332 @@ namespace collection { class PipelineCollector { + class CollectablesByClock + { + public: + CollectablesByClock(const Clock * clk, + SchedulingPhase collection_phase) : + ev_set_(nullptr) + { + switch(collection_phase) + { + case SchedulingPhase::Trigger: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_trigger", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::Update: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_update", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::PortUpdate: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_portupdate", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::Flush: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_flush", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::Collection: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_collection", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::Tick: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_tick", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::PostTick: + ev_collect_.reset(new sparta::UniqueEvent + (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_posttick", + CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection), 1)); + break; + case SchedulingPhase::__last_scheduling_phase: + sparta_assert(!"Should not have gotten here"); + break; + // NO DEFAULT! Allows for compiler errors if the enum + // class is updated. + } + ev_collect_->setScheduleableClock(clk); + ev_collect_->setScheduler(clk->getScheduler()); + ev_collect_->setContinuing(false); + } + + void enable(CollectableTreeNode * ctn) { + enabled_ctns_.insert(ctn); + // Schedule collect event in the next cycle in case + // this is called in an unavailable pphase. + ev_collect_->schedule(sparta::Clock::Cycle(1)); + } + + void disable(CollectableTreeNode * ctn) { + enabled_ctns_.erase(ctn); + } + + bool anyCollected() const { + return !enabled_ctns_.empty(); + } + + void performCollection() { + for(auto & ctn : enabled_ctns_) { + if(ctn->isCollected()) { + ctn->collect(); + } + } + if(!enabled_ctns_.empty()) { + ev_collect_->schedule(); + } + } + + private: + EventSet ev_set_; + std::unique_ptr ev_collect_; + std::set enabled_ctns_; + }; + + // A map of the clock pointer and the structures that + // represent collectables on that clock. + std::map, + sparta::NUM_SCHEDULING_PHASES>> clock_ctn_map_; + + // Registered collectables + std::set registered_collectables_; + public: PipelineCollector(const std::string& simdb_filename, - const size_t heartbeat = 10) - : db_mgr_(simdb_filename, true) - , filename_(simdb_filename) + const size_t heartbeat, + const sparta::TreeNode * root) + : db_mgr_(std::make_unique(simdb_filename, true)) { + sparta_assert(root->isFinalized() == true, + "Pipeline collection cannot be constructed until the sparta tree has been finalized."); + + sparta_assert(root->getClock() != nullptr, + "Cannot construct PipelineCollector because root clock is a nullptr"); + + sparta_assert(root->getClock()->getScheduler()->isFinalized() == false, + "Pipeline Collection cannot be instantiated after scheduler finalization"); + + scheduler_ = root->getClock()->getScheduler(); + // Note that we only care about the collection data and have // no need for any other tables, aside from the tables that the // DatabaseManager adds automatically to support this feature. simdb::Schema schema; - db_mgr_.createDatabaseFromSchema(schema); + db_mgr_->createDatabaseFromSchema(schema); std::function func_ptr = std::bind(&PipelineCollector::getCollectionTick_, this); - db_mgr_.getCollectionMgr()->useTimestampsFrom(func_ptr); - db_mgr_.getCollectionMgr()->setHeartbeat(heartbeat); + db_mgr_->getCollectionMgr()->useTimestampsFrom(func_ptr); + db_mgr_->getCollectionMgr()->setHeartbeat(heartbeat); + + // Initialize the clock/collectable map + std::function addClks; + addClks = [&addClks, this] (const sparta::Clock* clk) + { + if(clk != nullptr){ + auto & u_p = clock_ctn_map_[clk]; + for(uint32_t i = 0; i < NUM_SCHEDULING_PHASES; ++i) { + u_p[i].reset(new CollectablesByClock(clk, static_cast(i))); + } + for(const sparta::TreeNode* child : sparta::TreeNodePrivateAttorney::getAllChildren(clk)) { + const sparta::Clock* child_clk = dynamic_cast(child); + if(child_clk){ + addClks(child_clk); + } + } + } + }; + + addClks(root->getClock()); } ~PipelineCollector() { - db_mgr_.closeDatabase(); - } + db_mgr_->closeDatabase(); - //! \return the pipeout file path - const std::string & getFilePath() const { - return filename_; + // XXX This is a little goofy looking. Should we make sparta_abort that takes conditional + // be sparta_abort_unless()? + sparta_abort(collection_active_ != true, + "The PipelineCollector was not torn down properly. Before " + "tearing down the simulation tree, you must call " + "destroy() on the collector"); } - void enableCollection(TreeNode* starting_node) { - for (auto c : getCollectablesFrom_(starting_node)) { - if (!scheduler_) { - scheduler_ = c->getScheduler(); - } - auto &collectables = collectables_by_clk_[c->getClock()]; - if (!collectables) { - collectables.reset(new CollectablesByClock(c->getClock(), db_mgr_)); + /*! + * \brief Teardown the pipeline collector + * + * Tear down the PipelineCollector. Should be called before + * Tree teardown to close all open transactions. + */ + void destroy() + { + if(collection_active_) { + for(auto & ctn : registered_collectables_) { + if(ctn->isCollected()) { + ctn->closeRecord(true); + } } - collectables->addToAutoCollection(c); } + registered_collectables_.clear(); + collection_active_ = false; } - void finalizeCollector(RootTreeNode* rtn) { - createCollections_(rtn); + void reactivate(const std::string& simdb_filename) + { + (void)simdb_filename; + sparta_assert(false, "TODO cnyce: Not implemented"); + + if (db_mgr_) { + db_mgr_->closeDatabase(); + db_mgr_.reset(); + } } - void disableCollection() { - db_mgr_.onPipelineCollectorClosing(); - db_mgr_.getConnection()->getTaskQueue()->stopThread(); - db_mgr_.closeDatabase(); + /** + * \brief Turn on collection for everything below a TreeNode. + * Recursively transverse the tree and turn on child + * nodes for pipeline collection. + * + * \param starting_node TreeNode to start collection on. This + * TreeNode will try to start collection + * as well as any node below it. + * + * \note The Scheduler MUST be finalized before this method is + * called + */ + void startCollection(TreeNode* starting_node) + { + // Recursively collect the start node and children + std::function recursiveCollect; + recursiveCollect = [&recursiveCollect, this] (sparta::TreeNode* starting_node) + { + // First turn on this node if it's actually a CollectableTreeNode + CollectableTreeNode* c_node = dynamic_cast(starting_node); + if(c_node != nullptr) { + c_node->startCollecting(this, db_mgr_.get()); + registered_collectables_.insert(c_node); + } + + // Recursive step. Go through the children and turn them on as well. + for(sparta::TreeNode* node : sparta::TreeNodePrivateAttorney::getAllChildren(starting_node)) + { + recursiveCollect(node); + } + }; + + recursiveCollect(starting_node); } - private: - std::vector getCollectablesFrom_(TreeNode* starting_node) const + /** + * \brief Stop pipeline collection on only those + * CollectableTreeNodes given + * \param starting_node The node to shut collection down on + * + */ + void stopCollection(sparta::TreeNode* starting_node) { - std::vector collectables; - std::function recursiveFindCollectables; - recursiveFindCollectables = [&recursiveFindCollectables, &collectables](TreeNode* node) { - if (auto c = dynamic_cast(node)) { - collectables.push_back(c); + std::function recursiveStopCollect; + recursiveStopCollect = [&recursiveStopCollect, this] (sparta::TreeNode* starting_node) { + // First turn off this node if it's actually a CollectableTreeNode + CollectableTreeNode* c_node = dynamic_cast(starting_node); + if(c_node != nullptr) + { + c_node->stopCollecting(this, db_mgr_.get()); + registered_collectables_.erase(c_node); } - for (auto & child : node->getChildren()) { - recursiveFindCollectables(child); + + // Recursive step. Go through the children and turn them on as well. + for(sparta::TreeNode* node : sparta::TreeNodePrivateAttorney::getAllChildren(starting_node)) + { + recursiveStopCollect(node); } }; - recursiveFindCollectables(starting_node); - return collectables; - } + recursiveStopCollect(starting_node); - uint64_t getCollectionTick_() const { - return scheduler_->getCurrentTick() - 1; + collection_active_ = !registered_collectables_.empty(); } - void createCollections_(RootTreeNode* root) { - CollectionPoints collectables; - recurseAddCollectables_(root, collectables); - collectables.createCollections(db_mgr_.getCollectionMgr()); - db_mgr_.finalizeCollections(); + /** + * \brief Stop pipeline collection on only those + * CollectableTreeNodes that this PipelineCollector + * was started with + */ + void stopCollection() { + for(auto & col : registered_collectables_) { + col->stopCollecting(this, db_mgr_.get()); + } + registered_collectables_.clear(); } - void recurseAddCollectables_(sparta::TreeNode * node, - CollectionPoints & collectables) + /** + * \brief Add the CollectableTreeNode to auto collection + * \param ctn The CollectableTreeNode that is to be collected + * \param collection_phase The phase to collect the object in + * + * Enable collection on the given CollectableTreeNode. This + * is a runtime call. There are some rules here: + * + * #. The Scheduler must be finialized and simulation started + * #. The clock that the CollectableTreeNode belongs to must + * have been registered with the PipelineCollector at init time. + */ + void addToAutoCollection(CollectableTreeNode * ctn, + SchedulingPhase collection_phase = SchedulingPhase::Tick) { - if (auto c = dynamic_cast(node)) { - if (c->isCollected()) { - c->addCollectionPoint(collectables); - } - } - for (auto & child : node->getChildren()) { - recurseAddCollectables_(child, collectables); - } + auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); + sparta_assert(ccm_pair != clock_ctn_map_.end()); + ccm_pair->second[static_cast(collection_phase)]->enable(ctn); } - class CollectablesByClock + /** + * \brief Remove the given CollectableTreeNode from collection + * \param ctn The CollectableTreeNode that is to be removed from collection + * + * Disable collection on the given CollectableTreeNode. This + * is a runtime call. There are some rules here: + * + * #. The Scheduler must be finialized and simulation started + * #. The clock that the CollectableTreeNode belongs to must + * have been registered with the PipelineCollector at init time. + */ + void removeFromAutoCollection(CollectableTreeNode * ctn) { - public: - CollectablesByClock(const Clock* clk, simdb::DatabaseManager &db_mgr) - : db_mgr_(db_mgr) - , ev_set_(nullptr) - { - ev_collect_.reset(new sparta::UniqueEvent - (&ev_set_, sparta::notNull(clk)->getName() + "_auto_collection_event_collection", - CREATE_SPARTA_HANDLER(CollectablesByClock, performCollection_), 1)); - ev_collect_->setScheduleableClock(clk); - ev_collect_->setScheduler(clk->getScheduler()); - ev_collect_->setContinuing(false); + auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); + sparta_assert(ccm_pair != clock_ctn_map_.end()); + for(auto & u_p : ccm_pair->second) { + u_p->disable(ctn); } + } - void addToAutoCollection(CollectableTreeNode* ctn) { - ctn->enable(); - sparta_assert(clk_domain_ == ctn->getClock()->getName() || clk_domain_.empty()); - clk_domain_ = ctn->getClock()->getName(); - - // Schedule collect event in the next cycle in case - // this is called in an unavailable phase. - ev_collect_->schedule(sparta::Clock::Cycle(1)); - } + //! \return the pipeout file path + const std::string & getFilePath() const + { + return db_mgr_->getDatabaseFilePath(); + } - private: - void performCollection_() { - db_mgr_.getCollectionMgr()->collectDomain(clk_domain_); - ev_collect_->schedule(); - } + //! \return the scheduler for this collector + Scheduler* getScheduler() const { + return scheduler_; + } - simdb::DatabaseManager &db_mgr_; - sparta::EventSet ev_set_; - std::unique_ptr> ev_collect_; - std::string clk_domain_; - }; + private: + uint64_t getCollectionTick_() const + { + return scheduler_->getCurrentTick() - 1; + } //! The SimDB database - simdb::DatabaseManager db_mgr_; - - //! The SimDB filename e.g. "pipeline.db" - std::string filename_; + std::unique_ptr db_mgr_; - //! Scheduler - sparta::Scheduler * scheduler_ = nullptr; + //! Scheduler on which this collector operates + Scheduler * scheduler_ = nullptr; - //! Collectables by clock - std::unordered_map> collectables_by_clk_; + //! Is collection enabled on at least one node? + bool collection_active_ = false; }; }// namespace collection diff --git a/sparta/sparta/ports/DataPort.hpp b/sparta/sparta/ports/DataPort.hpp index ce96943557..5e2c364908 100644 --- a/sparta/sparta/ports/DataPort.hpp +++ b/sparta/sparta/ports/DataPort.hpp @@ -18,7 +18,7 @@ #include "sparta/events/PayloadEvent.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/Scheduleable.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" +#include "sparta/collection/Collectable.hpp" namespace sparta { @@ -288,7 +288,7 @@ namespace sparta class DataInPort final : public InPort, public DataContainer { // Pipeline collection type - typedef collection::ManualCollectable CollectorType; + typedef collection::Collectable CollectorType; public: @@ -445,7 +445,8 @@ namespace sparta * \param node The TreeNode to add the collector */ void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, Port::name_, &data_in_port_events_, "Data being received on this DataInPort"); + collector_.reset(new CollectorType(node, Port::name_, 0, + "Data being received on this DataInPort")); } private: @@ -529,7 +530,7 @@ namespace sparta //! The receiving clock const Clock * receiver_clock_ = nullptr; - //! Pipeline collection + /// Pipeline collection std::unique_ptr collector_; //! Data receiving point diff --git a/sparta/sparta/ports/SyncPort.hpp b/sparta/sparta/ports/SyncPort.hpp index c89bb5ced4..3a64f08237 100644 --- a/sparta/sparta/ports/SyncPort.hpp +++ b/sparta/sparta/ports/SyncPort.hpp @@ -72,10 +72,10 @@ #include "sparta/simulation/TreeNode.hpp" #include "sparta/utils/DataContainer.hpp" +#include "sparta/collection/DelayedCollectable.hpp" #include "sparta/ports/Port.hpp" #include "sparta/events/Precedence.hpp" #include "sparta/events/EventSet.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -130,8 +130,7 @@ namespace sparta SyncOutPort(TreeNode * portset, const std::string & name, const Clock * clk, bool presume_zero_delay = true) : OutPort(portset, name, presume_zero_delay), - clk_(clk), info_logger_(this, "pinfo", getLocation() + "_info"), - sync_port_events_(this) + clk_(clk), info_logger_(this, "pinfo", getLocation() + "_info") { sparta_assert(name.length() != 0, "You cannot have an unnamed port."); sparta_assert(clk_ != 0, "Clock ptr cannot be null in port: " << name); @@ -375,9 +374,9 @@ namespace sparta SyncOutPort & operator=(const SyncOutPort &) = delete; //! Enable pipeline collection - void enableCollection(TreeNode* node) override - { - collector_ = std::make_unique(node, Port::name_, &sync_port_events_, "Data being sent out on this SyncOutPort"); + void enableCollection(TreeNode* node) override { + collector_.reset(new CollectorType(node, Port::name_, 0, + "Data being sent out on this SyncOutPort")); } private: @@ -388,7 +387,7 @@ namespace sparta /// The in-port all data will be sent to SyncInPort * sync_in_port_ = nullptr; - /// Pipeline collection + /// Pipeline collection: TODO - figure out how this works w/ sync ports std::unique_ptr collector_; /// Last cycle any data was sent @@ -400,9 +399,6 @@ namespace sparta //! The bound SyncIn ports std::vector *> bound_in_ports_; - - //! Event Set for this port - sparta::EventSet sync_port_events_; }; @@ -414,8 +410,7 @@ namespace sparta template class SyncInPort final : public InPort, public DataContainer { - typedef collection::ManualCollectable CollectorType; - + typedef collection::Collectable CollectorType; public: //! Expected typedef for DataT typedef DataT DataType; @@ -598,7 +593,8 @@ namespace sparta //! Enable pipeline collection void enableCollection(TreeNode* node) override { - collector_ = std::make_unique(node, Port::name_, &sync_port_events_, "Data being recirculated on this SyncInPort"); + collector_.reset(new CollectorType(node, Port::name_, 0, + "Data being recirculated on this SyncInPort")); } //! Set the ready state for the port before simulation begins @@ -1064,7 +1060,7 @@ namespace sparta //! only call setReady() once per cycle. Scheduler::Tick set_ready_tick_ = 0; - //! Pipeline collection + //! Pipeline collection. TODO: See if this works for syncports std::unique_ptr collector_; /// loggers diff --git a/sparta/sparta/resources/Array.hpp b/sparta/sparta/resources/Array.hpp index 6d1219edce..15e0432351 100644 --- a/sparta/sparta/resources/Array.hpp +++ b/sparta/sparta/resources/Array.hpp @@ -14,8 +14,8 @@ #include "sparta/utils/SpartaAssert.hpp" #include "sparta/statistics/CycleHistogram.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/utils/IteratorTraits.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -847,10 +847,17 @@ namespace sparta */ void enableCollection(TreeNode* parent) { - collector_ = std::make_unique>(parent, name_, this, capacity()); + // Create the collector instance of the appropriate type. + sparta_assert(parent != nullptr); + collector_. + reset(new collection::IterableCollector + (parent, name_, this, capacity())); if constexpr (ArrayT == ArrayType::AGED) { - age_collector_ = std::make_unique>(parent, name_ + "_age_ordered", &aged_array_col_, capacity()); + age_collector_.reset(new collection::IterableCollector + (parent, name_ + "_age_ordered", + &aged_array_col_, capacity())); } } @@ -862,9 +869,6 @@ namespace sparta //! Typedef for size_type typedef uint32_t size_type; - //! Typedef for value_type - typedef Array value_type; - AgedArrayCollectorProxy(FullArrayType * array) : array_(array) { } typedef FullArrayType::iterator iterator; @@ -1006,8 +1010,10 @@ namespace sparta //////////////////////////////////////////////////////////// // Collectors - std::unique_ptr> collector_; - std::unique_ptr> age_collector_; + std::unique_ptr> collector_; + std::unique_ptr > age_collector_; }; template diff --git a/sparta/sparta/resources/Buffer.hpp b/sparta/sparta/resources/Buffer.hpp index c2ee543846..9f23785ea5 100644 --- a/sparta/sparta/resources/Buffer.hpp +++ b/sparta/sparta/resources/Buffer.hpp @@ -19,9 +19,9 @@ #include "sparta/statistics/CycleHistogram.hpp" #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/StatisticDef.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/Counter.hpp" #include "sparta/utils/IteratorTraits.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -731,7 +731,9 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_ = std::make_unique(parent, name_, this, capacity()); + collector_. + reset(new collection::IterableCollector >(parent, getName(), + this, capacity())); } /** @@ -1018,8 +1020,7 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - using IterableCollectorType = sparta::collection::IterableCollector>; - std::unique_ptr collector_; + std::unique_ptr > > collector_; //! Flag which tells various methods if infinite_mode is turned on or not. // The behaviour of these methods change accordingly. @@ -1096,5 +1097,8 @@ namespace sparta rval.utilization_ = nullptr; rval.collector_ = nullptr; validator_->validator_ = std::move(rval.validator_->validator_); + if(collector_) { + collector_->reattach(this); + } } } diff --git a/sparta/sparta/resources/CircularBuffer.hpp b/sparta/sparta/resources/CircularBuffer.hpp index 4bb359bcb1..4549fbad75 100644 --- a/sparta/sparta/resources/CircularBuffer.hpp +++ b/sparta/sparta/resources/CircularBuffer.hpp @@ -19,9 +19,9 @@ #include "sparta/statistics/CycleCounter.hpp" #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/StatisticDef.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/Counter.hpp" #include "sparta/utils/IteratorTraits.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { diff --git a/sparta/sparta/resources/Pipe.hpp b/sparta/sparta/resources/Pipe.hpp index fb926813cf..f041552a76 100644 --- a/sparta/sparta/resources/Pipe.hpp +++ b/sparta/sparta/resources/Pipe.hpp @@ -17,11 +17,9 @@ #include "sparta/ports/Port.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/MathUtils.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/utils/ValidValue.hpp" #include "sparta/utils/IteratorTraits.hpp" -#include "sparta/events/UniqueEvent.hpp" -#include "sparta/events/EventSet.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -515,12 +513,8 @@ class Pipe */ template void enableCollection(TreeNode * parent) { - if constexpr (phase == SchedulingPhase::Collection) { - collector_ = std::make_unique(parent, name_, this, capacity()); - } else { - std::cout << "ERROR: sparta::Pipe '" << (parent->getLocation() + "." + name_ + "' ") - << "only supports collection in SchedulingPhase::Collection" << std::endl; - } + collector_.reset (new collection::IterableCollector, phase, true> + (parent, name_, this, capacity())); } /** @@ -599,8 +593,8 @@ class Pipe ////////////////////////////////////////////////////////////////////// // Collectors - using CollectorType = collection::IterableCollector, true>; - std::unique_ptr collector_; + std::unique_ptr collector_; + }; } diff --git a/sparta/sparta/resources/Queue.hpp b/sparta/sparta/resources/Queue.hpp index eecd4db66b..a88ef50bc2 100644 --- a/sparta/sparta/resources/Queue.hpp +++ b/sparta/sparta/resources/Queue.hpp @@ -17,9 +17,9 @@ #include "sparta/ports/Port.hpp" #include "sparta/statistics/CycleHistogram.hpp" #include "sparta/statistics/StatisticSet.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/statistics/InstrumentationNode.hpp" #include "sparta/utils/IteratorTraits.hpp" -#include "sparta/collection/CollectableTreeNode.hpp" namespace sparta { @@ -447,7 +447,8 @@ namespace sparta InstrumentationNode::visibility_t stat_vis_avg = InstrumentationNode::AUTO_VISIBILITY) : num_entries_(num_entries), vector_size_(nextPowerOfTwo_(num_entries*2)), - name_(name) + name_(name), + collector_(nullptr) { if((num_entries > 0) && statset) @@ -600,7 +601,8 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_ = std::make_unique(parent, name_, this, capacity()); + collector_.reset(new collection::IterableCollector > + (parent, name_, this, capacity())); } /** @@ -824,8 +826,7 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - using IterableCollectorType = sparta::collection::IterableCollector>; - std::unique_ptr collector_; + std::unique_ptr > > collector_; // Notice that our list for storing data is a dynamic array. // This is used instead of a stl vector to promote debug diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index 08980e2732..c4d2ea6c46 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -148,13 +148,13 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, { static std::stringstream heartbeat_doc; heartbeat_doc << \ - "The interval in ticks at which index pointers will be written to file during pipeline " - "collection. The heartbeat also represents the longest life duration of lingering " - "transactions. Transactions with a life span longer than the heartbeat will be finalized " - "and then restarted with a new start time. Must be a multiple of 100 for efficient reading " - "by pipeViewer. Large values will reduce responsiveness of pipeViewer when jumping to different " - "areas of the file and loading.\nDefault = " - << DefaultHeartbeat << " ticks.\n"; + "The maximum number of 'carry-overs' allowed during SimDB collection when encountering " + "unchanged collectable values. For example, if a collectable has the same value 5 for " + "100 cycles in a row, how often should we force a write to the database? This value is " + "used to determine how 'wide' of a SQLite query we have to cast to ensure we know the actual " + "collectable value. Lower values result in a larger database but with a more responsive UI. " + "Higher values gives higher compression, but some UI operations might take extra time. " + "The heartbeat value must be between 1 and 25, and defaults to 10."; sparta_opts_.add_options() ("help,h", @@ -375,7 +375,7 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, "Example: \"--pipeViewer-collection-at layouts/exe40.alf\" " "This option can be specified none or many times.") // Brief ("heartbeat", - named_value("HEARTBEAT", &pipeline_heartbeat_)->default_value(pipeline_heartbeat_), + named_value("HEARTBEAT", &pipeline_heartbeat_)->default_value(pipeline_heartbeat_), heartbeat_doc.str().c_str()) ; @@ -1773,10 +1773,6 @@ bool CommandLineSimulator::parse(int argc, if (!fs_path.has_extension() || fs_path.extension() != ".db") { simdb_filename += ".db"; } - - if (pipeline_heartbeat_ == DefaultHeartbeat) { - pipeline_heartbeat_ = "10"; - } } //pevents @@ -1879,13 +1875,11 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) throw SpartaException("Cannot setup the simulation more than once"); } - // Convert heartbeat command line string to int - try{ - size_t end_pos; - utils::smartLexicalCast(pipeline_heartbeat_, end_pos); - }catch (SpartaException const&){ - throw SpartaException("HEARTBEAT for pipeline collection must be an integer value > 0, not \"") - << pipeline_heartbeat_ << "\""; + if(pipeline_heartbeat_ == 0) { + throw SpartaException("HEARTBEAT for pipeline collection must be an integer value > 0"); + } + if(pipeline_heartbeat_ > 25) { + throw SpartaException("HEARTBEAT for pipeline collection must be an integer value <= 25"); } // Pevent @@ -2033,11 +2027,12 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) if(sim_config_.pipeline_collection_file_prefix != NoPipelineCollectionStr) { - pipeline_collection_triggerable_.reset(new PipelineTrigger( - sim_config_.pipeline_collection_file_prefix, - sim->getRoot(), - std::atoi(pipeline_heartbeat_.c_str()), - pipeline_enabled_node_names_)); + const bool multiple_triggers = sim_config_.trigger_on_type == SimulationConfiguration::TriggerSource::TRIGGER_ON_ROI; + pipeline_collection_triggerable_.reset(new PipelineTrigger(sim_config_.pipeline_collection_file_prefix, + pipeline_enabled_node_names_, + pipeline_heartbeat_, + multiple_triggers, + sim->getRoot())); } // Finalize the pevent controller now that the tree is built. diff --git a/sparta/test/Array/Array_test.cpp b/sparta/test/Array/Array_test.cpp index 31e68d57b8..df0744407e 100644 --- a/sparta/test/Array/Array_test.cpp +++ b/sparta/test/Array/Array_test.cpp @@ -17,12 +17,15 @@ #include "sparta/resources/Array.hpp" #include "sparta/resources/FrontArray.hpp" #include "sparta/simulation/ClockManager.hpp" + #include "sparta/collection/PipelineCollector.hpp" #include TEST_INIT +#define PIPEOUT_GEN + struct dummy_struct { uint16_t int16_field; @@ -134,9 +137,15 @@ int main() root_node.enterConfiguring(); root_node.enterFinalized(); - sparta::collection::PipelineCollector pc("test_collection_", &root_node); +#ifdef PIPEOUT_GEN + sparta::collection::PipelineCollector pc("test_collection_", 1000, &root); +#endif + sched.finalize(); - pc.startCollecting(); + +#ifdef PIPEOUT_GEN + pc.startCollection(&root_node); +#endif // Test perfect forwarding arrays move { @@ -458,13 +467,17 @@ int main() EXPECT_EQUAL(aged_array_test.getAge(i), age_vec_0[idxx++]); } +#ifdef PIPEOUT_GEN sched.run(1); +#endif aged_collected_array.erase(0); //+9 records. aged_collected_array.erase(1); //+8 records. aged_collected_array.write(0, 0); +#ifdef PIPEOUT_GEN sched.run(1); +#endif EXPECT_EQUAL(aged_array.abegin().getIndex(), aged_array.getOldestIndex().getIndex()); bit = aged_array.abegin(); @@ -503,13 +516,17 @@ int main() } EXPECT_EQUAL(cnt, 7); +#ifdef PIPEOUT_GEN sched.run(1); +#endif aged_array.write(4, 4); aged_array.write(2, 2); aged_array.write(1, 1); +#ifdef PIPEOUT_GEN sched.run(1); +#endif AgedArray::iterator it = aged_array.getCircularIterator(); while(it != aged_array.getCircularIterator(aged_array.capacity() - 1)) @@ -522,7 +539,9 @@ int main() EXPECT_EQUAL(*dat, 9); aged_array.erase(it); +#ifdef PIPEOUT_GEN sched.run(1); +#endif //set up the pipeline collection. @@ -533,7 +552,9 @@ int main() ns_array.write(1, 1); ns_array.write(2, 2); +#ifdef PIPEOUT_GEN sched.run(1); +#endif //ns_array.getOldestIndex(0); THROWS a static assert message since //ns_array is not aged. @@ -547,13 +568,18 @@ int main() EXPECT_EQUAL(ns_array.numValid(), 1); EXPECT_EQUAL(ns_array.capacity(), 10); +#ifdef PIPEOUT_GEN sched.run(1); +#endif + ns_array.write(5, 5); ns_array.write(3, 3); ns_array.write(0, 0); +#ifdef PIPEOUT_GEN sched.run(1); +#endif MyArray::iterator iter = ns_array.begin(); uint32_t i = 0; @@ -569,8 +595,9 @@ int main() ++i; } +#ifdef PIPEOUT_GEN sched.run(1); - +#endif iter = ns_array.begin(); std::advance(iter, 3); @@ -676,8 +703,10 @@ int main() // std::cout << "valid: " << iter.isValid() << std::endl; // } +#ifdef PIPEOUT_GEN sched.run(10); - pc.stopCollecting(); + pc.destroy(); +#endif //it's now safe to tear down our dummy tree root_node.enterTeardown(); diff --git a/sparta/test/Buffer/Buffer_test.cpp b/sparta/test/Buffer/Buffer_test.cpp index 556ccb806f..99113ba1d5 100644 --- a/sparta/test/Buffer/Buffer_test.cpp +++ b/sparta/test/Buffer/Buffer_test.cpp @@ -16,10 +16,11 @@ #include "sparta/statistics/StatisticInstance.hpp" #include "sparta/statistics/CycleCounter.hpp" -#include "sparta/collection/PipelineCollector.hpp" TEST_INIT +#define PIPEOUT_GEN + #define QUICK_PRINT(x) \ std::cout << x << std::endl @@ -95,7 +96,9 @@ void generalTest() &buf10_stats); rtn.setClock(root_clk.get()); +#ifdef PIPEOUT_GEN buf10.enableCollection(&rtn); +#endif rtn.enterConfiguring(); rtn.enterFinalized(); @@ -103,9 +106,15 @@ void generalTest() // Get info messages from the scheduler node and send them to this file sparta::log::Tap t2(root_clk.get()->getScheduler(), "debug", "scheduler.log.debug"); - sparta::collection::PipelineCollector pc("testBuffer", &rtn); +#ifdef PIPEOUT_GEN + sparta::collection::PipelineCollector pc("testBuffer", 1000000, &rtn); +#endif + sched.finalize(); - pc.startCollecting(); + +#ifdef PIPEOUT_GEN + pc.startCollection(&rtn); +#endif //////////////////////////////////////////////////////////// sched.run(1); @@ -571,7 +580,9 @@ void generalTest() sched.run(5); rtn.enterTeardown(); - pc.stopCollecting(); +#ifdef PIPEOUT_GEN + pc.destroy(); +#endif } struct B { diff --git a/sparta/test/CircularBuffer/CircularBuffer_test.cpp b/sparta/test/CircularBuffer/CircularBuffer_test.cpp index 8a2cbcaef0..debcb0781f 100644 --- a/sparta/test/CircularBuffer/CircularBuffer_test.cpp +++ b/sparta/test/CircularBuffer/CircularBuffer_test.cpp @@ -11,10 +11,10 @@ #include "sparta/kernel/Scheduler.hpp" #include "sparta/utils/SpartaTester.hpp" #include "sparta/report/Report.hpp" -#include "sparta/collection/PipelineCollector.hpp" TEST_INIT +//#define PIPEOUT_GEN struct dummy_struct { uint16_t int16_field; @@ -463,7 +463,8 @@ void testCollection() rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testCircBuffer", &rtn); + sparta::collection::PipelineCollector pc("testCircBuffer", 1000000, &rtn); + sched.finalize(); for(uint32_t i = 0; i < BUF_SIZE/2; ++i) { diff --git a/sparta/test/Pipe/Pipe_test.cpp b/sparta/test/Pipe/Pipe_test.cpp index 34110944a0..1b8e998388 100644 --- a/sparta/test/Pipe/Pipe_test.cpp +++ b/sparta/test/Pipe/Pipe_test.cpp @@ -10,9 +10,9 @@ #include "sparta/sparta.hpp" #include "sparta/events/EventSet.hpp" #include "sparta/events/PayloadEvent.hpp" -#include "sparta/collection/PipelineCollector.hpp" TEST_INIT +#define PIPEOUT_GEN /* * This test creates a producer and a consumer for two staged pipes. @@ -68,9 +68,10 @@ int main () // sparta::log::Tap scheduler_debug(sparta::TreeNode::getVirtualGlobalNode(), // sparta::log::categories::DEBUG, std::cout); +#ifdef PIPEOUT_GEN pipe1.enableCollection(&rtn); pipe2.enableCollection(&rtn); - +#endif sparta::PayloadEvent ev(&es, "dummy_ev", CREATE_SPARTA_HANDLER_WITH_DATA_WITH_OBJ(sparta::Pipe, &pipe1, push_front, uint32_t)); @@ -78,13 +79,16 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testPipe", &rtn); +#ifdef PIPEOUT_GEN + sparta::collection::PipelineCollector pc("testPipe", 1000000, &rtn); +#endif sched.finalize(); - //TODO cnyce: when pipe2.enableCollection() is fixed, put back these two checks. - //EXPECT_THROW(pipe2.resize(5)); - //EXPECT_EQUAL(pipe2.capacity(), 10); // Make sure it really didn't get resized - pc.startCollecting(); +#ifdef PIPEOUT_GEN + EXPECT_THROW(pipe2.resize(5)); + EXPECT_EQUAL(pipe2.capacity(), 10); // Make sure it really didn't get resized + pc.startCollection(&rtn); +#endif // Check initials EXPECT_EQUAL(pipe1.capacity(), 10); @@ -315,7 +319,9 @@ int main () EXPECT_EQUAL(pipe2.size(), 0); rtn.enterTeardown(); - pc.stopCollecting(); +#ifdef PIPEOUT_GEN + pc.destroy(); +#endif // Returns error if one REPORT_ERROR; diff --git a/sparta/test/Pipeline/Pipeline_test.cpp b/sparta/test/Pipeline/Pipeline_test.cpp index e914ed1ab6..d59be69ee2 100644 --- a/sparta/test/Pipeline/Pipeline_test.cpp +++ b/sparta/test/Pipeline/Pipeline_test.cpp @@ -12,10 +12,9 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/simulation/ClockManager.hpp" #include "sparta/utils/SpartaTester.hpp" -#include "sparta/events/PayloadEvent.hpp" -#include "sparta/collection/PipelineCollector.hpp" TEST_INIT +#define PIPEOUT_GEN #define TEST_MANUAL_UPDATE @@ -311,6 +310,7 @@ int main () sparta::PayloadEvent ev_flush_one (&es, "ev_flush_one", CREATE_SPARTA_HANDLER_WITH_DATA_WITH_OBJ(DummyClass2, &dummyObj2, flushOne, uint32_t)); +#ifdef PIPEOUT_GEN EXPECT_FALSE(examplePipeline1.isCollected()); examplePipeline1.enableCollection(&rtn); EXPECT_FALSE(examplePipeline1.isCollected()); @@ -323,6 +323,7 @@ int main () examplePipeline8.enableCollection(&rtn); examplePipeline9.enableCollection(&rtn); stwr_pipe.enableCollection(&rtn); +#endif //////////////////////////////////////////////////////////////////////////////// // Pipeline stage handler registration @@ -540,7 +541,10 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("examplePipeline1", &rtn); +#ifdef PIPEOUT_GEN + sparta::collection::PipelineCollector pc("examplePipeline1", 1000000, &rtn); +#endif + //////////////////////////////////////////////////////////////////////////////// // Pipeline stage handling event precedence setup @@ -664,9 +668,12 @@ int main () sched.finalize(); +#ifdef PIPEOUT_GEN EXPECT_FALSE(examplePipeline1.isCollected()); - pc.startCollecting(); + pc.startCollection(&rtn); EXPECT_TRUE(examplePipeline1.isCollected()); +#endif + //////////////////////////////////////////////////////////////////////////////// // Pipeline Forward Progression Test @@ -1620,7 +1627,10 @@ int main () std::cout << "[FINISH] Pipeline Data Event Handling Test\n"; rtn.enterTeardown(); - pc.stopCollecting(); + +#ifdef PIPEOUT_GEN + pc.destroy(); +#endif // Returns error if one REPORT_ERROR; diff --git a/sparta/test/Port/Port_test.cpp b/sparta/test/Port/Port_test.cpp index f50a22e140..06bb111f15 100644 --- a/sparta/test/Port/Port_test.cpp +++ b/sparta/test/Port/Port_test.cpp @@ -123,6 +123,8 @@ int main () rtn.enterConfiguring(); rtn.enterFinalized(); + sparta::collection::PipelineCollector pc("testPipe", 1000000, &rtn); + sched.finalize(); #ifdef TEST_PIPEOUT_COLLECTION diff --git a/sparta/test/Queue/Queue_test.cpp b/sparta/test/Queue/Queue_test.cpp index 6981b0c22a..8631d2adaf 100644 --- a/sparta/test/Queue/Queue_test.cpp +++ b/sparta/test/Queue/Queue_test.cpp @@ -17,10 +17,10 @@ #include "sparta/statistics/CycleCounter.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" -#include "sparta/collection/PipelineCollector.hpp" - TEST_INIT +#define PIPEOUT_GEN + void testIteratorValidity(); void testIteratorValidity2(); void testPushClearAccess(); @@ -63,15 +63,23 @@ int main() sparta::Queue dummy_struct_queue_alloc("dummy_struct_queue_alloc", 5, root_clk.get(), &queue10_stats); rtn.setClock(root_clk.get()); + +#ifdef PIPEOUT_GEN queue10_untimed.enableCollection(&rtn); +#endif rtn.enterConfiguring(); rtn.enterFinalized(); - sparta::collection::PipelineCollector pc("testPipe", &rtn); +#ifdef PIPEOUT_GEN + sparta::collection::PipelineCollector pc("testPipe", 1000000, &rtn); +#endif + sched.finalize(); - pc.startCollecting(); +#ifdef PIPEOUT_GEN + pc.startCollection(&rtn); +#endif //////////////////////////////////////////////////////////// sched.run(1); @@ -381,7 +389,9 @@ int main() testPopBack(); rtn.enterTeardown(); - pc.stopCollecting(); +#ifdef PIPEOUT_GEN + pc.destroy(); +#endif REPORT_ERROR; return ERROR_CODE; diff --git a/sparta/test/SyncPort/SyncPort_test.cpp b/sparta/test/SyncPort/SyncPort_test.cpp index 4f0075cfe8..dfe9d396eb 100644 --- a/sparta/test/SyncPort/SyncPort_test.cpp +++ b/sparta/test/SyncPort/SyncPort_test.cpp @@ -13,7 +13,6 @@ #include "sparta/simulation/ParameterSet.hpp" #include "sparta/events/Event.hpp" #include "sparta/log/Tap.hpp" -#include "sparta/collection/PipelineCollector.hpp" #include #include @@ -21,6 +20,9 @@ TEST_INIT +// Pipeout generation does not work with this test +#define PIPEOUT_GEN + // Be verbose -- very verbose // #define MAKE_NOISE @@ -162,6 +164,7 @@ class TestSystem std::unique_ptr master_tn; std::unique_ptr slave_tn; std::unique_ptr pc; + }; ////////////////////////////////////////////////////////////////////// @@ -176,8 +179,10 @@ Unit::Unit(sparta::TreeNode* node, const Unit::ParameterSet*) : ev_set(node), ev_do_work(&ev_set, "unit_do_work_event", CREATE_SPARTA_HANDLER(Unit, doWork)) { +#ifdef PIPEOUT_GEN out_cmd.enableCollection(node); in_cmd.enableCollection(node); +#endif in_cmd.registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Unit, cmd_callback, DataType)); in_data.registerConsumerHandler(CREATE_SPARTA_HANDLER_WITH_DATA(Unit, data_callback, char)); @@ -350,6 +355,8 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) master_clk = cm.makeClock("master_clk", root_clk, master_frequency_mhz); slave_clk = cm.makeClock("slave_clk", root_clk, slave_frequency_mhz); + rtn.setClock(root_clk.get()); + master_tn.reset(new sparta::ResourceTreeNode(&rtn, "master", "master", &rfact)); master_tn->setClock(master_clk.get()); @@ -389,14 +396,19 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) slave_unit->in_cmd.precedes(slave_unit->in_data); master_unit->in_cmd.precedes(master_unit->in_data); - pc.reset(new sparta::collection::PipelineCollector("testPipe", &rtn)); +#ifdef PIPEOUT_GEN + pc.reset(new sparta::collection::PipelineCollector("testPipe", 1000000, &rtn)); +#endif + sched.finalize(); // Align the scheduler to the rising edge of both clocks while(!(master_clk->isPosedge() && slave_clk->isPosedge())) { sched.run(1, true, false); // exacting_run = true, measure time = false } - pc->startCollecting(); +#ifdef PIPEOUT_GEN + pc->startCollection(&rtn); +#endif //sparta::log::Tap scheduler_tap(sparta::Scheduler::getScheduler(), "debug", "sched_cmds.out"); master_unit->schedule_commands(slave_frequency_mhz); @@ -408,7 +420,9 @@ TestSystem::TestSystem(double master_frequency_mhz, double slave_frequency_mhz) TestSystem::~TestSystem() { - pc->stopCollecting(); +#ifdef PIPEOUT_GEN + pc->destroy(); +#endif rtn.enterTeardown(); sched.restartAt(0); } From 735b1d2ac3e0d03e66a3a04460b4012d48ada9c9 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 11:35:17 -0600 Subject: [PATCH 09/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 31 ------------------- .../CoreModel/src/LoadStoreInstInfo.hpp | 24 -------------- .../example/CoreModel/src/MemAccessInfo.hpp | 26 ---------------- .../DynamicModelPipeline/src/ExampleInst.hpp | 30 ------------------ 4 files changed, 111 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 2e0c031238..eb582b6a89 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -6,7 +6,6 @@ #include "sparta/decode/DecoderBase.hpp" #include "sparta/memory/AddressTypes.hpp" #include "sparta/resources/SharedData.hpp" -#include "sparta/pairs/SpartaKeyPairs.hpp" #include "sparta/simulation/State.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" @@ -23,15 +22,9 @@ namespace core_example * \brief Example instruction that flows through the example/CoreModel */ - // Forward declaration of the Pair Definition class is must as we are friending it. - class ExampleInstPairDef; - class ExampleInst { public: - // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself - using SpartaPairDefinitionType = ExampleInstPairDef; - enum class Status : std::uint16_t{ FETCHED = 0, __FIRST = FETCHED, @@ -226,30 +219,6 @@ namespace core_example } return os; } - - /*! - * \class ExampleInstPairDef - * \brief Pair Definition class of the Example instruction that flows through the example/CoreModel - */ - // This is the definition of the PairDefinition class of ExampleInst. - // This PairDefinition class could be named anything but it needs to - // inherit publicly from sparta::PairDefinition templatized on the actual class ExampleInst. - class ExampleInstPairDef : public sparta::PairDefinition{ - public: - - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - ExampleInstPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(ExampleInst); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &ExampleInst::getUniqueID), - SPARTA_ADDPAIR("uid", &ExampleInst::getUniqueID), - SPARTA_ADDPAIR("mnemonic", &ExampleInst::getMnemonic), - SPARTA_ADDPAIR("complete", &ExampleInst::getCompletedStatus), - SPARTA_ADDPAIR("unit", &ExampleInst::getUnit), - SPARTA_ADDPAIR("latency", &ExampleInst::getExecuteTime), - SPARTA_ADDPAIR("raddr", &ExampleInst::getRAdr, std::ios::hex), - SPARTA_ADDPAIR("vaddr", &ExampleInst::getVAdr, std::ios::hex)) - }; } namespace simdb diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index fc9b4b84d8..a5fdb95bd4 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -7,14 +7,10 @@ namespace core_example class LoadStoreInstInfo; using LoadStoreInstInfoPtr = sparta::SpartaSharedPointer; - // Forward declaration of the Pair Definition class is must as we are friending it. - class LoadStoreInstInfoPairDef; // Keep record of instruction issue information class LoadStoreInstInfo { public: - // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself - using SpartaPairDefinitionType = LoadStoreInstInfoPairDef; enum class IssuePriority : std::uint16_t { HIGHEST = 0, @@ -99,26 +95,6 @@ namespace core_example sparta::State state_; }; // class LoadStoreInstInfo - - /*! - * \class LoadStoreInstInfoPairDef - * \brief Pair Definition class of the load store instruction that flows through the example/CoreModel - */ - // This is the definition of the PairDefinition class of LoadStoreInstInfo. - // This PairDefinition class could be named anything but it needs to inherit - // publicly from sparta::PairDefinition templatized on the actual class LoadStoreInstInfo. - class LoadStoreInstInfoPairDef : public sparta::PairDefinition{ - public: - - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - LoadStoreInstInfoPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(LoadStoreInstInfo); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &LoadStoreInstInfo::getInstUniqueID), - SPARTA_ADDPAIR("rank", &LoadStoreInstInfo::getPriority), - SPARTA_ADDPAIR("state", &LoadStoreInstInfo::getState), - SPARTA_FLATTEN( &LoadStoreInstInfo::getMemoryAccessInfoPtr)) - }; } // namespace core_example namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index bf0aa0edfc..4af84c2ddc 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -8,15 +8,10 @@ namespace core_example class MemoryAccessInfo; using MemoryAccessInfoPtr = sparta::SpartaSharedPointer; - // Forward declaration of the Pair Definition class is must as we are friending it. - class MemoryAccessInfoPairDef; // Keep record of memory access information in LSU class MemoryAccessInfo { public: - // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself - using SpartaPairDefinitionType = MemoryAccessInfoPairDef; - enum class MMUState : std::uint32_t { NO_ACCESS = 0, __FIRST = NO_ACCESS, @@ -96,27 +91,6 @@ namespace core_example }; // class MemoryAccessInfo - /*! - * \class MemoryAccessInfoPairDef - * \brief Pair Definition class of the Memory Access Information that flows through the example/CoreModel - */ - - // This is the definition of the PairDefinition class of MemoryAccessInfo. - // This PairDefinition class could be named anything but it needs to inherit - // publicly from sparta::PairDefinition templatized on the actual class MemoryAcccessInfo. - class MemoryAccessInfoPairDef : public sparta::PairDefinition{ - public: - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - MemoryAccessInfoPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(MemoryAccessInfo); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &MemoryAccessInfo::getInstUniqueID), - SPARTA_ADDPAIR("valid", &MemoryAccessInfo::getPhyAddrIsReady), - SPARTA_ADDPAIR("mmu", &MemoryAccessInfo::getMMUState), - SPARTA_ADDPAIR("cache", &MemoryAccessInfo::getCacheState), - SPARTA_FLATTEN( &MemoryAccessInfo::getInstPtr)) - }; - inline std::ostream & operator<<(std::ostream & os, const core_example::MemoryAccessInfo::MMUState & mmu_access_state){ switch(mmu_access_state){ diff --git a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp index b20b3d321b..3248b2a1ed 100644 --- a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp +++ b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp @@ -23,15 +23,8 @@ namespace core_example * \brief Example instruction that flows through the example/CoreModel */ - // Forward declaration of the Pair Definition class is must as we are friending it. - class ExampleInstPairDef; - class ExampleInst { public: - - // The modeler needs to alias a type called "type" to the Pair Definition class of itself - using type = ExampleInstPairDef; - enum class Status : std::uint16_t{ FETCHED = 0, __FIRST = FETCHED, @@ -216,27 +209,4 @@ namespace core_example } return os; } - - /*! - * \class ExampleInstPairDef - * \brief Pair Definition class of the Example instruction that flows through the example/CoreModel - */ - // This is the definition of the PairDefinition class of ExampleInst. - // This PairDefinition class could be named anything but it needs to - // inherit publicly from sparta::PairDefinition templatized on the actual class ExampleInst. - class ExampleInstPairDef : public sparta::PairDefinition{ - public: - - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class - ExampleInstPairDef() : PairDefinition(){ - SPARTA_INVOKE_PAIRS(ExampleInst); - } - SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("mnemonic", &ExampleInst::getMnemonicAsString), - SPARTA_ADDPAIR("complete", &ExampleInst::getCompletedStatus), - SPARTA_ADDPAIR("unit", &ExampleInst::getUnit), - SPARTA_ADDPAIR("latency", &ExampleInst::getExecuteTime), - SPARTA_ADDPAIR("uid", &ExampleInst::getUniqueID), - SPARTA_ADDPAIR("raddr", &ExampleInst::getRAdr, std::ios::hex), - SPARTA_ADDPAIR("vaddr", &ExampleInst::getVAdr, std::ios::hex)); - }; } From 2c2ccf7bbab59ae734a3d368330173021c574e26 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 12:01:38 -0600 Subject: [PATCH 10/49] SimDB / v3 integration --- sparta/example/CoreModel/src/LSU.hpp | 2 +- sparta/example/CoreModel/src/MSS.hpp | 1 + sparta/sparta/app/AppTriggers.hpp | 1 - sparta/sparta/collection/Collectable.hpp | 9 +- .../sparta/collection/CollectableTreeNode.hpp | 72 +++-- sparta/sparta/collection/CollectionPoints.hpp | 280 ------------------ .../sparta/collection/PipelineCollector.hpp | 1 - 7 files changed, 44 insertions(+), 322 deletions(-) delete mode 100644 sparta/sparta/collection/CollectionPoints.hpp diff --git a/sparta/example/CoreModel/src/LSU.hpp b/sparta/example/CoreModel/src/LSU.hpp index 9a3574e9e2..9fdf66589c 100644 --- a/sparta/example/CoreModel/src/LSU.hpp +++ b/sparta/example/CoreModel/src/LSU.hpp @@ -9,6 +9,7 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" +#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "sparta/resources/Pipeline.hpp" #include "sparta/resources/Buffer.hpp" @@ -165,7 +166,6 @@ namespace core_example // Keep track of the instruction that causes current outstanding cache miss ExampleInstPtr cache_pending_inst_ptr_ = nullptr; - // Collection sparta::collection::Collectable cache_busy_collectable_{ getContainer(), "dcache_busy", &cache_busy_}; diff --git a/sparta/example/CoreModel/src/MSS.hpp b/sparta/example/CoreModel/src/MSS.hpp index f84242084f..9e8f6b5fc7 100644 --- a/sparta/example/CoreModel/src/MSS.hpp +++ b/sparta/example/CoreModel/src/MSS.hpp @@ -9,6 +9,7 @@ #include "sparta/simulation/Unit.hpp" #include "sparta/simulation/ParameterSet.hpp" #include "sparta/simulation/TreeNode.hpp" +#include "sparta/collection/Collectable.hpp" #include "sparta/events/StartupEvent.hpp" #include "sparta/ports/SyncPort.hpp" #include "sparta/resources/Pipe.hpp" diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index 3b0d15b76d..dd6d04686e 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -109,7 +109,6 @@ class PipelineTrigger : public trigger::Triggerable std::vector results; root_->getSearchScope()->findChildren(node_name, results); for(auto & tn : results) { - (void)tn; pipeline_collector_->stopCollection(tn); } } diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index 4927977817..f44a654b7d 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -240,11 +240,12 @@ namespace sparta{ //! \brief Get a reference to the internal event set //! \return Reference to event set -- used by DelayedCollectable - EventSet & getEventSet_() { + inline EventSet & getEventSet_() { return event_set_; } private: + //! Return true if the record was written; false otherwise bool writeRecord_(bool simulation_ending = false) { //TODO cnyce @@ -252,6 +253,12 @@ namespace sparta{ return true; } + //! Start a new record + void startNewRecord_() + { + //TODO cnyce + } + void setCollecting_(bool collect, PipelineCollector * collector, simdb::DatabaseManager*) override final { // If the collected object is null, this Collectable diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 0d8107f844..349de5b47b 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -11,10 +11,6 @@ #pragma once #include "sparta/simulation/TreeNode.hpp" -#include "sparta/collection/CollectionPoints.hpp" -#include "sparta/events/PayloadEvent.hpp" -#include "sparta/events/PhasedUniqueEvent.hpp" -#include "sparta/events/EventSet.hpp" namespace simdb { class DatabaseManager; @@ -30,32 +26,32 @@ namespace collection * has virtual calls to start collection on this node, * and stop collection on this node. */ - class CollectableTreeNode : public TreeNode + class CollectableTreeNode : public sparta::TreeNode { public: /** - * \brief Construct. - * \param parent a pointer to the parent treenode - * \param name the name of this treenode - * \param group the name of the group for this treenode - * \param index the index within the group - * \param desc A description for this treenode. - */ + * \brief Construct. + * \param parent a pointer to the parent treenode + * \param name the name of this treenode + * \param group the name of the group for this treenode + * \param index the index within the group + * \param desc A description for this treenode. + */ CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, const std::string& group, uint32_t index, const std::string& desc = "CollectableTreeNode ") : sparta::TreeNode(parent, name, group, index, desc) { markHidden(); // Mark self as hidden from the default - // printouts (to reduce clutter) + // printouts (to reduce clutter) } /** - * \brief Construct. - * \param parent a pointer to the parent treenode - * \param name the name of this treenode - * \param desc Description of this CollectableTreeNode - */ + * \brief Construct. + * \param parent a pointer to the parent treenode + * \param name the name of this treenode + * \param desc Description of this CollectableTreeNode + */ CollectableTreeNode(sparta::TreeNode* parent, const std::string& name, const std::string& desc= "CollectableTreeNode ") : CollectableTreeNode(parent, name, sparta::TreeNode::GROUP_NAME_NONE, @@ -67,32 +63,32 @@ namespace collection {} /** - * \brief Method that tells this treenode that is now running - * collection. - * \param collector The collector that is performing the collection - * - * This method should instantiate any necessary values for the - * treenode necessary to collection as well as flip the - * is_collecting boolean. - */ + * \brief Method that tells this treenode that is now running + * collection. + * \param collector The collector that is performing the collection + * + * This method should instantiate any necessary values for the + * treenode necessary to collection as well as flip the + * is_collecting boolean. + */ void startCollecting(PipelineCollector* collector, simdb::DatabaseManager* db_mgr) { is_collected_ = true; setCollecting_(true, collector, db_mgr); } /** - * \brief Method that tells this treenode that is now not - * running collection. - */ + * \brief Method that tells this treenode that is now not + * running collection. + */ void stopCollecting(PipelineCollector* collector, simdb::DatabaseManager* db_mgr) { setCollecting_(false, collector, db_mgr); is_collected_ = false; } /** - * \brief Determine whether or not this node has collection - * turned on or off. - */ + * \brief Determine whether or not this node has collection + * turned on or off. + */ bool isCollected() const { return is_collected_; } //! Pure virtual method used by deriving classes to be @@ -112,17 +108,17 @@ namespace collection protected: /** - * \brief Indicate to sub-classes that collection has flipped - * \param collect true if collection is enabled; false otherwise - */ + * \brief Indicate to sub-classes that collection has flipped + * \param collect true if collection is enabled; false otherwise + */ virtual void setCollecting_(bool collect, PipelineCollector*, simdb::DatabaseManager*) { (void)collect; } private: /** - * \brief A value that represents whether or not this TreeNode is - * being collected by a collector - */ + * \brief A value that represents whether or not this TreeNode is + * being collected by a collector + */ bool is_collected_ = false; }; diff --git a/sparta/sparta/collection/CollectionPoints.hpp b/sparta/sparta/collection/CollectionPoints.hpp deleted file mode 100644 index a00a7bbfe0..0000000000 --- a/sparta/sparta/collection/CollectionPoints.hpp +++ /dev/null @@ -1,280 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/collection/Scalars.hpp" -#include "simdb/collection/Structs.hpp" -#include "simdb/collection/IterableStructs.hpp" -#include "simdb/collection/TimeseriesCollector.hpp" -#include "simdb/collection/Any.hpp" -#include "sparta/simulation/Clock.hpp" -#include "sparta/utils/MetaStructs.hpp" - -#include -#include // For __cxa_demangle -#include - -namespace sparta -{ - inline std::string demangle(const char* mangled_name) { - int status = 0; - char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, nullptr, &status); - if (status == 0 && demangled_name) { - std::string result(demangled_name); - std::free(demangled_name); - return result; - } - return mangled_name; - } - - template - inline std::string demangled_type() { - return demangle(typeid(T).name()); - } -} - -namespace sparta { -namespace collection { - -class CollectionPoints -{ -public: - template - typename std::enable_if::value && std::is_standard_layout::value, void>::type - addStat(const std::string& location, const Clock* clk, const StatT* back_ptr, const simdb::Format format = simdb::Format::none) - { - auto demangled = demangled_type(); - auto& instantiator = instantiators_[demangled]; - if (!instantiator) { - instantiator.reset(new StatInstantiator()); - } - - dynamic_cast*>(instantiator.get())->addStat(location, clk, back_ptr, format); - } - - template - typename std::enable_if::value && std::is_standard_layout::value, void>::type - addStat(const std::string& location, const Clock* clk, std::function func_ptr, const simdb::Format format = simdb::Format::none) - { - auto demangled = demangled_type(); - auto& instantiator = instantiators_[demangled]; - if (!instantiator) { - instantiator.reset(new StatInstantiator()); - } - - dynamic_cast*>(instantiator.get())->addStat(location, clk, func_ptr, format); - } - - template - void addContainer(const std::string& location, const Clock* clk, const ContainerT* container, const size_t capacity) - { - auto demangled = demangled_type() + (Sparse ? "Sparse" : "Contig"); - auto& instantiator = instantiators_[demangled]; - if (!instantiator) { - instantiator.reset(new IterStructInstantiator()); - } - - dynamic_cast*>(instantiator.get())->addContainer(location, clk, container, capacity); - } - - template - simdb::AnyCollection* addManualCollectable(const std::string& location, const Clock* clk) - { - auto& instantiator = instantiators_["ManualCollectable"]; - if (!instantiator) { - instantiator.reset(new ManualCollectableInstantiator()); - } - - return dynamic_cast(instantiator.get())->addManualCollectable(location, clk); - } - - void createCollections(simdb::Collections* collections) - { - std::unordered_map clk_periods; - for (const auto& kvp : instantiators_) { - kvp.second->getClockPeriods(clk_periods); - } - - std::unordered_map clk_names_by_location; - for (const auto& kvp : instantiators_) { - kvp.second->getClockNamesByLocation(clk_names_by_location); - } - - for (const auto& kvp : clk_periods) { - collections->addClock(kvp.first, kvp.second); - } - - for (const auto& kvp : clk_names_by_location) { - collections->setClock(kvp.first, kvp.second); - } - - size_t idx = 0; - for (auto& kvp : instantiators_) { - const auto collection_prefix = "Collection" + std::to_string(idx); - kvp.second->createCollections(collections, collection_prefix); - ++idx; - } - - instantiators_.clear(); - } - -private: - class CollectableInstantiator - { - public: - virtual ~CollectableInstantiator() = default; - virtual void getClockPeriods(std::unordered_map& clk_periods) const = 0; - virtual void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const = 0; - virtual void createCollections(simdb::Collections* collections, const std::string& collection_prefix) = 0; - }; - - class ManualCollectableInstantiator : public CollectableInstantiator - { - public: - template - simdb::AnyCollection* addManualCollectable(const std::string& location, const Clock* clk) - { - clocks_.emplace_back(location, clk); - - using CollectionT = simdb::AnyCollection; - std::unique_ptr collection(new CollectionT(location)); - auto ret = collection.get(); - collections_.emplace_back(std::move(collection)); - return ret; - } - - private: - void getClockPeriods(std::unordered_map& clk_periods) const override - { - for (const auto& tup : clocks_) { - const auto clk = std::get<1>(tup); - clk_periods[clk->getName()] = clk->getPeriod(); - } - } - - void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override - { - for (const auto& tup : clocks_) { - const auto location = std::get<0>(tup); - const auto clk = std::get<1>(tup); - clk_names_by_location[location] = clk->getName(); - } - } - - void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override - { - for (auto& collection : collections_) { - collections->addCollection(std::move(collection)); - } - } - - std::vector> clocks_; - std::vector> collections_; - }; - - template - class StatInstantiator : public CollectableInstantiator - { - public: - void addStat(const std::string& location, const Clock* clk, const StatT* back_ptr, simdb::Format format = simdb::Format::none) - { - simdb::ScalarValueReader reader(back_ptr); - simdb::Stat stat(location, reader, format); - stats_.emplace_back(location, clk, stat); - } - - void addStat(const std::string& location, const Clock* clk, std::function func_ptr, simdb::Format format = simdb::Format::none) - { - simdb::ScalarValueReader reader(func_ptr); - simdb::Stat stat(location, reader, format); - stats_.emplace_back(location, clk, stat); - } - - void getClockPeriods(std::unordered_map& clk_periods) const override - { - for (const auto& tup : stats_) { - const auto clk = std::get<1>(tup); - clk_periods[clk->getName()] = clk->getPeriod(); - } - } - - void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override - { - for (const auto& tup : stats_) { - const auto location = std::get<0>(tup); - const auto clk = std::get<1>(tup); - clk_names_by_location[location] = clk->getName(); - } - } - - void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override - { - using CollectionT = simdb::StatCollection; - const auto collection_name = collection_prefix + "_" + demangled_type(); - std::unique_ptr collection(new CollectionT(collection_name)); - - for (const auto& tup : stats_) { - const Clock* clk = std::get<1>(tup); - const simdb::Stat& stat = std::get<2>(tup); - collection->addStat(stat, clk->getName()); - } - - collections->addCollection(std::move(collection)); - } - - private: - std::vector>> stats_; - }; - - template - class IterStructInstantiator : public CollectableInstantiator - { - public: - void addContainer(const std::string& location, const Clock* clk, const ContainerT* obj, const size_t capacity) - { - containers_.emplace_back(location, clk, obj, capacity); - } - - void getClockPeriods(std::unordered_map& clk_periods) const override - { - for (const auto& tup : containers_) { - const auto clk = std::get<1>(tup); - clk_periods[clk->getName()] = clk->getPeriod(); - } - } - - void getClockNamesByLocation(std::unordered_map& clk_names_by_location) const override - { - for (const auto& tup : containers_) { - const auto location = std::get<0>(tup); - const auto clk = std::get<1>(tup); - clk_names_by_location[location] = clk->getName(); - } - } - - void createCollections(simdb::Collections* collections, const std::string& collection_prefix) override - { - using CollectionT = simdb::IterableStructCollection; - - for (size_t idx = 0; idx < containers_.size(); ++idx) { - const std::string &location = std::get<0>(containers_[idx]); - const ContainerT *obj = std::get<2>(containers_[idx]); - const size_t capacity = std::get<3>(containers_[idx]); - - const auto collection_name = collection_prefix + "_" + demangled_type() + "_" + std::to_string(idx); - std::unique_ptr collection(new CollectionT(collection_name)); - - collection->addContainer(location, obj, capacity); - collections->addCollection(std::move(collection)); - } - } - - private: - std::vector> containers_; - }; - - std::unordered_map> instantiators_; -}; - -} // namespace collection -} // namespace sparta diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index ef4452c800..ebf87aa435 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -15,7 +15,6 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/collection/CollectableTreeNode.hpp" -#include "sparta/collection/CollectionPoints.hpp" #include "sparta/events/EventSet.hpp" #include "sparta/events/UniqueEvent.hpp" #include "sparta/events/GlobalOrderingPoint.hpp" From 7e7a5f31342619847b0c6907636ff2c610d82cf2 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 12:33:19 -0600 Subject: [PATCH 11/49] SimDB / v3 integration --- .../sparta/collection/PipelineCollector.hpp | 6 +- sparta/sparta/resources/CircularBuffer.hpp | 7 +- sparta/sparta/resources/Pipeline.hpp | 1 + .../TransactionDatabaseAPI/CMakeLists.txt | 41 ---- .../cpp_interface/CMakeLists.txt | 5 - .../TransactionDatabaseAPI_main.cpp | 192 ------------------ .../data_test_0simulation.info | 8 - .../scripts/test_transactiondb.py | 138 ------------- 8 files changed, 9 insertions(+), 389 deletions(-) delete mode 100644 sparta/test/TransactionDatabaseAPI/CMakeLists.txt delete mode 100644 sparta/test/TransactionDatabaseAPI/cpp_interface/CMakeLists.txt delete mode 100644 sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp delete mode 100644 sparta/test/TransactionDatabaseAPI/data_test_0simulation.info delete mode 100755 sparta/test/TransactionDatabaseAPI/scripts/test_transactiondb.py diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index ebf87aa435..22666efed2 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -195,7 +195,7 @@ namespace collection if(collection_active_) { for(auto & ctn : registered_collectables_) { if(ctn->isCollected()) { - ctn->closeRecord(true); + ctn->closeRecord(true); // set true for simulation termination } } } @@ -205,7 +205,9 @@ namespace collection void reactivate(const std::string& simdb_filename) { - (void)simdb_filename; + sparta_assert(simdb_filename.find(".db") != std::string::npos, + "Database filename must end in .db"); + sparta_assert(false, "TODO cnyce: Not implemented"); if (db_mgr_) { diff --git a/sparta/sparta/resources/CircularBuffer.hpp b/sparta/sparta/resources/CircularBuffer.hpp index 4549fbad75..3630fee22f 100644 --- a/sparta/sparta/resources/CircularBuffer.hpp +++ b/sparta/sparta/resources/CircularBuffer.hpp @@ -442,7 +442,9 @@ namespace sparta * instatiation of the PipelineCollector */ void enableCollection(TreeNode * parent) { - collector_ = std::make_unique(parent, name_, this, capacity()); + collector_. + reset(new collection::IterableCollector >(parent, getName(), + this, capacity())); } //! Get this CircularBuffer's name @@ -786,8 +788,7 @@ namespace sparta ////////////////////////////////////////////////////////////////////// // Collectors - using IterableCollectorType = sparta::collection::IterableCollector>; - std::unique_ptr collector_; + std::unique_ptr > > collector_; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/sparta/sparta/resources/Pipeline.hpp b/sparta/sparta/resources/Pipeline.hpp index 628133a2bf..ba74effccc 100644 --- a/sparta/sparta/resources/Pipeline.hpp +++ b/sparta/sparta/resources/Pipeline.hpp @@ -18,6 +18,7 @@ #include "sparta/simulation/Clock.hpp" #include "sparta/utils/SpartaAssert.hpp" #include "sparta/utils/MathUtils.hpp" +#include "sparta/collection/IterableCollector.hpp" #include "sparta/resources/Pipe.hpp" #include "sparta/events/Scheduleable.hpp" diff --git a/sparta/test/TransactionDatabaseAPI/CMakeLists.txt b/sparta/test/TransactionDatabaseAPI/CMakeLists.txt deleted file mode 100644 index a35b8b27b0..0000000000 --- a/sparta/test/TransactionDatabaseAPI/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -project(TransactionDatabaseAPI) - - -set( TransactionDatabase_scripts ${PROJECT_SOURCE_DIR}/scripts/) -set ( TransactionFile ${PROJECT_SOURCE_DIR}/data_test_0) - - - - -################################################################## -# Build the TransactionDatabaseAPI test command -################################################################## -add_custom_command(OUTPUT TransactionDatabaseAPI_test.sh - COMMAND printf 'export TRANSACTION_MODULE_DIR="${PROJECT_BINARY_DIR}/../../python/transactiondb/lib/" && export TRANSACTION_FILE="${TransactionFile}" && && python ${TransactionDatabase_scripts}/test_transactiondb.py' > TransactionDatabaseAPI_test.sh && chmod +x TransactionDatabaseAPI_test.sh - DEPENDS - ${TransactionDatabase_scripts}/test_transactiondb.py - COMMENT ${TransactionDatabaseAPI_test_cmd}) - - -add_custom_target (TransactionDatabaseAPI_test_sh_driver ALL DEPENDS TransactionDatabaseAPI_test.sh transactiondb2_driver - transactiondb_driver) - -add_custom_command(TARGET TransactionDatabaseAPI_test - COMMAND ./TransactionDatabaseAPI_test.sh - DEPENDS - TransactionDatabaseAPI_test.sh - ) - - - -# We need the python scripts to run currectly. -#sparta_test(TransactionDatabaseAPI_test) -add_test(NAME TransactionDatabaseAPI_test COMMAND - TransactionDatabaseAPI_test.sh - ) - -sparta_regress(TransactionDatabaseAPI_test_sh_driver) - -# We need the pipeViewer files copied over. -sparta_copy(TransactionDatabaseAPI_test_sh_driver data_test_0* .) -add_subdirectory(cpp_interface) diff --git a/sparta/test/TransactionDatabaseAPI/cpp_interface/CMakeLists.txt b/sparta/test/TransactionDatabaseAPI/cpp_interface/CMakeLists.txt deleted file mode 100644 index 9e9584668a..0000000000 --- a/sparta/test/TransactionDatabaseAPI/cpp_interface/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -project(Transactiondatabaseapi_cpp_test_cpp_interface) - -sparta_add_test_executable(TransactionDatabaseAPI_cpp_test TransactionDatabaseAPI_main.cpp) - -sparta_test(TransactionDatabaseAPI_cpp_test) diff --git a/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp b/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp deleted file mode 100644 index d5e402b171..0000000000 --- a/sparta/test/TransactionDatabaseAPI/cpp_interface/TransactionDatabaseAPI_main.cpp +++ /dev/null @@ -1,192 +0,0 @@ - -#include -#include - -// Required to enable std::this_thread::sleep_for -#ifndef _GLIBCXX_USE_NANOSLEEP -#define _GLIBCXX_USE_NANOSLEEP -#endif - -#include - -#include "sparta/sparta.hpp" -#include "sparta/Tree.hpp" -#include "sparta/functional/Register.hpp" -#include "sparta/functional/RegisterSet.hpp" -#include "sparta/utils/SpartaTester.hpp" -#include "sparta/utils/Utils.hpp" - -/*! - * \file TransactionDatabaseAPI_main.cpp - * \brief Test for reading from pipeout transaction database - */ - -TEST_INIT - -#define QUERIES_PER_SEC(num, boost_timer) ((num)/(boost_timer.elapsed().user/1000000000.0)) -#define SEC_PER_QUERY(num, boost_timer) ((boost_timer.elapsed().user/1000000000.0)/float(num)) - -using sparta::pipeViewer::TransactionDatabaseInterface; - -/*! - * \brief Helper for handling query responses - */ -class QueryResponse -{ -public: - uint64_t hits; - uint64_t empty; - uint64_t occupied; - - bool print; - - // TransactionDatabaseInterface::callback_fxn - static void tickDataHandler(void* obj, - uint64_t tick, - //TransactionDatabaseInterface::Transaction const * const * content, - TransactionDatabaseInterface::const_interval_idx * content, - const TransactionDatabaseInterface::Transaction * transactions, - uint32_t content_len) { - auto qr = static_cast(obj); - qr->gotTickData(tick, content, transactions, content_len); - } - - QueryResponse() : - hits(0), - empty(0), - occupied(0), - print(false) - {;} - - void reset() { - hits = empty = occupied = 0; - } - - // TransactionDatabaseInterface::callback_fxn without first argument - void gotTickData(uint64_t tick, - //TransactionDatabaseInterface::Transaction const * const * content, // indexed by location - TransactionDatabaseInterface::const_interval_idx * content, - const TransactionDatabaseInterface::Transaction * transactions, - uint32_t content_len) - { - (void) tick; - (void) content; - (void) transactions; - (void) content_len; - if(!print){ - //for(auto i = 0ul; i < content_len; ++i){ - // const TransactionDatabaseInterface::Transaction* ti = content[i]; - // if(nullptr == ti){ - // ++empty; - // }else{ - // ++occupied; - // } - //} - uint32_t num_inspected = std::min(200ul, content_len); - for(auto i = 0ul; i < num_inspected; ++i){ - const TransactionDatabaseInterface::const_interval_idx ti = content[i]; - if(TransactionDatabaseInterface::NO_TRANSACTION == ti){ - ++empty; - }else{ - ++occupied; - } - } - }else{ - std::cout << std::setw(6) << tick << ": "; - // Print all transaction ids in a row - for(auto i = 0ul; i < content_len; ++i){ - const TransactionDatabaseInterface::const_interval_idx ti = content[i]; - if(TransactionDatabaseInterface::NO_TRANSACTION == ti){ - std::cout << "---- "; - ++empty; - }else{ - //std::cout << std::setw(4) << ti->transaction_ID % 10000 << ' '; - std::cout << std::setw(4) << transactions[ti].transaction_ID % 10000 << ' '; - ++occupied; - } - } - std::cout << std::endl; - } - - ++hits; - } -}; - -void query(TransactionDatabaseInterface& db, QueryResponse& qr, - uint64_t start_inc, uint64_t end_inc, uint32_t num_queries){ - boost::timer::cpu_timer t; - qr.reset(); - t.start(); - std::cout << "query [" << start_inc << "," << end_inc << "] x " << num_queries << std::endl; - for(uint32_t n = 0; n < num_queries; ++n){ - db.query(start_inc, end_inc, QueryResponse::tickDataHandler, &qr); - } - std::cout << " " << QUERIES_PER_SEC(num_queries, t) << " qps, " << SEC_PER_QUERY(num_queries, t) << std::endl; - std::cout << " " << qr.hits << " cycles, " << qr.empty << " empty, " << qr.occupied << " occupied\n"; - std::cout << " " << db.stringize() << std::endl; - db.writeNodeStates(std::cout) << std::endl; -} - -int main(int argc, char** argv) -{ - std::string dbname = "../data_test_0"; - uint32_t num_locs = 1000; - uint64_t print_start = 0; - uint64_t print_stop = 0; - - if(argc > 1){ - dbname = argv[1]; - if(argc > 2){ - num_locs = atoi(argv[2]); - if(argc > 3){ - if(argc > 4){ - char* end; - print_start = strtoull(argv[3], &end, 10); - print_stop = strtoull(argv[4], &end, 10); - }else{ - std::cerr << "Arguments 3 and 4 (print start tick, print stop tick) are both " - "required if one is specified" << std::endl; - return 1; - } - } - } - } - - std::cout << "db: \"" << dbname << "\", num_locs: " << num_locs << std::endl; - - TransactionDatabaseInterface db(dbname, // db - num_locs); // delibarately fewer slots than transactions - - std::cout << "File: [" << db.getFileStart() << ", " << db.getFileEnd() << ')' << std::endl; - - QueryResponse qr; // Query response handler - boost::timer::cpu_timer t; - //double dt; - - const uint32_t NUM_QUERIES = 5000; - - query(db, qr, 0, 100, 1); - query(db, qr, 0, 100, NUM_QUERIES); - query(db, qr, 500, 600, 1); - query(db, qr, 500, 600, NUM_QUERIES); - query(db, qr, 0, 700, 1);; - query(db, qr, 0, 700, NUM_QUERIES/2); - query(db, qr, 200, 3760, 1); - query(db, qr, 200, 3760, NUM_QUERIES/4); - query(db, qr, 2999, 4000, 1); - query(db, qr, 2999, 4000, NUM_QUERIES/4); - query(db, qr, 6000, 6300, 1); - query(db, qr, 6000, 6300, NUM_QUERIES/4); - //qr.print = true; - //query(db, qr, 2999, 4000, 1); - - // print if enabled - if(print_stop > 0){ - qr.print = true; - query(db, qr, print_start, print_stop, 1); - } - - REPORT_ERROR; - - return ERROR_CODE; -} diff --git a/sparta/test/TransactionDatabaseAPI/data_test_0simulation.info b/sparta/test/TransactionDatabaseAPI/data_test_0simulation.info deleted file mode 100644 index 66b0002714..0000000000 --- a/sparta/test/TransactionDatabaseAPI/data_test_0simulation.info +++ /dev/null @@ -1,8 +0,0 @@ -Pipeline Collection files generated from Simulation sparta example_core - -Simulation started at: Fri Jul 27 17:01:04 2012 - -Simulation ended at: Fri Jul 27 17:01:04 2012 - -Num Transactions Written: 0 -Heartbeat interval: 50000 cycles diff --git a/sparta/test/TransactionDatabaseAPI/scripts/test_transactiondb.py b/sparta/test/TransactionDatabaseAPI/scripts/test_transactiondb.py deleted file mode 100755 index d4af928bbb..0000000000 --- a/sparta/test/TransactionDatabaseAPI/scripts/test_transactiondb.py +++ /dev/null @@ -1,138 +0,0 @@ -import sys -import time -import os - -# Setup import path for transactiondb module -TRANSACTION_MODULE_DIR = os.environ.get('TRANSACTION_MODULE_DIR', None) -if TRANSACTION_MODULE_DIR: - sys.path.append(TRANSACTION_MODULE_DIR) - -try: - import transactiondb2 as tdb2 -except Exception as e: - print >> sys.stderr, 'Failed to import transactiondb2: {0}'.format(e) - print >> sys.stderr, 'TRANSACTION_MODULE_DIR was: {0}'.format(TRANSACTION_MODULE_DIR) - print >> sys.stderr, 'sys.path was {0}'.format(sys.path) - raise - -print sys.modules['transactiondb2'] - -TRANSACTION_FILE = os.environ['TRANSACTION_FILE'] - -from check_interval_end import check_interval_endings - -# Try opening a non-existant file and ensure the exception is caught -NONEXISTANT_FILE = '/dev/null/nonexistant_file' -try: - w = tdb2.TransactionDatabase(NONEXISTANT_FILE, 2000) -except Exception as e: - print('Successfully caught bad construction with exception: {0} type: {1}' \ - .format(e, type(e))) - pass -else: - raise RuntimeError('Should have failed to open "{0}"'.format(NONEXISTANT_FILE)) - -window = tdb2.TransactionDatabase(TRANSACTION_FILE, 2000) -print(window) -print(str(window)) -print(repr(window)) - -# Make a query to kick off the background loading -window.query(0,1, lambda t,q: true) - -time.sleep(2.0) # Allow data to load - -assert window.getFileStart() < 1000, window.getFileStart() -assert window.getFileEnd() > 1000, window.getFileEnd() -assert window.getWindowLeft() < 1000, window.getWindowLeft() -assert window.getWindowRight() > 1000, window.getWindowRight() - -results = [] -def query_callback(tick, qobj): - ##print 'Query at ', tick - results.append([tick] + - [qobj.getTransactionID(i) for i in range(1800)[::10]] - - #[qobj.getTransactionAnnotation(3), - # qobj.getTransactionAnnotation(12), - # qobj.getTransactionAnnotation(33), - # qobj.getTransactionAnnotation(45), - # qobj.getTransactionAnnotation(51), - ) - -print window.getLocationMap() - -window.query(0, 250, query_callback) - -import pprint -for r in results: - print r - -print('Number of results: {0}'.format(len(results))) - -print('Locations: {0}'.format(window.getLocationMap())) - -print(window) -print('Cached Annotations: {0}'.format(window.getNumCachedAnnotations())) - -time.sleep(1.0) - -##real = None -##prox = None -##max_loc = 0 -##max_id = 0 -##for idx,trans in enumerate(results): -## max_loc = max(trans.getLocationID(), max_loc) -## max_id = max(trans.getTransactionID(), max_id) -## -## if idx == 10: -## print trans.getType() -## print trans.getTypeString() -## real = trans.makeRealCopy() -## prox = trans.makeProxy() -## -##print('max_locid {0}'.format(max_loc)) -##print('max_transid {0}'.format(max_id)) -## -### Both safe until IntervalList is destroyed. After, they are dangerous to access -### and they don't know it (segfaults could occur) -##print real -##print prox -## -##assert not real.isProxy() -##assert real.isValid() -##assert prox.isProxy() -##assert prox.isValid() -## -##check_interval_endings(window) -## -##print('Destroying IntervalList') -##results._destroy() - -print('Destroying and IntervalWindow') -window._destroy() - -# Must be able to print destroyed objects just to show their destroyed state -print window - -### Must not be able to access once destroyed. -### Should really try every method to be sure -##try: -## results.getLength() -##except Exception as e: -## print('Caught access to destroyed IntervalList as expected') -##else: -## raise RuntimeError('Should have failed to access destroyed IntervalList') - -try: - window.getFileStart() -except Exception as e: - print('Caught access to destroyed IntervalWindow as expected') -else: - raise RuntimeError('Should have failed to access destroyed IntervalWindow') - -del window - -print 'Success' -print 'Used transactiondb2 lib from: {0}'.format(sys.modules['transactiondb2']) -exit(0) From 552e3466568f80eea5245afcde3e00bbba973115 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 13:35:23 -0600 Subject: [PATCH 12/49] SimDB / v3 integration --- sparta/sparta/collection/Collectable.hpp | 6 ++-- .../sparta/collection/DelayedCollectable.hpp | 4 +-- .../sparta/collection/IterableCollector.hpp | 31 +++++++++++++++++-- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index f44a654b7d..c62cebc79a 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -114,7 +114,7 @@ namespace sparta{ //TODO cnyce (void)val; - if(SPARTA_EXPECT_FALSE(isCollected())) + if(SPARTA_EXPECT_TRUE(isCollected())) { //std::ostringstream ss; //ss << val; @@ -161,7 +161,7 @@ namespace sparta{ template MetaStruct::enable_if_t::value, void> collectWithDuration(const T & val, sparta::Clock::Cycle duration){ - if(SPARTA_EXPECT_FALSE(isCollected())) + if(SPARTA_EXPECT_TRUE(isCollected())) { if(duration != 0) { ev_close_record_.preparePayload(false)->schedule(duration); @@ -220,7 +220,7 @@ namespace sparta{ //! Force close a record. void closeRecord(const bool & simulation_ending = false) override final { - if(SPARTA_EXPECT_FALSE(isCollected())) + if(SPARTA_EXPECT_TRUE(isCollected())) { if(!record_closed_) { writeRecord_(simulation_ending); diff --git a/sparta/sparta/collection/DelayedCollectable.hpp b/sparta/sparta/collection/DelayedCollectable.hpp index 159b885677..9287b00f00 100644 --- a/sparta/sparta/collection/DelayedCollectable.hpp +++ b/sparta/sparta/collection/DelayedCollectable.hpp @@ -111,7 +111,7 @@ namespace collection */ void collect(const DataT & val, sparta::Clock::Cycle delay) { - if(SPARTA_EXPECT_FALSE(isCollected())) + if(SPARTA_EXPECT_TRUE(isCollected())) { if(delay != 0) { ev_collect_.schedule(val, delay); @@ -136,7 +136,7 @@ namespace collection sparta::Clock::Cycle delay, sparta::Clock::Cycle duration) { - if(SPARTA_EXPECT_FALSE(isCollected())) + if(SPARTA_EXPECT_TRUE(isCollected())) { if(delay != 0) { ev_collect_duration_.preparePayload({val, duration})->schedule(delay); diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index c23bf4a27c..baea40e283 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -21,6 +21,31 @@ namespace sparta { namespace collection { +/** + * \class IterableCollector + * \brief A collector of any iterable type (std::vector, std::list, sparta::Buffer, etc) + * + * \tname IterableType The type of the collected object + * \tname collection_phase The phase collection will occur. + * Collection happens automatically in this + * phase, unless it is disabled by a call to + * setManualCollection() + * \tname sparse_array_type Set to true if the iterable type is + * sparse, meaning iteration will occur on + * the entire iterable object, but each + * iterator might not be valid to + * de-reference. When this is true, it is + * expected that the iterator returned from + * the IterableType can be queried for + * validity (by a call to itr->isValid()). + * + * This collector will iterable over an std::array type, std::list + * type, std::vector type, sparta::Buffer, sparta::Queue, sparta::Array, or + * even a simple C-array type. The class needs to constructed with an + * expected capacity of the container, and the container should never + * grow beyond this expected capacity. If so, during collection, the + * class will output a warning message (only once). + */ template class IterableCollector : public CollectableTreeNode { @@ -155,7 +180,7 @@ class IterableCollector : public CollectableTreeNode if(nullptr == iterable_object) { closeRecord(); } - else if (SPARTA_EXPECT_FALSE(isCollected())) + else if (SPARTA_EXPECT_TRUE(isCollected())) { if(SPARTA_EXPECT_FALSE(iterable_object->size() > expected_capacity_)) { @@ -191,7 +216,7 @@ class IterableCollector : public CollectableTreeNode //! Schedule event to close all records for this iterable type. void scheduleCloseRecord(sparta::Clock::Cycle cycle) { - if(SPARTA_EXPECT_FALSE(isCollected())) { + if(SPARTA_EXPECT_TRUE(isCollected())) { ev_close_record_.preparePayload(false)->schedule(cycle); } } @@ -205,7 +230,7 @@ class IterableCollector : public CollectableTreeNode //! \brief Perform a collection, then close the records in the future //! \param duration The time to close the records, 0 is not allowed void collectWithDuration(sparta::Clock::Cycle duration) { - if(SPARTA_EXPECT_FALSE(isCollected())) { + if(SPARTA_EXPECT_TRUE(isCollected())) { collect(); if(duration != 0) { ev_close_record_.preparePayload(false)->schedule(duration); From 335b8ccaeffa09067c07df2721518fc321df7ad6 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 15:40:31 -0600 Subject: [PATCH 13/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index f80a4cb268..7932083217 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit f80a4cb268e491c86f78f8d6de41a9f767068331 +Subproject commit 79320832179d9ace8386c72870ccac3f82337506 From 14e8e50bd1c0f5b55ffea09a40d9eb1ecebb2db3 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 12 Feb 2025 17:04:26 -0600 Subject: [PATCH 14/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index 7932083217..d47c36651d 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 79320832179d9ace8386c72870ccac3f82337506 +Subproject commit d47c36651dff037512d106159cef330dde438eb2 From 9229a6891a2c7d967e47fdc677044eeea34977d1 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 15 Feb 2025 07:27:07 -0600 Subject: [PATCH 15/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index d47c36651d..59c7267a5f 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit d47c36651dff037512d106159cef330dde438eb2 +Subproject commit 59c7267a5fd4a3c5ee756006cd60a96125ca6679 From b4a18ed30fa15a7c51b039a3185ce4afc62f0962 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 15 Feb 2025 16:47:43 -0600 Subject: [PATCH 16/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 2 +- .../CoreModel/src/LoadStoreInstInfo.hpp | 1 + .../example/CoreModel/src/MemAccessInfo.hpp | 2 +- sparta/simdb | 2 +- sparta/sparta/collection/Collectable.hpp | 75 +++++----- .../sparta/collection/IterableCollector.hpp | 138 +++--------------- .../sparta/collection/PipelineCollector.hpp | 102 +++++++++++-- sparta/sparta/simulation/TreeNode.hpp | 10 ++ 8 files changed, 166 insertions(+), 166 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index eb582b6a89..02ef248579 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -9,7 +9,7 @@ #include "sparta/simulation/State.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" -#include "simdb/collection/Structs.hpp" +#include "simdb/serialize/Serialize.hpp" #include #include diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index a5fdb95bd4..d3abfc06a8 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -1,6 +1,7 @@ #pragma once #include "MemAccessInfo.hpp" +#include "simdb/serialize/Serialize.hpp" namespace core_example { diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index 4af84c2ddc..47c0e1f343 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -1,7 +1,7 @@ #pragma once #include "ExampleInst.hpp" -#include "simdb/collection/Structs.hpp" +#include "simdb/serialize/Serialize.hpp" namespace core_example { diff --git a/sparta/simdb b/sparta/simdb index 59c7267a5f..e761860957 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 59c7267a5fd4a3c5ee756006cd60a96125ca6679 +Subproject commit e7618609570ba1f5c4542922f8c235685534e7c2 diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index c62cebc79a..b1f49760c1 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -21,6 +21,7 @@ #include "sparta/events/SchedulingPhases.hpp" #include "sparta/utils/Utils.hpp" #include "sparta/utils/MetaStructs.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" namespace sparta{ namespace collection @@ -82,8 +83,6 @@ namespace sparta{ desc) { collected_object_ = collected_object; - - //TODO cnyce: prev_annot_ / initialize() } /** @@ -111,26 +110,9 @@ namespace sparta{ MetaStruct::enable_if_t::value, void> collect(const T & val) { - //TODO cnyce - (void)val; - if(SPARTA_EXPECT_TRUE(isCollected())) { - //std::ostringstream ss; - //ss << val; - //if((ss.str() != prev_annot_) && !record_closed_) - //{ - // // Close the old record (if there is one) - // closeRecord(); - //} - - // Remember the new string for a new record and start - // a new record if not empty. - //prev_annot_ = ss.str(); - //if(!prev_annot_.empty() && record_closed_) { - // startNewRecord_(); - // record_closed_ = false; - //} + simdb_collectable_->activate(val); } } @@ -200,7 +182,7 @@ namespace sparta{ closeRecord(); return; } - collect(*collected_object_); + simdb_collectable_->activate(collected_object_); } /*! @@ -223,7 +205,7 @@ namespace sparta{ if(SPARTA_EXPECT_TRUE(isCollected())) { if(!record_closed_) { - writeRecord_(simulation_ending); + simdb_collectable_->deactivate(); record_closed_ = true; } } @@ -236,6 +218,15 @@ namespace sparta{ auto_collect_ = false; } + /*! + * \brief The pipeline collector will call this method on all nodes + * as soon as the collector is created. + */ + void configCollectable(simdb::CollectionMgr *mgr) override final { + using value_type = MetaStruct::remove_any_pointer_t; + simdb_collectable_ = mgr->createCollectable(getLocation(), getClock()->getName()); + } + protected: //! \brief Get a reference to the internal event set @@ -245,20 +236,6 @@ namespace sparta{ } private: - //! Return true if the record was written; false otherwise - bool writeRecord_(bool simulation_ending = false) - { - //TODO cnyce - (void)simulation_ending; - return true; - } - - //! Start a new record - void startNewRecord_() - { - //TODO cnyce - } - void setCollecting_(bool collect, PipelineCollector * collector, simdb::DatabaseManager*) override final { // If the collected object is null, this Collectable @@ -275,6 +252,21 @@ namespace sparta{ // collection collector->removeFromAutoCollection(this); } + } else { + if(collect) { + // If we are manually collecting, we still need to tell the collector + // to run the sweep() method every cycle on our clock. + // + // Note that addToAutoCollection() implicitly calls addToAutoSweep(). + collector->addToAutoSweep(this); + } + else { + // If we are no longer collecting, remove this Collectable from the + // once-a-cycle sweep() method. + // + // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). + collector->removeFromAutoSweep(this); + } } if(!collect && !record_closed_) { @@ -296,7 +288,12 @@ namespace sparta{ // Should we auto-collect? bool auto_collect_ = true; - }; -}//namespace collection -}//namespace sparta + // The simdb collectable object. We will "activate" and + // "deactivate" this object when we want to collect data. + // While it is activated, the SimDB collection system will + // collect our data along with everyone else's. + std::shared_ptr simdb_collectable_; + }; + } // namespace collection +} // namespace sparta diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index baea40e283..c6c6285c7d 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -71,11 +71,11 @@ class IterableCollector : public CollectableTreeNode const size_type expected_capacity) : CollectableTreeNode(parent, name, group, index, desc), iterable_object_(iterable), - expected_capacity_(expected_capacity), - event_set_(this), - ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", - CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)) + expected_capacity_(expected_capacity) { + sparta_assert(iterable_object_ != nullptr, + "IterableCollector " << getLocation() << " cannot be constructed with a nullptr iterable object"); + for (size_type i = 0; i < expected_capacity_; ++i) { std::stringstream name_str; @@ -170,71 +170,10 @@ class IterableCollector : public CollectableTreeNode // Delegated constructor } - //! Collect the contents of the iterable object. This function - //! will walk starting from index 0 -> expected_capacity, clearing - //! out any records where the iterable object does not contain - //! data. - void collect(const IterableType * iterable_object) - { - // If pointer has become nullified, close the records - if(nullptr == iterable_object) { - closeRecord(); - } - else if (SPARTA_EXPECT_TRUE(isCollected())) - { - if(SPARTA_EXPECT_FALSE(iterable_object->size() > expected_capacity_)) - { - if(SPARTA_EXPECT_FALSE(warn_on_size_)) - { - sparta::log::MessageSource::getGlobalWarn() - << "WARNING! The collected object '" - << getLocation() << "' has grown beyond the " - << "expected capacity (given at construction) for collection. " - << "Expected " << expected_capacity_ << " but grew to " - << iterable_object->size() - << " This is your first and last warning."; - warn_on_size_ = false; - } - } - collectImpl_(iterable_object, std::integral_constant()); - } - } - //! Collect the contents of the associated iterable object void collect() override { - collect(iterable_object_); - } - - //! Force close all records for this iterable type. This will - //! close the record immediately and clear the field for the next - //! cycle - void closeRecord(const bool & simulation_ending = false) override { - for (size_type i = 0; i < positions_.size(); ++i) { - positions_[i]->closeRecord(simulation_ending); - } - } - - //! Schedule event to close all records for this iterable type. - void scheduleCloseRecord(sparta::Clock::Cycle cycle) { - if(SPARTA_EXPECT_TRUE(isCollected())) { - ev_close_record_.preparePayload(false)->schedule(cycle); - } - } - - //! \brief Do not perform any automatic collection - //! The SchedulingPhase is ignored - void setManualCollection() { - auto_collect_ = false; - } - - //! \brief Perform a collection, then close the records in the future - //! \param duration The time to close the records, 0 is not allowed - void collectWithDuration(sparta::Clock::Cycle duration) { if(SPARTA_EXPECT_TRUE(isCollected())) { - collect(); - if(duration != 0) { - ev_close_record_.preparePayload(false)->schedule(duration); - } + simdb_collectable_->activate(iterable_object_); } } @@ -243,55 +182,33 @@ class IterableCollector : public CollectableTreeNode iterable_object_ = obj; } -private: - typedef Collectable::value_type> CollectableT; - // Standard walk of iterable types - void collectImpl_(const IterableType * iterable_object, std::false_type) - { - sparta_assert(nullptr != iterable_object, - "Can't collect iterable_object because it's a nullptr! How did we get here?"); - auto itr = iterable_object->begin(); - auto eitr = iterable_object->end(); - for (uint32_t i = 0; i < expected_capacity_; ++i) - { - if (itr != eitr) { - positions_[i]->collect(*itr); - ++itr; - } else { - positions_[i]->closeRecord(); - } - } + /*! + * \brief The pipeline collector will call this method on all nodes + * as soon as the collector is created. + */ + void configCollectable(simdb::CollectionMgr *mgr) override final { + simdb_collectable_ = mgr->createIterableCollector( + getLocation(), + getClock()->getName(), + expected_capacity_); } - // Full iteration walk, checking validity of the iterator. This - // is used for Pipe and Array where the iterator points to valid - // and not valid entries in the component - void collectImpl_(const IterableType * iterable_object, std::true_type) - { - sparta_assert(nullptr != iterable_object, - "Can't collect iterable_object because it's a nullptr! How did we get here?"); - uint32_t s = 0; - auto itr = iterable_object->begin(); - for (uint32_t i = 0; i < expected_capacity_; ++i, ++s) - { - if (itr.isValid()) { - positions_[i]->collect(*itr); - } else { - positions_[i]->closeRecord(); - } - ++itr; - } - } +private: + typedef Collectable::value_type> CollectableT; //! Virtual method called by CollectableTreeNode when collection //! is enabled on the TreeNode void setCollecting_(bool collect, PipelineCollector* collector, simdb::DatabaseManager* db_mgr) override { - if(collect && auto_collect_) { + if (collect) { // Add this Collectable to the PipelineCollector's // list of objects requiring collection collector->addToAutoCollection(this, collection_phase); - } - else { + } else { + // If we are no longer collecting, remove this Collectable from the + // once-a-cycle sweep() method. + // + // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). + collector->removeFromAutoSweep(this); closeRecord(); } } @@ -299,14 +216,7 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable_object_; std::vector> positions_; const size_type expected_capacity_ = 0; - bool auto_collect_ = true; - bool warn_on_size_ = true; - - // For those folks that want a value to automatically - // disappear in the future - sparta::EventSet event_set_; - sparta::PayloadEvent ev_close_record_; - + std::shared_ptr> simdb_collectable_; }; } // namespace collection } // namespace sparta diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 22666efed2..3c1bdc97ed 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -20,8 +20,6 @@ #include "sparta/events/GlobalOrderingPoint.hpp" #include "sparta/kernel/Scheduler.hpp" #include "sparta/simulation/TreeNodePrivateAttorney.hpp" - -#include "simdb/collection/Scalars.hpp" #include "simdb/sqlite/DatabaseManager.hpp" namespace sparta { @@ -102,6 +100,13 @@ namespace collection void performCollection() { for(auto & ctn : enabled_ctns_) { if(ctn->isCollected()) { + //This is happening on a specific clock and a specific phase. + //We need to honor the collectable value at this very time, + //even though the actual sweep() does not occur until PostTick. + // + //This only has an effect for automatically collected types. + //Manually collected types always ignore the phase and collect + //immediately. ctn->collect(); } } @@ -128,7 +133,7 @@ namespace collection public: PipelineCollector(const std::string& simdb_filename, const size_t heartbeat, - const sparta::TreeNode * root) + sparta::TreeNode * root) : db_mgr_(std::make_unique(simdb_filename, true)) { sparta_assert(root->isFinalized() == true, @@ -147,16 +152,14 @@ namespace collection // DatabaseManager adds automatically to support this feature. simdb::Schema schema; db_mgr_->createDatabaseFromSchema(schema); - - std::function func_ptr = std::bind(&PipelineCollector::getCollectionTick_, this); - db_mgr_->getCollectionMgr()->useTimestampsFrom(func_ptr); - db_mgr_->getCollectionMgr()->setHeartbeat(heartbeat); + db_mgr_->enableCollection(heartbeat); // Initialize the clock/collectable map std::function addClks; addClks = [&addClks, this] (const sparta::Clock* clk) { if(clk != nullptr){ + db_mgr_->getCollectionMgr()->addClock(clk->getName(), clk->getPeriod()); auto & u_p = clock_ctn_map_[clk]; for(uint32_t i = 0; i < NUM_SCHEDULING_PHASES; ++i) { u_p[i].reset(new CollectablesByClock(clk, static_cast(i))); @@ -171,6 +174,24 @@ namespace collection }; addClks(root->getClock()); + + std::queue q; + std::unordered_set visited; + q.push(root); + + while (!q.empty()) { + auto node = q.front(); + q.pop(); + + if (!visited.insert(node).second) { + continue; + } + + node->configCollectable(db_mgr_->getCollectionMgr()); + for (auto child : sparta::TreeNodePrivateAttorney::getAllChildren(node)) { + q.push(child); + } + } } ~PipelineCollector() { @@ -310,6 +331,7 @@ namespace collection auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); sparta_assert(ccm_pair != clock_ctn_map_.end()); ccm_pair->second[static_cast(collection_phase)]->enable(ctn); + addToAutoSweep(ctn); } /** @@ -330,6 +352,26 @@ namespace collection for(auto & u_p : ccm_pair->second) { u_p->disable(ctn); } + removeFromAutoSweep(ctn); + } + + void addToAutoSweep(CollectableTreeNode * ctn) + { + auto& sweep = sweepers_[ctn->getClock()]; + if (!sweep) { + sweep.reset(new ClockDomainSweeper(db_mgr_->getCollectionMgr(), ctn->getClock())); + } + sweep->enable(ctn); + } + + void removeFromAutoSweep(CollectableTreeNode * ctn) + { + auto iter = sweepers_.find(ctn->getClock()); + if (iter == sweepers_.end()) { + return; + } + + iter->second->disable(ctn); } //! \return the pipeout file path @@ -344,10 +386,47 @@ namespace collection } private: - uint64_t getCollectionTick_() const + class ClockDomainSweeper { - return scheduler_->getCurrentTick() - 1; - } + public: + ClockDomainSweeper(simdb::CollectionMgr* mgr, const Clock* clk) + : clk_(clk) + , collection_mgr_(mgr) + , ev_set_(nullptr) + , ev_sweep_(&ev_set_, clk->getName() + "_sweep_event", + CREATE_SPARTA_HANDLER(ClockDomainSweeper, performSweep_), 1) + { + ev_sweep_.setScheduleableClock(clk); + ev_sweep_.setScheduler(clk->getScheduler()); + ev_sweep_.setContinuing(false); + } + + void enable(CollectableTreeNode* ctn) { + sweepables_.insert(ctn); + ev_sweep_.schedule(); + } + + void disable(CollectableTreeNode* ctn) { + sweepables_.erase(ctn); + } + + private: + void performSweep_() { + auto tick = clk_->getScheduler()->getCurrentTick(); + collection_mgr_->sweep(clk_->getName(), tick); + + if (!sweepables_.empty()) { + ev_sweep_.schedule(); + } + } + + const Clock* clk_; + simdb::CollectionMgr* collection_mgr_; + std::unordered_set sweepables_; + + EventSet ev_set_; + sparta::UniqueEvent ev_sweep_; + }; //! The SimDB database std::unique_ptr db_mgr_; @@ -357,6 +436,9 @@ namespace collection //! Is collection enabled on at least one node? bool collection_active_ = false; + + //! Actively auto-sweeping nodes + std::unordered_map> sweepers_; }; }// namespace collection diff --git a/sparta/sparta/simulation/TreeNode.hpp b/sparta/sparta/simulation/TreeNode.hpp index 73b76f5e12..15a9add2a7 100644 --- a/sparta/sparta/simulation/TreeNode.hpp +++ b/sparta/sparta/simulation/TreeNode.hpp @@ -40,6 +40,10 @@ namespace sparta { class PostRunValidationInfo; } // namespace sparta +namespace simdb { +class CollectionMgr; +} // namespace simdb + #ifndef TREENODE_LIFETIME_TRACE /*! * \brief Enables tracing of TreeNode lifetimes in a set of output txt files. @@ -2169,6 +2173,12 @@ namespace sparta */ virtual void activateLink(const std::string &label); + /*! + * \brief The pipeline collector will call this method on all nodes + * as soon as the collector is created. + */ + virtual void configCollectable(simdb::CollectionMgr *) {} + /*! * \brief Compute a regex pattern for a node child path containing any * number of wildcard characters (not a dot-separated location) which From d711e308f481e5cfc8e595f624e985a6c344b8e9 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sun, 16 Feb 2025 11:03:54 -0600 Subject: [PATCH 17/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 3 +-- .../CoreModel/src/LoadStoreInstInfo.hpp | 3 +-- .../example/CoreModel/src/MemAccessInfo.hpp | 3 +-- sparta/simdb | 2 +- .../sparta/collection/PipelineCollector.hpp | 25 +++++++++++-------- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 02ef248579..0b7d91daa5 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -239,11 +239,10 @@ inline void defineEnumMap(std::string& en } template <> -inline void defineStructSchema(StructSchema& schema) +inline void defineStructSchema(StructSchema& schema) { using TargetUnit = core_example::ExampleInst::TargetUnit; - schema.setStructName("ExampleInst"); schema.addField("DID"); schema.addField("uid"); schema.addField("mnemonic"); diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index d3abfc06a8..15ec4b1265 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -182,9 +182,8 @@ namespace simdb { template <> -inline void defineStructSchema(StructSchema& schema) +inline void defineStructSchema(StructSchema& schema) { - schema.setStructName("LSInstInfo"); schema.addField("DID"); schema.addField("rank"); schema.addField("state"); diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index 47c0e1f343..b5b0d78031 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -155,9 +155,8 @@ inline void defineEnumMap(std::strin } template <> -inline void defineStructSchema(StructSchema& schema) +inline void defineStructSchema(StructSchema& schema) { - schema.setStructName("MemoryAccessInfo"); schema.addField("DID"); schema.addBoolField("valid"); schema.addField("mmu"); diff --git a/sparta/simdb b/sparta/simdb index e761860957..5e5e83a405 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit e7618609570ba1f5c4542922f8c235685534e7c2 +Subproject commit 5e5e83a405fb5c3ee3f0b2d3196ef9bacbf0c55d diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 3c1bdc97ed..0859f1131e 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -179,19 +179,24 @@ namespace collection std::unordered_set visited; q.push(root); - while (!q.empty()) { - auto node = q.front(); - q.pop(); + db_mgr_->safeTransaction([&](){ + while (!q.empty()) { + auto node = q.front(); + q.pop(); - if (!visited.insert(node).second) { - continue; - } + if (!visited.insert(node).second) { + continue; + } - node->configCollectable(db_mgr_->getCollectionMgr()); - for (auto child : sparta::TreeNodePrivateAttorney::getAllChildren(node)) { - q.push(child); + node->configCollectable(db_mgr_->getCollectionMgr()); + for (auto child : sparta::TreeNodePrivateAttorney::getAllChildren(node)) { + q.push(child); + } } - } + + db_mgr_->finalizeCollections(); + return true; + }); } ~PipelineCollector() { From 6c610a82e2e0d18aabb13f532fcc921c1b7ce381 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 17 Feb 2025 08:59:43 -0600 Subject: [PATCH 18/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleSimulation.cpp | 7 +++---- sparta/simdb | 2 +- sparta/sparta/collection/IterableCollector.hpp | 7 ++++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleSimulation.cpp b/sparta/example/CoreModel/src/ExampleSimulation.cpp index f9a9fb5482..4490644015 100644 --- a/sparta/example/CoreModel/src/ExampleSimulation.cpp +++ b/sparta/example/CoreModel/src/ExampleSimulation.cpp @@ -369,14 +369,13 @@ void ExampleSimulator::configureTree_() // To get better coverage of the pipeline collector, we will use different clock // domains for each core when collection is enabled. auto pc = getSimulationConfiguration()->pipeline_collection_file_prefix != sparta::app::NoPipelineCollectionStr; - if (pc) { + if (pc && num_cores_ > 1) { auto root_clk = getClockManager().getRoot(); for (uint32_t core_idx = 1; core_idx < num_cores_; ++core_idx) { const std::string clk_name = "core" + std::to_string(core_idx) + "_clk"; - const auto numer = 1; - const auto denom = core_idx + 1; - auto core_clk = getClockManager().makeClock("core1_clk", root_clk, numer, denom); + auto core_clk = getClockManager().makeClock(clk_name, root_clk); auto core = getRoot()->getChild("cpu.core" + std::to_string(core_idx)); + core_clk->setPeriod(root_clk->getPeriod() * (core_idx + 1)); core->setClock(core_clk.get()); } } diff --git a/sparta/simdb b/sparta/simdb index 5e5e83a405..6fcc148296 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 5e5e83a405fb5c3ee3f0b2d3196ef9bacbf0c55d +Subproject commit 6fcc148296194fcf608412500d07ddc9f7c00103 diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index c6c6285c7d..d4493849f1 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -216,7 +216,12 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable_object_; std::vector> positions_; const size_type expected_capacity_ = 0; - std::shared_ptr> simdb_collectable_; + + using simdb_collectable_type = std::conditional_t; + + std::shared_ptr simdb_collectable_; }; } // namespace collection } // namespace sparta From 1c4dba52f43ac96b926c9e45a4be31edbd188576 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Tue, 18 Feb 2025 07:36:46 -0600 Subject: [PATCH 19/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/collection/CollectableTreeNode.hpp | 8 ++++++++ sparta/sparta/collection/PipelineCollector.hpp | 14 ++++++++++---- sparta/sparta/simulation/TreeNode.hpp | 6 ------ 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index 6fcc148296..f0aebde02a 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 6fcc148296194fcf608412500d07ddc9f7c00103 +Subproject commit f0aebde02aaf3b48f330c36c8a8cb17d4a52d55e diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 349de5b47b..2c0a3ac819 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -62,6 +62,14 @@ namespace collection virtual ~CollectableTreeNode() {} + /*! + * \brief The pipeline collector will call this method on all nodes + * as soon as the collector is created. + * + * \return true if we should recurse, false otherwise + */ + virtual void configCollectable(simdb::CollectionMgr *) = 0; + /** * \brief Method that tells this treenode that is now running * collection. diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 0859f1131e..248f6fa792 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -188,9 +188,12 @@ namespace collection continue; } - node->configCollectable(db_mgr_->getCollectionMgr()); - for (auto child : sparta::TreeNodePrivateAttorney::getAllChildren(node)) { - q.push(child); + if (auto ctn = dynamic_cast(node)) { + ctn->configCollectable(db_mgr_->getCollectionMgr()); + } else { + for (auto child : sparta::TreeNodePrivateAttorney::getAllChildren(node)) { + q.push(child); + } } } @@ -417,8 +420,11 @@ namespace collection private: void performSweep_() { + // Note that we subtract one from the current tick + // since this sweep runs in the very last phase + // when the tick has already advanced. auto tick = clk_->getScheduler()->getCurrentTick(); - collection_mgr_->sweep(clk_->getName(), tick); + collection_mgr_->sweep(clk_->getName(), tick - 1); if (!sweepables_.empty()) { ev_sweep_.schedule(); diff --git a/sparta/sparta/simulation/TreeNode.hpp b/sparta/sparta/simulation/TreeNode.hpp index 15a9add2a7..6ce8e7c828 100644 --- a/sparta/sparta/simulation/TreeNode.hpp +++ b/sparta/sparta/simulation/TreeNode.hpp @@ -2173,12 +2173,6 @@ namespace sparta */ virtual void activateLink(const std::string &label); - /*! - * \brief The pipeline collector will call this method on all nodes - * as soon as the collector is created. - */ - virtual void configCollectable(simdb::CollectionMgr *) {} - /*! * \brief Compute a regex pattern for a node child path containing any * number of wildcard characters (not a dot-separated location) which From 63b58ddc90558aa5f8aacf0c86e4cc2bc4b7a2bd Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Tue, 18 Feb 2025 13:59:20 -0600 Subject: [PATCH 20/49] SimDB / v3 integration --- .../sparta/collection/IterableCollector.hpp | 26 ++++++++++++++++--- sparta/sparta/resources/Array.hpp | 3 +++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index d4493849f1..d68de57d47 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -80,7 +80,7 @@ class IterableCollector : public CollectableTreeNode { std::stringstream name_str; name_str << name << i; - positions_.emplace_back(new CollectableT(this, name_str.str(), group, i)); + positions_.emplace_back(new IterableCollectorBin(this, name_str.str(), group, i)); } } @@ -194,7 +194,26 @@ class IterableCollector : public CollectableTreeNode } private: - typedef Collectable::value_type> CollectableT; + class IterableCollectorBin : public CollectableTreeNode + { + public: + IterableCollectorBin(TreeNode* parent, + const std::string& name, + const std::string& group, + const uint32_t bin_idx) + : CollectableTreeNode(parent, name, group, bin_idx, "IterableCollectorBin ") + {} + + void configCollectable(simdb::CollectionMgr *) override final + { + // Nothing to do here + } + + void collect() override final + { + // Nothing to do here + } + }; //! Virtual method called by CollectableTreeNode when collection //! is enabled on the TreeNode @@ -214,7 +233,7 @@ class IterableCollector : public CollectableTreeNode } const IterableType * iterable_object_; - std::vector> positions_; + std::vector> positions_; const size_type expected_capacity_ = 0; using simdb_collectable_type = std::conditional_t simdb_collectable_; }; + } // namespace collection } // namespace sparta diff --git a/sparta/sparta/resources/Array.hpp b/sparta/sparta/resources/Array.hpp index 15e0432351..7775405d2c 100644 --- a/sparta/sparta/resources/Array.hpp +++ b/sparta/sparta/resources/Array.hpp @@ -869,6 +869,9 @@ namespace sparta //! Typedef for size_type typedef uint32_t size_type; + //! Typedef for value_type + typedef DataT value_type; + AgedArrayCollectorProxy(FullArrayType * array) : array_(array) { } typedef FullArrayType::iterator iterator; From c9113689a2e7f5a1e5aed77483e8514a36b990ca Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 19 Feb 2025 07:45:00 -0600 Subject: [PATCH 21/49] SimDB / v3 integration --- sparta/cmake/FindSparta.cmake | 5 +++-- sparta/cmake/sparta-config.cmake | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sparta/cmake/FindSparta.cmake b/sparta/cmake/FindSparta.cmake index 9035f58e0f..48e8c7e742 100644 --- a/sparta/cmake/FindSparta.cmake +++ b/sparta/cmake/FindSparta.cmake @@ -2,6 +2,7 @@ include(FindPackageHandleStandardArgs) if(NOT SPARTA_FOUND) find_package(Boost REQUIRED COMPONENTS timer filesystem serialization program_options) + find_package(HDF5 REQUIRED COMPONENTS CXX) find_package(SQLite3 REQUIRED) find_package(ZLIB REQUIRED) find_package(yaml-cpp REQUIRED) @@ -64,9 +65,9 @@ if(NOT SPARTA_FOUND) get_target_property(YAML_CPP_INCLUDE_DIR yaml-cpp::yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) endif () set_property(TARGET SPARTA::sparta - PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPARTA_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${RAPIDJSON_INCLUDE_DIR} ${RapidJSON_INCLUDE_DIR} ${YAML_CPP_INCLUDE_DIR}) + PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPARTA_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${HDF5_CXX_INCLUDE_DIRS} ${RAPIDJSON_INCLUDE_DIR} ${RapidJSON_INCLUDE_DIR} ${YAML_CPP_INCLUDE_DIR}) set_property(TARGET SPARTA::sparta - PROPERTY INTERFACE_LINK_LIBRARIES SPARTA::libsparta SQLite::SQLite3 + PROPERTY INTERFACE_LINK_LIBRARIES SPARTA::libsparta HDF5::HDF5 SQLite::SQLite3 Boost::filesystem Boost::serialization Boost::timer Boost::program_options ZLIB::ZLIB yaml-cpp Threads::Threads) diff --git a/sparta/cmake/sparta-config.cmake b/sparta/cmake/sparta-config.cmake index 03cb638d3c..e2b234d283 100644 --- a/sparta/cmake/sparta-config.cmake +++ b/sparta/cmake/sparta-config.cmake @@ -74,9 +74,14 @@ find_package(ZLIB REQUIRED) include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) message (STATUS "Using zlib ${ZLIB_VERSION_STRING}") +# Find HDF5. +find_package (HDF5 1.10 REQUIRED COMPONENTS CXX) +include_directories (SYSTEM ${HDF5_INCLUDE_DIRS}) +message (STATUS "Using HDF5 ${HDF5_VERSION}") + # Populate the Sparta_LIBS variable with the required libraries for # basic Sparta linking -set (Sparta_LIBS sparta sqlite3 yaml-cpp ZLIB::ZLIB pthread +set (Sparta_LIBS sparta HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread Boost::date_time Boost::iostreams Boost::serialization Boost::timer Boost::program_options) # On Linux we need to link against rt as well From 4558439a1ea1c42c67c198f1ed879f1d7cd04636 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 19 Feb 2025 08:26:49 -0600 Subject: [PATCH 22/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 30 +++++++++++++++++++ .../CoreModel/src/LoadStoreInstInfo.hpp | 27 +++++++++++++++++ .../example/CoreModel/src/MemAccessInfo.hpp | 28 +++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 0b7d91daa5..1e7c65a8cd 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -6,6 +6,7 @@ #include "sparta/decode/DecoderBase.hpp" #include "sparta/memory/AddressTypes.hpp" #include "sparta/resources/SharedData.hpp" +#include "sparta/pairs/SpartaKeyPairs.hpp" #include "sparta/simulation/State.hpp" #include "sparta/utils/SpartaSharedPointer.hpp" #include "sparta/utils/SpartaSharedPointerAllocator.hpp" @@ -22,9 +23,15 @@ namespace core_example * \brief Example instruction that flows through the example/CoreModel */ + // Forward declaration of the Pair Definition class is must as we are friending it. + class ExampleInstPairDef; + class ExampleInst { public: + // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself + using SpartaPairDefinitionType = ExampleInstPairDef; + enum class Status : std::uint16_t{ FETCHED = 0, __FIRST = FETCHED, @@ -219,6 +226,29 @@ namespace core_example } return os; } + + /*! + * \class ExampleInstPairDef + * \brief Pair Definition class of the Example instruction that flows through the example/CoreModel + */ + // This is the definition of the PairDefinition class of ExampleInst. + // This PairDefinition class could be named anything but it needs to + // inherit publicly from sparta::PairDefinition templatized on the actual class ExampleInst. + class ExampleInstPairDef : public sparta::PairDefinition{ + public: + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + ExampleInstPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(ExampleInst); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &ExampleInst::getUniqueID), + SPARTA_ADDPAIR("uid", &ExampleInst::getUniqueID), + SPARTA_ADDPAIR("mnemonic", &ExampleInst::getMnemonic), + SPARTA_ADDPAIR("complete", &ExampleInst::getCompletedStatus), + SPARTA_ADDPAIR("unit", &ExampleInst::getUnit), + SPARTA_ADDPAIR("latency", &ExampleInst::getExecuteTime), + SPARTA_ADDPAIR("raddr", &ExampleInst::getRAdr, std::ios::hex), + SPARTA_ADDPAIR("vaddr", &ExampleInst::getVAdr, std::ios::hex)) + }; } namespace simdb diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index 15ec4b1265..e0730ce9fe 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -8,10 +8,16 @@ namespace core_example class LoadStoreInstInfo; using LoadStoreInstInfoPtr = sparta::SpartaSharedPointer; + // Forward declaration of the Pair Definition class is must as we are friending it. + class LoadStoreInstInfoPairDef; + // Keep record of instruction issue information class LoadStoreInstInfo { public: + // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself + using SpartaPairDefinitionType = LoadStoreInstInfoPairDef; + enum class IssuePriority : std::uint16_t { HIGHEST = 0, @@ -96,6 +102,27 @@ namespace core_example sparta::State state_; }; // class LoadStoreInstInfo + + /*! + * \class LoadStoreInstInfoPairDef + * \brief Pair Definition class of the load store instruction that flows through the example/CoreModel + */ + // This is the definition of the PairDefinition class of LoadStoreInstInfo. + // This PairDefinition class could be named anything but it needs to inherit + // publicly from sparta::PairDefinition templatized on the actual class LoadStoreInstInfo. + class LoadStoreInstInfoPairDef : public sparta::PairDefinition{ + public: + + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + LoadStoreInstInfoPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(LoadStoreInstInfo); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &LoadStoreInstInfo::getInstUniqueID), + SPARTA_ADDPAIR("rank", &LoadStoreInstInfo::getPriority), + SPARTA_ADDPAIR("state", &LoadStoreInstInfo::getState), + SPARTA_FLATTEN( &LoadStoreInstInfo::getMemoryAccessInfoPtr)) + }; + } // namespace core_example namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index b5b0d78031..af8ac5cf86 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -8,10 +8,16 @@ namespace core_example class MemoryAccessInfo; using MemoryAccessInfoPtr = sparta::SpartaSharedPointer; + // Forward declaration of the Pair Definition class is must as we are friending it. + class MemoryAccessInfoPairDef; + // Keep record of memory access information in LSU class MemoryAccessInfo { public: + // The modeler needs to alias a type called "SpartaPairDefinitionType" to the Pair Definition class of itself + using SpartaPairDefinitionType = MemoryAccessInfoPairDef; + enum class MMUState : std::uint32_t { NO_ACCESS = 0, __FIRST = NO_ACCESS, @@ -127,6 +133,28 @@ namespace core_example return os; } + /*! + * \class MemoryAccessInfoPairDef + * \brief Pair Definition class of the Memory Access Information that flows through the example/CoreModel + */ + + // This is the definition of the PairDefinition class of MemoryAccessInfo. + // This PairDefinition class could be named anything but it needs to inherit + // publicly from sparta::PairDefinition templatized on the actual class MemoryAcccessInfo. + class MemoryAccessInfoPairDef : public sparta::PairDefinition{ + public: + + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + MemoryAccessInfoPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(MemoryAccessInfo); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("DID", &MemoryAccessInfo::getInstUniqueID), + SPARTA_ADDPAIR("valid", &MemoryAccessInfo::getPhyAddrIsReady), + SPARTA_ADDPAIR("mmu", &MemoryAccessInfo::getMMUState), + SPARTA_ADDPAIR("cache", &MemoryAccessInfo::getCacheState), + SPARTA_FLATTEN( &MemoryAccessInfo::getInstPtr)) + }; + } // namespace core_example namespace simdb From f12adf9bb04a688285d5e23521f683fd027eb41f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 19 Feb 2025 08:36:52 -0600 Subject: [PATCH 23/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 4 +-- .../DynamicModelPipeline/src/ExampleInst.hpp | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 1e7c65a8cd..85e02f06cd 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -235,8 +235,8 @@ namespace core_example // This PairDefinition class could be named anything but it needs to // inherit publicly from sparta::PairDefinition templatized on the actual class ExampleInst. class ExampleInstPairDef : public sparta::PairDefinition{ - public: - // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + public: + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class ExampleInstPairDef() : PairDefinition(){ SPARTA_INVOKE_PAIRS(ExampleInst); } diff --git a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp index 3248b2a1ed..38835e96c8 100644 --- a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp +++ b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp @@ -23,8 +23,15 @@ namespace core_example * \brief Example instruction that flows through the example/CoreModel */ + // Forward declaration of the Pair Definition class is must as we are friending it. + class ExampleInstPairDef; + class ExampleInst { public: + + // The modeler needs to alias a type called "type" to the Pair Definition class of itself + using type = ExampleInstPairDef; + enum class Status : std::uint16_t{ FETCHED = 0, __FIRST = FETCHED, @@ -209,4 +216,27 @@ namespace core_example } return os; } + + /*! + * \class ExampleInstPairDef + * \brief Pair Definition class of the Example instruction that flows through the example/CoreModel + */ + // This is the definition of the PairDefinition class of ExampleInst. + // This PairDefinition class could be named anything but it needs to + // inherit publicly from sparta::PairDefinition templatized on the actual class ExampleInst. + class ExampleInstPairDef : public sparta::PairDefinition{ + public: + + // The SPARTA_ADDPAIRs APIs must be called during the construction of the PairDefinition class + ExampleInstPairDef() : PairDefinition(){ + SPARTA_INVOKE_PAIRS(ExampleInst); + } + SPARTA_REGISTER_PAIRS(SPARTA_ADDPAIR("mnemonic", &ExampleInst::getMnemonicAsString), + SPARTA_ADDPAIR("complete", &ExampleInst::getCompletedStatus), + SPARTA_ADDPAIR("unit", &ExampleInst::getUnit), + SPARTA_ADDPAIR("latency", &ExampleInst::getExecuteTime), + SPARTA_ADDPAIR("uid", &ExampleInst::getUniqueID), + SPARTA_ADDPAIR("raddr", &ExampleInst::getRAdr, std::ios::hex), + SPARTA_ADDPAIR("vaddr", &ExampleInst::getVAdr, std::ios::hex)); + }; } From 82425f79f5c88a81898fc5d08cfb9fea6c864e91 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 19 Feb 2025 10:48:28 -0600 Subject: [PATCH 24/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/collection/CollectableTreeNode.hpp | 1 + sparta/sparta/kernel/Scheduler.hpp | 4 ---- sparta/sparta/simulation/TreeNode.hpp | 4 ---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index f0aebde02a..54bbe82fb6 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit f0aebde02aaf3b48f330c36c8a8cb17d4a52d55e +Subproject commit 54bbe82fb6e423577667d87f06c03b0e08956d2e diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 2c0a3ac819..81db1ffe56 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -14,6 +14,7 @@ namespace simdb { class DatabaseManager; + class CollectionMgr; } namespace sparta{ diff --git a/sparta/sparta/kernel/Scheduler.hpp b/sparta/sparta/kernel/Scheduler.hpp index 7379afddd0..24d7d84590 100644 --- a/sparta/sparta/kernel/Scheduler.hpp +++ b/sparta/sparta/kernel/Scheduler.hpp @@ -45,10 +45,6 @@ namespace sparta { class GlobalTreeNode; - -namespace collection { - class PipelineCollector; -} } // namespace sparta diff --git a/sparta/sparta/simulation/TreeNode.hpp b/sparta/sparta/simulation/TreeNode.hpp index 6ce8e7c828..73b76f5e12 100644 --- a/sparta/sparta/simulation/TreeNode.hpp +++ b/sparta/sparta/simulation/TreeNode.hpp @@ -40,10 +40,6 @@ namespace sparta { class PostRunValidationInfo; } // namespace sparta -namespace simdb { -class CollectionMgr; -} // namespace simdb - #ifndef TREENODE_LIFETIME_TRACE /*! * \brief Enables tracing of TreeNode lifetimes in a set of output txt files. From 305fe3ba4a9359a1e17b81175c82756442892c14 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Wed, 19 Feb 2025 10:59:16 -0600 Subject: [PATCH 25/49] SimDB / v3 integration --- sparta/sparta/collection/CollectableTreeNode.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 81db1ffe56..3aee7c184c 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -21,6 +21,8 @@ namespace sparta{ namespace collection { + class PipelineCollector; + /** * \class CollectableTreeNode * \brief An abstract type of TreeNode that From ac80d7531d2c1dd82e24bae339178c9715ef2588 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Thu, 20 Feb 2025 16:44:07 -0600 Subject: [PATCH 26/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/collection/PipelineCollector.hpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index 54bbe82fb6..89e79490f2 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 54bbe82fb6e423577667d87f06c03b0e08956d2e +Subproject commit 89e79490f2dd741353546c687c3865ef7a398fe1 diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 248f6fa792..444d09d77d 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -221,6 +221,8 @@ namespace collection */ void destroy() { + db_mgr_->postSim(); + if(collection_active_) { for(auto & ctn : registered_collectables_) { if(ctn->isCollected()) { @@ -228,6 +230,7 @@ namespace collection } } } + registered_collectables_.clear(); collection_active_ = false; } From 18b09224712dbd59382959edb5b333c7c73bf500 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 21 Feb 2025 07:45:19 -0600 Subject: [PATCH 27/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index 89e79490f2..5718f82f81 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 89e79490f2dd741353546c687c3865ef7a398fe1 +Subproject commit 5718f82f818217acec4e5a2454eddd0e27cb243c From db8bcc267ba74bac57a92c87cfebd8d943ba17f3 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 21 Feb 2025 08:33:11 -0600 Subject: [PATCH 28/49] SimDB / v3 integration --- .../sparta/collection/IterableCollector.hpp | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index d68de57d47..9d19867cc9 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -170,6 +170,44 @@ class IterableCollector : public CollectableTreeNode // Delegated constructor } + /** + * \brief constructor with no iterable object associated + * \param parent the parent treenode for the collector + * \param name the name of the collector + * \param expected_capacity The maximum size this item should grow to + */ + IterableCollector (TreeNode * parent, + const std::string & name, + const size_type expected_capacity) : + IterableCollector (parent, name, name + " Iterable Collector", + nullptr, expected_capacity) + { + // Can't auto collect without setting iterable_object_ + setManualCollection(); + } + + //! \brief Do not perform any automatic collection + //! The SchedulingPhase is ignored + void setManualCollection() { + auto_collect_ = false; + } + + //! Collect the contents of the iterable object. This function + //! will walk starting from index 0 -> expected_capacity, clearing + //! out any records where the iterable object does not contain + //! data. + void collect(const IterableType * iterable_object) + { + // If pointer has become nullified, close the records + if(nullptr == iterable_object) { + closeRecord(); + } + else if (SPARTA_EXPECT_TRUE(isCollected())) + { + simdb_collectable_->activate(iterable_object); + } + } + //! Collect the contents of the associated iterable object void collect() override { if(SPARTA_EXPECT_TRUE(isCollected())) { @@ -218,7 +256,7 @@ class IterableCollector : public CollectableTreeNode //! Virtual method called by CollectableTreeNode when collection //! is enabled on the TreeNode void setCollecting_(bool collect, PipelineCollector* collector, simdb::DatabaseManager* db_mgr) override { - if (collect) { + if (collect && auto_collect_) { // Add this Collectable to the PipelineCollector's // list of objects requiring collection collector->addToAutoCollection(this, collection_phase); @@ -227,7 +265,7 @@ class IterableCollector : public CollectableTreeNode // once-a-cycle sweep() method. // // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). - collector->removeFromAutoSweep(this); + collector->removeFromAutoCollection(this); closeRecord(); } } @@ -235,6 +273,7 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable_object_; std::vector> positions_; const size_type expected_capacity_ = 0; + bool auto_collect_ = false; using simdb_collectable_type = std::conditional_t Date: Fri, 21 Feb 2025 09:09:26 -0600 Subject: [PATCH 29/49] SimDB / v3 integration --- sparta/CMakeLists.txt | 1 - .../src/SimpleAnnotationOutputter.hpp | 3 - sparta/sparta/app/CommandLineSimulator.hpp | 1 - sparta/sparta/app/Simulation.hpp | 1 - .../sparta/collection/DelayedCollectable.hpp | 1 - sparta/src/ArgosOutputter.cpp | 73 ------------------- sparta/src/CommandLineSimulator.cpp | 1 - 7 files changed, 81 deletions(-) delete mode 100644 sparta/src/ArgosOutputter.cpp diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index 084a61b82f..0676874347 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -12,7 +12,6 @@ include (${SPARTA_BASE}/cmake/sparta-config.cmake) # Add the source for libsparta.a list (APPEND SourceCppFiles - src/ArgosOutputter.cpp src/Backtrace.cpp src/BaseFormatter.cpp src/Clock.cpp diff --git a/sparta/example/SimpleAnnotationWriter/src/SimpleAnnotationOutputter.hpp b/sparta/example/SimpleAnnotationWriter/src/SimpleAnnotationOutputter.hpp index 171de9e5df..265f48df05 100644 --- a/sparta/example/SimpleAnnotationWriter/src/SimpleAnnotationOutputter.hpp +++ b/sparta/example/SimpleAnnotationWriter/src/SimpleAnnotationOutputter.hpp @@ -3,9 +3,6 @@ #include #include -#include "sparta/pipeViewer/Outputter.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" - class SimpleAnnotationOutputter { private: static constexpr uint64_t ROOT_CLOCK_ID = 1; diff --git a/sparta/sparta/app/CommandLineSimulator.hpp b/sparta/sparta/app/CommandLineSimulator.hpp index b7a9d57b91..a2eebe7841 100644 --- a/sparta/sparta/app/CommandLineSimulator.hpp +++ b/sparta/sparta/app/CommandLineSimulator.hpp @@ -455,7 +455,6 @@ class CommandLineSimulator */ std::unique_ptr pipeline_collection_triggerable_; std::unique_ptr pipeline_trigger_; - std::unique_ptr info_out_; /*! * \brief Heartbeat period of pipeline collection file. diff --git a/sparta/sparta/app/Simulation.hpp b/sparta/sparta/app/Simulation.hpp index f058afe37f..17424811dc 100644 --- a/sparta/sparta/app/Simulation.hpp +++ b/sparta/sparta/app/Simulation.hpp @@ -24,7 +24,6 @@ #include "sparta/app/SimulationConfiguration.hpp" #include "sparta/control/TemporaryRunController.hpp" #include "sparta/simulation/State.hpp" -#include "sparta/pipeViewer/InformationWriter.hpp" namespace YP = YAML; // Prevent collision with YAML class in ConfigParser namespace. diff --git a/sparta/sparta/collection/DelayedCollectable.hpp b/sparta/sparta/collection/DelayedCollectable.hpp index 9287b00f00..0be29d4af7 100644 --- a/sparta/sparta/collection/DelayedCollectable.hpp +++ b/sparta/sparta/collection/DelayedCollectable.hpp @@ -12,7 +12,6 @@ #include #include #include "sparta/collection/Collectable.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" #include "sparta/events/EventSet.hpp" #include "sparta/events/PayloadEvent.hpp" diff --git a/sparta/src/ArgosOutputter.cpp b/sparta/src/ArgosOutputter.cpp deleted file mode 100644 index 108a127f98..0000000000 --- a/sparta/src/ArgosOutputter.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// -*- C++ -*- - -/** - * \file Outputter - * \brief Outputs Transactions to record file and builds index file while running - * - */ -#include -#include -#include -#include - -#include "sparta/pipeViewer/Outputter.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" - -// #define PIPELINE_DBG 1 -namespace sparta::pipeViewer -{ - Outputter::Outputter(const std::string& filepath, const uint64_t interval) : - record_file_(filepath + "record.bin", std::fstream::out | std::fstream::binary), - index_file_(filepath + "index.bin", std::fstream::out | std::fstream::binary), - map_file_(filepath + "map.dat", std::ios::out), - data_file_(filepath + "data.dat", std::ios::out), - string_file_(filepath + "string_map.dat", std::ios::out), - display_format_file_(filepath + "display_format.dat", std::ios::out), - last_record_pos_(0) - { - // Make sure the files opened correctly! - sparta_assert(index_file_.is_open() && record_file_.is_open(), - "Failed to open the path to write pipeline collection files." - " It may be possible that the directory does not exist." - " at filepath: FILEPATH+PREFIX=" << filepath); - // Throw on write failure - record_file_.exceptions(std::ostream::eofbit | std::ostream::badbit | std::ostream::failbit | std::ostream::goodbit); - index_file_.exceptions(std::ostream::eofbit | std::ostream::badbit | std::ostream::failbit | std::ostream::goodbit); - // Write the index file version first. The index file should naturally skip this - // Do not 0-pad the version number - it will be cast from a string to an int - std::ostringstream t; - t << HEADER_PREFIX << std::setw(VERSION_LENGTH) << FILE_VERSION << '\n'; - const std::string header = t.str(); - sparta_assert(header.size() == HEADER_SIZE); // Do not change the header format - static_assert(sizeof(transaction_t) == 48, - "size of a transaction changed. May want to increase file format version. " - "If no changes were made to SPARTA, compiler is generating a structure " - "layout that is incompatible with pipeViewer"); - writeData_(index_file_, header.data(), header.size()); - // Notice that we write the interval offset first. - writeData_(index_file_, interval); - index_file_.flush(); - } - Outputter::~Outputter(){ - //Write an index for the end of the record file, so that the last record - //is always easily accessable reguardless of indexing. - writeData_(index_file_, last_record_pos_); - //std::cout << "Writing Last Record Position " << last_record_pos_ << std::endl; - //std::cout << "The Pipeline Collection is closing the output stream" - // <<" this may take some time." << std::endl; - index_file_.close(); - record_file_.close(); - map_file_.close(); - data_file_.close(); - string_file_.close(); - display_format_file_.close(); - std::cout << "The outputter is done destructing." << std::endl; - } - void Outputter::writeIndex(){ - const uint64_t current_record_pos = record_file_.tellp(); - writeData_(index_file_, current_record_pos); - index_file_.flush(); - } -}//namespace sparta::pipeViewer diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index c4d2ea6c46..6a126c4eb2 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -58,7 +58,6 @@ #include "sparta/app/AppTriggers.hpp" #include "sparta/app/MetaTreeNode.hpp" #include "sparta/app/Simulation.hpp" -#include "sparta/pipeViewer/InformationWriter.hpp" #include "sparta/log/Destination.hpp" #include "sparta/log/MessageSource.hpp" #include "sparta/log/Tap.hpp" From 2d9333c8ad5ea5150377dd49fc7643fec4afde0f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 21 Feb 2025 09:58:37 -0600 Subject: [PATCH 30/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/collection/Collectable.hpp | 3 +- .../sparta/collection/IterableCollector.hpp | 78 +++++++++++++++---- 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index 5718f82f81..b8615fa5dc 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 5718f82f818217acec4e5a2454eddd0e27cb243c +Subproject commit b8615fa5dc46c275f9f4b82074c6c08ad095f4f5 diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index b1f49760c1..cbbe220695 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -112,7 +112,8 @@ namespace sparta{ { if(SPARTA_EXPECT_TRUE(isCollected())) { - simdb_collectable_->activate(val); + const bool once = !auto_collect_; + simdb_collectable_->activate(val, once); } } diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index 9d19867cc9..23b986eedf 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -70,6 +70,9 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable, const size_type expected_capacity) : CollectableTreeNode(parent, name, group, index, desc), + event_set_(this), + ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", + CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)), iterable_object_(iterable), expected_capacity_(expected_capacity) { @@ -102,7 +105,9 @@ class IterableCollector : public CollectableTreeNode const IterableType & iterable, const size_type expected_capacity) : IterableCollector(parent, name, group, index, desc, &iterable, expected_capacity) - {} + { + // Delegated constructor + } /** * \brief constructor @@ -118,7 +123,9 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable, const size_type expected_capacity) : IterableCollector(parent, name, name, 0, desc, iterable, expected_capacity) - {} + { + // Delegated constructor + } /** * \brief constructor @@ -134,7 +141,9 @@ class IterableCollector : public CollectableTreeNode const IterableType & iterable, const size_type expected_capacity) : IterableCollector(parent, name, name, 0, desc, &iterable, expected_capacity) - {} + { + // Delegated constructor + } /** * \brief constructor with no description @@ -192,6 +201,17 @@ class IterableCollector : public CollectableTreeNode auto_collect_ = false; } + //! \brief Perform a collection, then close the records in the future + //! \param duration The time to close the records, 0 is not allowed + void collectWithDuration(sparta::Clock::Cycle duration) { + if(SPARTA_EXPECT_FALSE(isCollected())) { + collect(); + if(duration != 0) { + ev_close_record_.preparePayload(false)->schedule(duration); + } + } + } + //! Collect the contents of the iterable object. This function //! will walk starting from index 0 -> expected_capacity, clearing //! out any records where the iterable object does not contain @@ -204,7 +224,8 @@ class IterableCollector : public CollectableTreeNode } else if (SPARTA_EXPECT_TRUE(isCollected())) { - simdb_collectable_->activate(iterable_object); + const bool once = !auto_collect_; + simdb_collectable_->activate(iterable_object, once); } } @@ -220,6 +241,13 @@ class IterableCollector : public CollectableTreeNode iterable_object_ = obj; } + //! Force close all records for this iterable type. This will + //! close the record immediately and clear the field for the next + //! cycle + void closeRecord(const bool & simulation_ending = false) override { + simdb_collectable_->deactivate(); + } + /*! * \brief The pipeline collector will call this method on all nodes * as soon as the collector is created. @@ -256,20 +284,42 @@ class IterableCollector : public CollectableTreeNode //! Virtual method called by CollectableTreeNode when collection //! is enabled on the TreeNode void setCollecting_(bool collect, PipelineCollector* collector, simdb::DatabaseManager* db_mgr) override { - if (collect && auto_collect_) { - // Add this Collectable to the PipelineCollector's - // list of objects requiring collection - collector->addToAutoCollection(this, collection_phase); + if (iterable_object_ && auto_collect_) { + if (collect) { + // Add this Collectable to the PipelineCollector's + // list of objects requiring collection + collector->addToAutoCollection(this, collection_phase); + } else { + // If we are no longer collecting, remove this Collectable from the + // once-a-cycle sweep() method. + // + // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). + collector->removeFromAutoCollection(this); + closeRecord(); + } } else { - // If we are no longer collecting, remove this Collectable from the - // once-a-cycle sweep() method. - // - // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). - collector->removeFromAutoCollection(this); - closeRecord(); + if (collect) { + // If we are manually collecting, we still need to tell the collector + // to run the sweep() method every cycle on our clock. + // + // Note that addToAutoCollection() implicitly calls addToAutoSweep(). + collector->addToAutoSweep(this); + } else { + // If we are no longer collecting, remove this Collectable from the + // once-a-cycle sweep() method. + // + // Note that removeFromAutoCollection() implicitly calls removeFromAutoSweep(). + collector->removeFromAutoSweep(this); + closeRecord(); + } } } + // For those folks that want a value to automatically + // disappear in the future + sparta::EventSet event_set_; + sparta::PayloadEvent ev_close_record_; + const IterableType * iterable_object_; std::vector> positions_; const size_type expected_capacity_ = 0; From b13e8b3aa7e5b349f8c106e94d107f8fa6aead10 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 21 Feb 2025 10:18:05 -0600 Subject: [PATCH 31/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index b8615fa5dc..ab4b97a32c 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit b8615fa5dc46c275f9f4b82074c6c08ad095f4f5 +Subproject commit ab4b97a32ca71185f982d3e3b0b9a8fbcb1818d2 From 47d2f6ea2f32131da630602813c6562aaf47754c Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Fri, 21 Feb 2025 10:42:52 -0600 Subject: [PATCH 32/49] SimDB / v3 integration --- sparta/sparta/collection/IterableCollector.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index 23b986eedf..927ae09979 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -76,9 +76,6 @@ class IterableCollector : public CollectableTreeNode iterable_object_(iterable), expected_capacity_(expected_capacity) { - sparta_assert(iterable_object_ != nullptr, - "IterableCollector " << getLocation() << " cannot be constructed with a nullptr iterable object"); - for (size_type i = 0; i < expected_capacity_; ++i) { std::stringstream name_str; From 72f3cc40a4dd4dba73167c9eca87bd16693f156f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 22 Feb 2025 08:22:10 -0600 Subject: [PATCH 33/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 10 +++++----- sparta/example/CoreModel/src/LoadStoreInstInfo.hpp | 4 ++-- sparta/example/CoreModel/src/MemAccessInfo.hpp | 6 +++--- sparta/simdb | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index 85e02f06cd..bfed45dd35 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -275,12 +275,12 @@ inline void defineStructSchema(StructSchema("DID"); schema.addField("uid"); - schema.addField("mnemonic"); - schema.addBoolField("complete"); - schema.addField("unit"); + schema.addString("mnemonic"); + schema.addBool("complete"); + schema.addEnum("unit"); schema.addField("latency"); - schema.addHexField("raddr"); - schema.addHexField("vaddr"); + schema.addHex("raddr"); + schema.addHex("vaddr"); schema.setAutoColorizeColumn("DID"); } diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index e0730ce9fe..bb9564f301 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -212,8 +212,8 @@ template <> inline void defineStructSchema(StructSchema& schema) { schema.addField("DID"); - schema.addField("rank"); - schema.addField("state"); + schema.addEnum("rank"); + schema.addEnum("state"); schema.setAutoColorizeColumn("DID"); } diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index af8ac5cf86..9d6c872167 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -186,9 +186,9 @@ template <> inline void defineStructSchema(StructSchema& schema) { schema.addField("DID"); - schema.addBoolField("valid"); - schema.addField("mmu"); - schema.addField("cache"); + schema.addBool("valid"); + schema.addEnum("mmu"); + schema.addEnum("cache"); schema.setAutoColorizeColumn("DID"); } diff --git a/sparta/simdb b/sparta/simdb index ab4b97a32c..9dd7d3e9ca 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit ab4b97a32ca71185f982d3e3b0b9a8fbcb1818d2 +Subproject commit 9dd7d3e9ca7f7a201cd821511b3e4e6ec2184df7 From 51af082cffbd8fec39dc740ff72e044a9c806477 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 22 Feb 2025 08:26:52 -0600 Subject: [PATCH 34/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 1 - sparta/example/CoreModel/src/LoadStoreInstInfo.hpp | 1 - sparta/example/CoreModel/src/MemAccessInfo.hpp | 1 - sparta/simdb | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index bfed45dd35..dd04e2abb3 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -281,7 +281,6 @@ inline void defineStructSchema(StructSchema("latency"); schema.addHex("raddr"); schema.addHex("vaddr"); - schema.setAutoColorizeColumn("DID"); } template <> diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index bb9564f301..c67c5c24c9 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -214,7 +214,6 @@ inline void defineStructSchema(StructSchema("DID"); schema.addEnum("rank"); schema.addEnum("state"); - schema.setAutoColorizeColumn("DID"); } template <> diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index 9d6c872167..dce0f8a51f 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -189,7 +189,6 @@ inline void defineStructSchema(StructSchema("mmu"); schema.addEnum("cache"); - schema.setAutoColorizeColumn("DID"); } template <> diff --git a/sparta/simdb b/sparta/simdb index 9dd7d3e9ca..21237a275a 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 9dd7d3e9ca7f7a201cd821511b3e4e6ec2184df7 +Subproject commit 21237a275a572304c2051fb6a07e71525d7b4f86 From 18778f067f5afa5f10c44ceeea85b4ce13dfd8a9 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 22 Feb 2025 08:32:48 -0600 Subject: [PATCH 35/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 13 ++++++----- .../CoreModel/src/LoadStoreInstInfo.hpp | 22 ++++++++++--------- .../example/CoreModel/src/MemAccessInfo.hpp | 14 +++++++----- sparta/simdb | 2 +- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index dd04e2abb3..fa59044fab 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -258,14 +258,15 @@ template <> inline void defineEnumMap(std::string& enum_name, std::map& map) { using TargetUnit = core_example::ExampleInst::TargetUnit; + using etype = std::underlying_type::type; enum_name = "TargetUnit"; - map["ALU0"] = static_cast(TargetUnit::ALU0); - map["ALU1"] = static_cast(TargetUnit::ALU1); - map["FPU"] = static_cast(TargetUnit::FPU); - map["BR"] = static_cast(TargetUnit::BR); - map["LSU"] = static_cast(TargetUnit::LSU); - map["ROB"] = static_cast(TargetUnit::ROB); + map["ALU0"] = static_cast(TargetUnit::ALU0); + map["ALU1"] = static_cast(TargetUnit::ALU1); + map["FPU"] = static_cast(TargetUnit::FPU); + map["BR"] = static_cast(TargetUnit::BR); + map["LSU"] = static_cast(TargetUnit::LSU); + map["ROB"] = static_cast(TargetUnit::ROB); } template <> diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index c67c5c24c9..3d5646befe 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -132,26 +132,28 @@ template <> inline void defineEnumMap(std::string& enum_name, std::map& map) { using IssuePriority = core_example::LoadStoreInstInfo::IssuePriority; + using etype = std::underlying_type::type; enum_name = "IssuePriority"; - map["HIGHEST"] = static_cast(IssuePriority::HIGHEST); - map["CACHE_RELOAD"] = static_cast(IssuePriority::CACHE_RELOAD); - map["CACHE_PENDING"] = static_cast(IssuePriority::CACHE_PENDING); - map["MMU_RELOAD"] = static_cast(IssuePriority::MMU_RELOAD); - map["MMU_PENDING"] = static_cast(IssuePriority::MMU_PENDING); - map["NEW_DISP"] = static_cast(IssuePriority::NEW_DISP); - map["LOWEST"] = static_cast(IssuePriority::LOWEST); + map["HIGHEST"] = static_cast(IssuePriority::HIGHEST); + map["CACHE_RELOAD"] = static_cast(IssuePriority::CACHE_RELOAD); + map["CACHE_PENDING"] = static_cast(IssuePriority::CACHE_PENDING); + map["MMU_RELOAD"] = static_cast(IssuePriority::MMU_RELOAD); + map["MMU_PENDING"] = static_cast(IssuePriority::MMU_PENDING); + map["NEW_DISP"] = static_cast(IssuePriority::NEW_DISP); + map["LOWEST"] = static_cast(IssuePriority::LOWEST); } template <> inline void defineEnumMap(std::string& enum_name, std::map& map) { using IssueState = core_example::LoadStoreInstInfo::IssueState; + using etype = std::underlying_type::type; enum_name = "IssueState"; - map["READY"] = static_cast(IssueState::READY); - map["ISSUED"] = static_cast(IssueState::ISSUED); - map["NOT_READY"] = static_cast(IssueState::NOT_READY); + map["READY"] = static_cast(IssueState::READY); + map["ISSUED"] = static_cast(IssueState::ISSUED); + map["NOT_READY"] = static_cast(IssueState::NOT_READY); } } // namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index dce0f8a51f..17354a5d3e 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -164,22 +164,24 @@ template <> inline void defineEnumMap(std::string& enum_name, std::map& map) { using MMUState = core_example::MemoryAccessInfo::MMUState; + using etype = std::underlying_type::type; enum_name = "MMUState"; - map["NoAccess"] = static_cast(MMUState::NO_ACCESS); - map["Miss"] = static_cast(MMUState::MISS); - map["Hit"] = static_cast(MMUState::HIT); + map["NoAccess"] = static_cast(MMUState::NO_ACCESS); + map["Miss"] = static_cast(MMUState::MISS); + map["Hit"] = static_cast(MMUState::HIT); } template <> inline void defineEnumMap(std::string& enum_name, std::map& map) { using CacheState = core_example::MemoryAccessInfo::CacheState; + using etype = std::underlying_type::type; enum_name = "CacheState"; - map["NoAccess"] = static_cast(CacheState::NO_ACCESS); - map["Miss"] = static_cast(CacheState::MISS); - map["Hit"] = static_cast(CacheState::HIT); + map["NoAccess"] = static_cast(CacheState::NO_ACCESS); + map["Miss"] = static_cast(CacheState::MISS); + map["Hit"] = static_cast(CacheState::HIT); } template <> diff --git a/sparta/simdb b/sparta/simdb index 21237a275a..e5be0cc8bc 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 21237a275a572304c2051fb6a07e71525d7b4f86 +Subproject commit e5be0cc8bcb6e537e2ff1f4fd92c633210592a36 From 3ec7c1aed16ba66d13c0efd6b09eaa2dada25f2e Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 22 Feb 2025 08:40:59 -0600 Subject: [PATCH 36/49] SimDB / v3 integration --- sparta/example/CoreModel/src/ExampleInst.hpp | 14 +++++++------- sparta/example/CoreModel/src/LoadStoreInstInfo.hpp | 6 +++--- sparta/example/CoreModel/src/MemAccessInfo.hpp | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sparta/example/CoreModel/src/ExampleInst.hpp b/sparta/example/CoreModel/src/ExampleInst.hpp index fa59044fab..69b9f34a81 100644 --- a/sparta/example/CoreModel/src/ExampleInst.hpp +++ b/sparta/example/CoreModel/src/ExampleInst.hpp @@ -287,14 +287,14 @@ inline void defineStructSchema(StructSchema inline void writeStructFields(const core_example::ExampleInst* inst, StructFieldSerializer* serializer) { - serializer->writeField(inst->getUniqueID()); - serializer->writeField(inst->getUniqueID()); + serializer->writeField(inst->getUniqueID()); + serializer->writeField(inst->getUniqueID()); serializer->writeField(inst->getMnemonic()); - serializer->writeField(inst->getCompletedStatus()); - serializer->writeField(inst->getUnit()); - serializer->writeField(inst->getExecuteTime()); - serializer->writeField(inst->getRAdr()); - serializer->writeField(inst->getVAdr()); + serializer->writeField(inst->getCompletedStatus()); + serializer->writeField(inst->getUnit()); + serializer->writeField(inst->getExecuteTime()); + serializer->writeField(inst->getRAdr()); + serializer->writeField(inst->getVAdr()); } } // namespace simdb diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index 3d5646befe..cae371e334 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -223,9 +223,9 @@ inline void writeStructFields( const core_example::LoadStoreInstInfo* inst, StructFieldSerializer* serializer) { - serializer->writeField(inst->getInstUniqueID()); - serializer->writeField(inst->getPriority()); - serializer->writeField(inst->getState()); + serializer->writeField(inst->getInstUniqueID()); + serializer->writeField(inst->getPriority()); + serializer->writeField(inst->getState()); } } // namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index 17354a5d3e..7ed61d3981 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -198,10 +198,10 @@ inline void writeStructFields( const core_example::MemoryAccessInfo* inst, StructFieldSerializer* serializer) { - serializer->writeField(inst->getInstUniqueID()); - serializer->writeField(inst->getPhyAddrIsReady()); - serializer->writeField(inst->getMMUState()); - serializer->writeField(inst->getCacheState()); + serializer->writeField(inst->getInstUniqueID()); + serializer->writeField(inst->getPhyAddrIsReady()); + serializer->writeField(inst->getMMUState()); + serializer->writeField(inst->getCacheState()); } } // namespace simdb From 222a489ed9da11a09057ee77f62d151957dda445 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 22 Feb 2025 10:42:53 -0600 Subject: [PATCH 37/49] SimDB / v3 integration --- sparta/sparta/collection/Collectable.hpp | 1 + sparta/sparta/collection/IterableCollector.hpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index cbbe220695..9af665b616 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -83,6 +83,7 @@ namespace sparta{ desc) { collected_object_ = collected_object; + auto_collect_ = collected_object_ != nullptr; } /** diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index 927ae09979..91bc3bcaa7 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -82,6 +82,10 @@ class IterableCollector : public CollectableTreeNode name_str << name << i; positions_.emplace_back(new IterableCollectorBin(this, name_str.str(), group, i)); } + + if (iterable_object_) { + auto_collect_ = true; + } } /** From 5d33f9e088b8dd199cfabc4f9259d26f6d7a7f14 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 10:27:22 -0600 Subject: [PATCH 38/49] SimDB / v3 integration --- sparta/CMakeLists.txt | 6 +++++- sparta/cmake/sparta-config.cmake | 2 +- sparta/simdb | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index 0676874347..5f4fed913a 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -206,11 +206,15 @@ add_library (sparta ${SourceCppFiles}) # Add local includes target_include_directories (sparta PUBLIC "./") -target_include_directories (sparta PUBLIC "simdb/include") set (SPARTA_STATIC_LIBS ${PROJECT_BINARY_DIR}/libsparta.a) set (SPARTA_CMAKE_MACRO_PATH ${SPARTA_BASE}/cmake) +# Add SimDB (submodule) +add_subdirectory(simdb) +target_include_directories (sparta PUBLIC "simdb/include") +target_link_libraries(sparta simdb) + # # Testing, examples, and tools # diff --git a/sparta/cmake/sparta-config.cmake b/sparta/cmake/sparta-config.cmake index e2b234d283..653129639d 100644 --- a/sparta/cmake/sparta-config.cmake +++ b/sparta/cmake/sparta-config.cmake @@ -81,7 +81,7 @@ message (STATUS "Using HDF5 ${HDF5_VERSION}") # Populate the Sparta_LIBS variable with the required libraries for # basic Sparta linking -set (Sparta_LIBS sparta HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread +set (Sparta_LIBS sparta simdb HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread Boost::date_time Boost::iostreams Boost::serialization Boost::timer Boost::program_options) # On Linux we need to link against rt as well diff --git a/sparta/simdb b/sparta/simdb index e5be0cc8bc..3e879e9ff1 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit e5be0cc8bcb6e537e2ff1f4fd92c633210592a36 +Subproject commit 3e879e9ff197f0d46ea3ab6e84abc06417125e8c From c924f88f26638130c52d709731cf43ac932a91dd Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 11:17:08 -0600 Subject: [PATCH 39/49] Make SimDB header-only again --- sparta/CMakeLists.txt | 6 +----- sparta/cmake/sparta-config.cmake | 2 +- sparta/simdb | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index 5f4fed913a..0676874347 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -206,15 +206,11 @@ add_library (sparta ${SourceCppFiles}) # Add local includes target_include_directories (sparta PUBLIC "./") +target_include_directories (sparta PUBLIC "simdb/include") set (SPARTA_STATIC_LIBS ${PROJECT_BINARY_DIR}/libsparta.a) set (SPARTA_CMAKE_MACRO_PATH ${SPARTA_BASE}/cmake) -# Add SimDB (submodule) -add_subdirectory(simdb) -target_include_directories (sparta PUBLIC "simdb/include") -target_link_libraries(sparta simdb) - # # Testing, examples, and tools # diff --git a/sparta/cmake/sparta-config.cmake b/sparta/cmake/sparta-config.cmake index 653129639d..e2b234d283 100644 --- a/sparta/cmake/sparta-config.cmake +++ b/sparta/cmake/sparta-config.cmake @@ -81,7 +81,7 @@ message (STATUS "Using HDF5 ${HDF5_VERSION}") # Populate the Sparta_LIBS variable with the required libraries for # basic Sparta linking -set (Sparta_LIBS sparta simdb HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread +set (Sparta_LIBS sparta HDF5::HDF5 sqlite3 yaml-cpp ZLIB::ZLIB pthread Boost::date_time Boost::iostreams Boost::serialization Boost::timer Boost::program_options) # On Linux we need to link against rt as well diff --git a/sparta/simdb b/sparta/simdb index 3e879e9ff1..7cc3648fb1 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 3e879e9ff197f0d46ea3ab6e84abc06417125e8c +Subproject commit 7cc3648fb1da146b846a3d8d28d43d70be476ded From b60b6ac550e680e8d151e8d69acc46d05d424592 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 14:31:27 -0600 Subject: [PATCH 40/49] SimDB / v3 integration --- sparta/CMakeLists.txt | 1 + .../CoreModel/src/LoadStoreInstInfo.hpp | 45 +++++++++---------- .../example/CoreModel/src/MemAccessInfo.hpp | 3 ++ sparta/simdb | 2 +- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/sparta/CMakeLists.txt b/sparta/CMakeLists.txt index 0676874347..86f9b09840 100644 --- a/sparta/CMakeLists.txt +++ b/sparta/CMakeLists.txt @@ -224,3 +224,4 @@ install(DIRECTORY sparta/ DESTINATION include/sparta) install(DIRECTORY cache/ DESTINATION include/cache) install(FILES ${SPARTA_STATIC_LIBS} DESTINATION lib) install(DIRECTORY cmake/ DESTINATION lib/cmake/sparta) +install(DIRECTORY simdb/include DESTINATION .) diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index cae371e334..1bc6b646aa 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -156,6 +156,28 @@ inline void defineEnumMap(std::stri map["NOT_READY"] = static_cast(IssueState::NOT_READY); } + +template <> +inline void defineStructSchema(StructSchema& schema) +{ + schema.addField("DID"); + schema.addEnum("rank"); + schema.addEnum("state"); +} + +template <> +inline void writeStructFields( + const core_example::LoadStoreInstInfo* inst, + StructFieldSerializer* serializer) +{ + serializer->writeField(inst->getInstUniqueID()); + serializer->writeField(inst->getPriority()); + serializer->writeField(inst->getState()); + + core_example::MemoryAccessInfo* mem_access_info = inst->getMemoryAccessInfoPtr().get(); + StructSerializer::getInstance()->writeStruct(mem_access_info, serializer->getBuffer()); +} + } // namespace simdb inline std::ostream& operator<<(std::ostream& os, @@ -206,26 +228,3 @@ inline std::ostream& operator<<(std::ostream& os, } return os; } - -namespace simdb -{ - -template <> -inline void defineStructSchema(StructSchema& schema) -{ - schema.addField("DID"); - schema.addEnum("rank"); - schema.addEnum("state"); -} - -template <> -inline void writeStructFields( - const core_example::LoadStoreInstInfo* inst, - StructFieldSerializer* serializer) -{ - serializer->writeField(inst->getInstUniqueID()); - serializer->writeField(inst->getPriority()); - serializer->writeField(inst->getState()); -} - -} // namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index 7ed61d3981..c30fbed459 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -202,6 +202,9 @@ inline void writeStructFields( serializer->writeField(inst->getPhyAddrIsReady()); serializer->writeField(inst->getMMUState()); serializer->writeField(inst->getCacheState()); + + core_example::ExampleInst* ex_inst = inst->getInstPtr().get(); + StructSerializer::getInstance()->writeStruct(ex_inst, serializer->getBuffer()); } } // namespace simdb diff --git a/sparta/simdb b/sparta/simdb index 7cc3648fb1..e6c4148178 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 7cc3648fb1da146b846a3d8d28d43d70be476ded +Subproject commit e6c4148178e67430a9f760fd269805f388ea1fe1 From 1118bb813b170a110cdfcd4794bff23fe56420cc Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 14:33:41 -0600 Subject: [PATCH 41/49] SimDB / v3 integration --- sparta/example/DynamicModelPipeline/src/ExampleInst.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp index 38835e96c8..b20b3d321b 100644 --- a/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp +++ b/sparta/example/DynamicModelPipeline/src/ExampleInst.hpp @@ -28,7 +28,7 @@ namespace core_example class ExampleInst { public: - + // The modeler needs to alias a type called "type" to the Pair Definition class of itself using type = ExampleInstPairDef; From 60a00b4830b7396ac3903eb082e54348fd173f33 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 15:16:19 -0600 Subject: [PATCH 42/49] SimDB / v3 integration --- .../sparta/collection/IterableCollector.hpp | 17 +++++----- .../sparta/collection/PipelineCollector.hpp | 6 ++-- sparta/src/StatisticInstance.cpp | 32 +++++++++---------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index 91bc3bcaa7..2510ffa59c 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -70,11 +70,11 @@ class IterableCollector : public CollectableTreeNode const IterableType * iterable, const size_type expected_capacity) : CollectableTreeNode(parent, name, group, index, desc), + iterable_object_(iterable), + expected_capacity_(expected_capacity), event_set_(this), ev_close_record_(&event_set_, name + "_pipeline_collectable_close_event", - CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)), - iterable_object_(iterable), - expected_capacity_(expected_capacity) + CREATE_SPARTA_HANDLER_WITH_DATA(IterableCollector, closeRecord, bool)) { for (size_type i = 0; i < expected_capacity_; ++i) { @@ -316,22 +316,21 @@ class IterableCollector : public CollectableTreeNode } } + const IterableType * iterable_object_; + std::vector> positions_; + const size_type expected_capacity_; + bool auto_collect_ = false; + // For those folks that want a value to automatically // disappear in the future sparta::EventSet event_set_; sparta::PayloadEvent ev_close_record_; - const IterableType * iterable_object_; - std::vector> positions_; - const size_type expected_capacity_ = 0; - bool auto_collect_ = false; - using simdb_collectable_type = std::conditional_t; std::shared_ptr simdb_collectable_; }; - } // namespace collection } // namespace sparta diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 444d09d77d..15a5f7377c 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -423,9 +423,9 @@ namespace collection private: void performSweep_() { - // Note that we subtract one from the current tick - // since this sweep runs in the very last phase - // when the tick has already advanced. + // "Current tick" has been changed to reflect that the Scheduler + // has moved on and is now on the "next" tick. Elapsed time + // should reflect current tick - 1 auto tick = clk_->getScheduler()->getCurrentTick(); collection_mgr_->sweep(clk_->getName(), tick - 1); diff --git a/sparta/src/StatisticInstance.cpp b/sparta/src/StatisticInstance.cpp index e1f3b45836..25c4f2be17 100644 --- a/sparta/src/StatisticInstance.cpp +++ b/sparta/src/StatisticInstance.cpp @@ -99,22 +99,6 @@ namespace sparta sparta_assert(false == node_ref_.expired()); } - //! \brief Copy Constructor - StatisticInstance::StatisticInstance(const StatisticInstance& rhp) : - node_ref_(rhp.node_ref_), - sdef_(rhp.sdef_), - ctr_(rhp.ctr_), - par_(rhp.par_), - stat_expr_(rhp.stat_expr_), - start_tick_(rhp.start_tick_), - end_tick_(rhp.end_tick_), - scheduler_(rhp.scheduler_), - initial_(rhp.initial_), - result_(rhp.result_), - sub_statistics_(rhp.sub_statistics_) - { - } - StatisticInstance::StatisticInstance(std::shared_ptr & calculator, std::vector& used) : StatisticInstance(nullptr, nullptr, nullptr, calculator->getNode(), &used) @@ -131,6 +115,22 @@ namespace sparta user_calculated_si_value_ = calculator; } + //! \brief Copy Constructor + StatisticInstance::StatisticInstance(const StatisticInstance& rhp) : + node_ref_(rhp.node_ref_), + sdef_(rhp.sdef_), + ctr_(rhp.ctr_), + par_(rhp.par_), + stat_expr_(rhp.stat_expr_), + start_tick_(rhp.start_tick_), + end_tick_(rhp.end_tick_), + scheduler_(rhp.scheduler_), + initial_(rhp.initial_), + result_(rhp.result_), + sub_statistics_(rhp.sub_statistics_) + { + } + //! \brief Move Constructor StatisticInstance::StatisticInstance(StatisticInstance&& rhp) : node_ref_(std::move(rhp.node_ref_)), From d5bd3336cc51279185542cb4bb634b5bac588e9c Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 15:43:09 -0600 Subject: [PATCH 43/49] SimDB / v3 integration --- .../sparta/statistics/StatisticInstance.hpp | 28 +------------------ sparta/src/StatisticInstance.cpp | 6 ++-- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/sparta/sparta/statistics/StatisticInstance.hpp b/sparta/sparta/statistics/StatisticInstance.hpp index dc0c88ca9e..ee08c27adb 100644 --- a/sparta/sparta/statistics/StatisticInstance.hpp +++ b/sparta/sparta/statistics/StatisticInstance.hpp @@ -1,3 +1,4 @@ + // -*- C++ -*- /*! @@ -161,33 +162,6 @@ namespace sparta //! \brief Move Constructor StatisticInstance(StatisticInstance&& rhp); - /*! - * \brief Construct a StatisticInstance with its metadata - * values set directly, as opposed to this SI asking its - * underlying counter/parameter/etc. for these values. - */ - StatisticInstance(const std::string & location, - const std::string & description, - const std::string & expression_str, - const StatisticDef::ValueSemantic value_semantic, - const InstrumentationNode::visibility_t visibility, - const InstrumentationNode::class_t cls, - const std::vector> & metadata = {}); - - /*! - * \brief Construct a StatisticInstance with its location and - * description set directly, along with a StatInstCalculator - * which can retrieve the SI value on demand from another - * source (such as a database file). - */ - StatisticInstance( - const std::string & location, - const std::string & description, - const std::shared_ptr & calculator, - const InstrumentationNode::visibility_t visibility = InstrumentationNode::DEFAULT_VISIBILITY, - const InstrumentationNode::class_t cls = InstrumentationNode::DEFAULT_CLASS, - const std::vector> & metadata = {}); - /*! * \brief Non-Virtual destructor */ diff --git a/sparta/src/StatisticInstance.cpp b/sparta/src/StatisticInstance.cpp index 25c4f2be17..fc32857fb4 100644 --- a/sparta/src/StatisticInstance.cpp +++ b/sparta/src/StatisticInstance.cpp @@ -127,7 +127,8 @@ namespace sparta scheduler_(rhp.scheduler_), initial_(rhp.initial_), result_(rhp.result_), - sub_statistics_(rhp.sub_statistics_) + sub_statistics_(rhp.sub_statistics_), + user_calculated_si_value_(rhp.user_calculated_si_value_) { } @@ -142,7 +143,8 @@ namespace sparta end_tick_(rhp.end_tick_), scheduler_(rhp.scheduler_), initial_(rhp.initial_), - result_(rhp.result_) + result_(rhp.result_), + user_calculated_si_value_(std::move(rhp.user_calculated_si_value_)) { rhp.sdef_ = nullptr; rhp.ctr_ = nullptr; From 0063a8bddbb33c9cb7fe955f31990c7c2665ca6f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 15:44:22 -0600 Subject: [PATCH 44/49] SimDB / v3 integration --- sparta/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/.gitignore b/sparta/.gitignore index 09ca92f5d8..0f83e90cef 100644 --- a/sparta/.gitignore +++ b/sparta/.gitignore @@ -12,4 +12,4 @@ build* cmake-build-* *~ compile_commands.json -.vscode +.vscode \ No newline at end of file From 921e4b494f54dcea8ad0d4a47a8c9942adfea8f5 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Mon, 24 Feb 2025 16:07:52 -0600 Subject: [PATCH 45/49] Delete legacy Argos/pipeViewer --- README.md | 20 +- helios/CMakeLists.txt | 5 - helios/README.md | 10 +- helios/pipeViewer/.gitignore | 4 - helios/pipeViewer/CMakeLists.txt | 108 - helios/pipeViewer/README.md | 111 - helios/pipeViewer/argos_dumper/.gitignore | 5 - .../argos_dumper/ArgosCollection_test.cpp | 141 -- helios/pipeViewer/argos_dumper/CMakeLists.txt | 15 - .../argos_dumper/DatabaseDump/CMakeLists.txt | 11 - .../DatabaseDump/Database_dumper.cpp | 218 -- .../db_pipeout/pipeoutsimulation.info | 5 - .../argos_dumper/testPipe_layout.alf | 338 --- .../pipeViewer/argos_dumper/testPipeline.alf | 282 --- helios/pipeViewer/mypy.ini | 31 - helios/pipeViewer/pipe_view/.gitignore | 2 - helios/pipeViewer/pipe_view/__init__.py | 0 helios/pipeViewer/pipe_view/argos.py | 468 ---- helios/pipeViewer/pipe_view/core/.gitignore | 1 - helios/pipeViewer/pipe_view/core/__init__.pyi | 42 - .../pipeViewer/pipe_view/core/src/common.pxd | 32 - helios/pipeViewer/pipe_view/core/src/core.pyx | 641 ----- .../pipeViewer/pipe_view/core/src/helpers.h | 74 - helios/pipeViewer/pipe_view/doc/Doxyfile | 1800 --------------- helios/pipeViewer/pipe_view/gui/__init__.py | 0 helios/pipeViewer/pipe_view/gui/argos_menu.py | 1554 ------------- .../pipeViewer/pipe_view/gui/autocoloring.py | 254 -- .../pipe_view/gui/dialogs/__init__.py | 0 .../pipe_view/gui/dialogs/console_dialog.py | 62 - .../pipe_view/gui/dialogs/element_propsdlg.py | 161 -- .../pipe_view/gui/dialogs/find_element_dlg.py | 174 -- .../gui/dialogs/layout_exit_dialog.py | 129 -- .../pipe_view/gui/dialogs/layout_varsdlg.py | 178 -- .../pipe_view/gui/dialogs/location_window.py | 367 --- .../pipe_view/gui/dialogs/search_dlg.py | 384 --- .../pipe_view/gui/dialogs/select_db_dlg.py | 195 -- .../gui/dialogs/select_layout_dlg.py | 209 -- .../pipe_view/gui/dialogs/shortcut_help.py | 144 -- .../gui/dialogs/translate_elements_dlg.py | 296 --- .../gui/dialogs/view_settings_dlg.py | 92 - .../pipe_view/gui/dialogs/watchlist_dialog.py | 111 - helios/pipeViewer/pipe_view/gui/font_utils.py | 59 - .../pipeViewer/pipe_view/gui/hover_preview.py | 654 ------ .../pipeViewer/pipe_view/gui/input_decoder.py | 614 ----- .../pipe_view/gui/key_definitions.py | 72 - .../pipeViewer/pipe_view/gui/layout_canvas.py | 663 ------ .../pipeViewer/pipe_view/gui/layout_frame.py | 529 ----- .../pipe_view/gui/selection_manager.py | 2054 ----------------- .../pipe_view/gui/widgets/__init__.py | 0 .../pipe_view/gui/widgets/element_list.py | 102 - .../gui/widgets/element_property_list.py | 825 ------- .../gui/widgets/frame_playback_bar.py | 1069 --------- .../pipe_view/gui/widgets/location_entry.py | 55 - .../pipe_view/gui/widgets/transaction_list.py | 105 - .../pipeViewer/pipe_view/logsearch/.gitignore | 1 - .../pipe_view/logsearch/__init__.pyi | 11 - .../pipe_view/logsearch/src/log_search.cpp | 101 - .../pipe_view/logsearch/src/logsearch.pyx | 51 - helios/pipeViewer/pipe_view/misc/__init__.py | 0 helios/pipeViewer/pipe_view/misc/version.py | 10 - helios/pipeViewer/pipe_view/model/__init__.py | 0 .../pipe_view/model/clock_manager.py | 300 --- .../pipe_view/model/content_options.py | 286 --- helios/pipeViewer/pipe_view/model/database.py | 126 - .../pipe_view/model/database_handle.py | 79 - helios/pipeViewer/pipe_view/model/element.py | 1115 --------- .../pipe_view/model/element_propsvalid.py | 238 -- .../pipeViewer/pipe_view/model/element_set.py | 288 --- .../pipe_view/model/element_types.py | 22 - .../pipe_view/model/element_value.py | 257 --- .../pipe_view/model/extension_manager.py | 44 - helios/pipeViewer/pipe_view/model/group.py | 71 - .../pipe_view/model/highlighting_utils.py | 18 - helios/pipeViewer/pipe_view/model/layout.py | 663 ------ .../pipe_view/model/layout_context.py | 733 ------ .../pipe_view/model/layout_delta.py | 170 -- .../pipe_view/model/location_manager.py | 307 --- .../pipeViewer/pipe_view/model/quad_tree.py | 391 ---- .../pipeViewer/pipe_view/model/query_set.py | 598 ----- .../pipeViewer/pipe_view/model/rpc_element.py | 217 -- .../pipe_view/model/schedule_element.py | 930 -------- .../pipe_view/model/search_handle.py | 186 -- helios/pipeViewer/pipe_view/model/settings.py | 119 - .../pipe_view/model/speedo_element.py | 341 --- .../pipeViewer/pipe_view/model/utilities.py | 90 - .../pipe_view/model/widget_element.py | 129 -- .../pipeViewer/pipe_view/model/workspace.py | 349 --- .../resources/Actions-transform-move-icon.png | Bin 1544 -> 0 bytes .../pipe_view/resources/add-space-down.png | Bin 1101 -> 0 bytes .../pipe_view/resources/add-space-left.png | Bin 1128 -> 0 bytes .../pipe_view/resources/add-space-right.png | Bin 1109 -> 0 bytes .../pipe_view/resources/add-space-up.png | Bin 1100 -> 0 bytes .../pipe_view/resources/bullet-arrow-up.png | Bin 651 -> 0 bytes .../pipeViewer/pipe_view/resources/logo.png | Bin 8146 -> 0 bytes .../pipe_view/resources/resize_nesw.png | Bin 276 -> 0 bytes .../pipe_view/resources/resize_nwse.png | Bin 272 -> 0 bytes .../resources/shape-align-bottom.png | Bin 499 -> 0 bytes .../pipe_view/resources/shape-align-left.png | Bin 457 -> 0 bytes .../pipe_view/resources/shape-align-right.png | Bin 469 -> 0 bytes .../pipe_view/resources/shape-align-top.png | Bin 516 -> 0 bytes .../resources/shape-flip-horizontal.png | Bin 604 -> 0 bytes .../resources/shape-flip-vertical.png | Bin 640 -> 0 bytes .../pipe_view/resources/shape-move-back.png | Bin 476 -> 0 bytes .../resources/shape-move-backward.png | Bin 399 -> 0 bytes .../resources/shape-move-forward.png | Bin 412 -> 0 bytes .../pipe_view/resources/shape-move-front.png | Bin 518 -> 0 bytes .../pipe_view/resources/sub-space-down.png | Bin 1001 -> 0 bytes .../pipe_view/resources/sub-space-left.png | Bin 1011 -> 0 bytes .../pipe_view/resources/sub-space-right.png | Bin 986 -> 0 bytes .../pipe_view/resources/sub-space-up.png | Bin 1002 -> 0 bytes .../pipe_view/transactiondb/.gitignore | 1 - .../pipe_view/transactiondb/__init__.pyi | 67 - .../src/PipelineDataCallback.hpp | 33 - .../pipe_view/transactiondb/src/Reader.hpp | 1216 ---------- .../src/SimpleOutputInterface.hpp | 84 - .../src/TransactionDatabaseInterface.hpp | 1902 --------------- .../transactiondb/src/TransactionInterval.hpp | 263 --- .../pipe_view/transactiondb/src/common.pxd | 59 - .../pipe_view/transactiondb/src/helpers.hpp | 13 - .../transactiondb/src/transactiondb.pyx | 738 ------ .../pipeViewer/scripts/alf_gen/ALFLayout.py | 665 ------ .../scripts/color_nodes_by_access.py | 20 - helios/pipeViewer/setup.py | 68 - helios/pipeViewer/stubs/wx/__init__.pyi | 1589 ------------- helios/pipeViewer/stubs/wx/html.pyi | 21 - .../transactionsearch/CMakeLists.txt | 11 - .../src/transaction_search.cpp | 411 ---- helios/pipeViewer/wxPythonInclude.py | 12 - sparta/test/CMakeLists.txt | 1 - 129 files changed, 17 insertions(+), 30873 deletions(-) delete mode 100644 helios/CMakeLists.txt delete mode 100644 helios/pipeViewer/.gitignore delete mode 100644 helios/pipeViewer/CMakeLists.txt delete mode 100644 helios/pipeViewer/README.md delete mode 100644 helios/pipeViewer/argos_dumper/.gitignore delete mode 100644 helios/pipeViewer/argos_dumper/ArgosCollection_test.cpp delete mode 100644 helios/pipeViewer/argos_dumper/CMakeLists.txt delete mode 100644 helios/pipeViewer/argos_dumper/DatabaseDump/CMakeLists.txt delete mode 100644 helios/pipeViewer/argos_dumper/DatabaseDump/Database_dumper.cpp delete mode 100644 helios/pipeViewer/argos_dumper/DatabaseDump/db_pipeout/pipeoutsimulation.info delete mode 100644 helios/pipeViewer/argos_dumper/testPipe_layout.alf delete mode 100644 helios/pipeViewer/argos_dumper/testPipeline.alf delete mode 100644 helios/pipeViewer/mypy.ini delete mode 100644 helios/pipeViewer/pipe_view/.gitignore delete mode 100644 helios/pipeViewer/pipe_view/__init__.py delete mode 100755 helios/pipeViewer/pipe_view/argos.py delete mode 100644 helios/pipeViewer/pipe_view/core/.gitignore delete mode 100644 helios/pipeViewer/pipe_view/core/__init__.pyi delete mode 100644 helios/pipeViewer/pipe_view/core/src/common.pxd delete mode 100644 helios/pipeViewer/pipe_view/core/src/core.pyx delete mode 100644 helios/pipeViewer/pipe_view/core/src/helpers.h delete mode 100644 helios/pipeViewer/pipe_view/doc/Doxyfile delete mode 100644 helios/pipeViewer/pipe_view/gui/__init__.py delete mode 100644 helios/pipeViewer/pipe_view/gui/argos_menu.py delete mode 100644 helios/pipeViewer/pipe_view/gui/autocoloring.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/__init__.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/console_dialog.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/element_propsdlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/find_element_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/layout_exit_dialog.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/layout_varsdlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/location_window.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/search_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/select_db_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/select_layout_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/shortcut_help.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/translate_elements_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/view_settings_dlg.py delete mode 100644 helios/pipeViewer/pipe_view/gui/dialogs/watchlist_dialog.py delete mode 100644 helios/pipeViewer/pipe_view/gui/font_utils.py delete mode 100644 helios/pipeViewer/pipe_view/gui/hover_preview.py delete mode 100644 helios/pipeViewer/pipe_view/gui/input_decoder.py delete mode 100644 helios/pipeViewer/pipe_view/gui/key_definitions.py delete mode 100644 helios/pipeViewer/pipe_view/gui/layout_canvas.py delete mode 100644 helios/pipeViewer/pipe_view/gui/layout_frame.py delete mode 100644 helios/pipeViewer/pipe_view/gui/selection_manager.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/__init__.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/element_list.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/element_property_list.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/frame_playback_bar.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/location_entry.py delete mode 100644 helios/pipeViewer/pipe_view/gui/widgets/transaction_list.py delete mode 100644 helios/pipeViewer/pipe_view/logsearch/.gitignore delete mode 100644 helios/pipeViewer/pipe_view/logsearch/__init__.pyi delete mode 100644 helios/pipeViewer/pipe_view/logsearch/src/log_search.cpp delete mode 100644 helios/pipeViewer/pipe_view/logsearch/src/logsearch.pyx delete mode 100644 helios/pipeViewer/pipe_view/misc/__init__.py delete mode 100644 helios/pipeViewer/pipe_view/misc/version.py delete mode 100644 helios/pipeViewer/pipe_view/model/__init__.py delete mode 100644 helios/pipeViewer/pipe_view/model/clock_manager.py delete mode 100644 helios/pipeViewer/pipe_view/model/content_options.py delete mode 100644 helios/pipeViewer/pipe_view/model/database.py delete mode 100644 helios/pipeViewer/pipe_view/model/database_handle.py delete mode 100644 helios/pipeViewer/pipe_view/model/element.py delete mode 100644 helios/pipeViewer/pipe_view/model/element_propsvalid.py delete mode 100644 helios/pipeViewer/pipe_view/model/element_set.py delete mode 100644 helios/pipeViewer/pipe_view/model/element_types.py delete mode 100644 helios/pipeViewer/pipe_view/model/element_value.py delete mode 100644 helios/pipeViewer/pipe_view/model/extension_manager.py delete mode 100644 helios/pipeViewer/pipe_view/model/group.py delete mode 100644 helios/pipeViewer/pipe_view/model/highlighting_utils.py delete mode 100644 helios/pipeViewer/pipe_view/model/layout.py delete mode 100644 helios/pipeViewer/pipe_view/model/layout_context.py delete mode 100644 helios/pipeViewer/pipe_view/model/layout_delta.py delete mode 100644 helios/pipeViewer/pipe_view/model/location_manager.py delete mode 100644 helios/pipeViewer/pipe_view/model/quad_tree.py delete mode 100644 helios/pipeViewer/pipe_view/model/query_set.py delete mode 100644 helios/pipeViewer/pipe_view/model/rpc_element.py delete mode 100644 helios/pipeViewer/pipe_view/model/schedule_element.py delete mode 100644 helios/pipeViewer/pipe_view/model/search_handle.py delete mode 100644 helios/pipeViewer/pipe_view/model/settings.py delete mode 100644 helios/pipeViewer/pipe_view/model/speedo_element.py delete mode 100644 helios/pipeViewer/pipe_view/model/utilities.py delete mode 100644 helios/pipeViewer/pipe_view/model/widget_element.py delete mode 100644 helios/pipeViewer/pipe_view/model/workspace.py delete mode 100644 helios/pipeViewer/pipe_view/resources/Actions-transform-move-icon.png delete mode 100644 helios/pipeViewer/pipe_view/resources/add-space-down.png delete mode 100644 helios/pipeViewer/pipe_view/resources/add-space-left.png delete mode 100644 helios/pipeViewer/pipe_view/resources/add-space-right.png delete mode 100644 helios/pipeViewer/pipe_view/resources/add-space-up.png delete mode 100644 helios/pipeViewer/pipe_view/resources/bullet-arrow-up.png delete mode 100644 helios/pipeViewer/pipe_view/resources/logo.png delete mode 100644 helios/pipeViewer/pipe_view/resources/resize_nesw.png delete mode 100644 helios/pipeViewer/pipe_view/resources/resize_nwse.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-align-bottom.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-align-left.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-align-right.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-align-top.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-flip-horizontal.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-flip-vertical.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-move-back.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-move-backward.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-move-forward.png delete mode 100644 helios/pipeViewer/pipe_view/resources/shape-move-front.png delete mode 100644 helios/pipeViewer/pipe_view/resources/sub-space-down.png delete mode 100644 helios/pipeViewer/pipe_view/resources/sub-space-left.png delete mode 100644 helios/pipeViewer/pipe_view/resources/sub-space-right.png delete mode 100644 helios/pipeViewer/pipe_view/resources/sub-space-up.png delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/.gitignore delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/__init__.pyi delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/PipelineDataCallback.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/Reader.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/SimpleOutputInterface.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/TransactionInterval.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/common.pxd delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/helpers.hpp delete mode 100644 helios/pipeViewer/pipe_view/transactiondb/src/transactiondb.pyx delete mode 100644 helios/pipeViewer/scripts/alf_gen/ALFLayout.py delete mode 100644 helios/pipeViewer/scripts/color_nodes_by_access.py delete mode 100644 helios/pipeViewer/setup.py delete mode 100644 helios/pipeViewer/stubs/wx/__init__.pyi delete mode 100644 helios/pipeViewer/stubs/wx/html.pyi delete mode 100644 helios/pipeViewer/transactionsearch/CMakeLists.txt delete mode 100644 helios/pipeViewer/transactionsearch/src/transaction_search.cpp delete mode 100644 helios/pipeViewer/wxPythonInclude.py diff --git a/README.md b/README.md index a2b8279b86..02cd0864ad 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,19 @@ MAP is broken into two parts: [![MacOS Build Status](https://dev.azure.com/sparcians/map/_apis/build/status/sparcians.map?branchName=master&label=MacOS)](https://dev.azure.com/sparcians/map/_build/latest?definitionId=1&branchName=master) [![Documentation](https://github.com/sparcians/map/workflows/Documentation/badge.svg)](https://sparcians.github.io/map/) -## Building MAP +## Cloning MAP + +MAP uses git submodules, so you should either check out MAP with the --recursive option: + +- `git clone git@github.com:sparcians/map.git --recursive` + +Or you can get the submodules manually: -Building MAP is done in two parts +- `git clone git@github.com:sparcians/map.git` +- `cd map` +- `git submodule update --init --recursive` -1. Sparta, the modeling framework: build sparta only in the sparta folder -2. Argos, the transaction viewer in Helios in the helios folder. Note that to build and use helios, you will need sparta built and installed somwehere on your system. +## Building MAP The MAP repository has numerous dependencies, which are listed in a [conda recipe](https://github.com/sparcians/map/blob/master/conda.recipe/meta.yaml), @@ -80,11 +87,6 @@ installed and would like to build everything (not just sparta). * `cmake -DCMAKE_BUILD_TYPE=Release ..` * `make` * `cmake --install . --prefix $CONDA_PREFIX` -1. To build Helios/Argos transaction viewer: - * `cd helios && mkdir release && cd release` - * `cmake -DCMAKE_BUILD_TYPE=Release ..` - * `make` - * `cmake --install . --prefix $CONDA_PREFIX` A few interesting cmake options to help resolve dependencies are: diff --git a/helios/CMakeLists.txt b/helios/CMakeLists.txt deleted file mode 100644 index 01920f509c..0000000000 --- a/helios/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required (VERSION 3.19) - -project(helios) - -add_subdirectory (pipeViewer) diff --git a/helios/README.md b/helios/README.md index 61fc74743e..a9e4900197 100644 --- a/helios/README.md +++ b/helios/README.md @@ -1,11 +1,13 @@ # Helios Visulation Tools -This part of the MAP repo contains two tools that work with +This part of the MAP repo contains tools that work with Sparta-developed models to visualize the data generated. 1. Plato -- (WIP) A web-based visualization tool that will consume database files generated by Sparta-based models and allow the user to "drag-n-drop" plots onto a main canvas -1. pipeViewer (a.k.a Argos) -- A pipeline crawler, consistent with the - concept of a waveform viewer, but with more information viewable. - It's sometimes been called a rainbox chart or waterfall diagram. + +//2. Note that prior Sparta releases had the Argos tool here +// in Helios (pipeline visualization). That has been moved +// to an external git repo which Sparta uses as a submodule. +// See the README file here: map/sparta/simdb/README.md diff --git a/helios/pipeViewer/.gitignore b/helios/pipeViewer/.gitignore deleted file mode 100644 index 76578b146d..0000000000 --- a/helios/pipeViewer/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -build/ -dist/ -pipe_view.egg-info/ -setup.cfg diff --git a/helios/pipeViewer/CMakeLists.txt b/helios/pipeViewer/CMakeLists.txt deleted file mode 100644 index 0d999d1355..0000000000 --- a/helios/pipeViewer/CMakeLists.txt +++ /dev/null @@ -1,108 +0,0 @@ -cmake_minimum_required(VERSION 3.19) -project (PipeViewer) - -# Set up Sparta -if(IS_DIRECTORY ${SPARTA_SEARCH_DIR}) - set(CMAKE_MODULE_PATH "${SPARTA_SEARCH_DIR}/lib/cmake/sparta" ${CMAKE_MODULE_PATH}) - find_package(Sparta REQUIRED) -else() - set(SPARTA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../sparta) - message (STATUS "Looking for a built version of sparta in the source tree at ${SPARTA_DIR}") - set(CMAKE_MODULE_PATH "${SPARTA_DIR}/cmake" ${CMAKE_MODULE_PATH}) - set(SPARTA_INCLUDE_DIR ${SPARTA_DIR} ${SPARTA_DIR}/simdb) - set(SPARTA_LIBRARY ${SPARTA_DIR}/release ${SPARTA_DIR}/release/simdb) - include(${SPARTA_DIR}/cmake/FindSparta.cmake) -endif() - -if(NOT SPARTA_FOUND) - message (FATAL_ERROR "Could not find Sparta. (${SPARTA_FOUND}) If needed, please provide the location where sparta is installed: -DSPARTA_SEARCH_DIR=") -endif() - -# If we are in virtualenv or conda, that takes prio over any system python there might be -set(Python3_FIND_VIRTUALENV FIRST) -find_package(Python3 3.7 REQUIRED COMPONENTS Interpreter) - -# Look for wxWidgets -find_package(wxWidgets REQUIRED core base) -include(${wxWidgets_USE_FILE}) - -# Populate list of include dirs: -# Little helper script to find wxPython include path -execute_process( - COMMAND ${Python3_EXECUTABLE} wxPythonInclude.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - OUTPUT_VARIABLE WX_PYTHON_INC - ERROR_VARIABLE WXCHECK_LOG - RESULT_VARIABLE WXCHECK_RESULT - OUTPUT_STRIP_TRAILING_WHITESPACE -) -if(NOT WXCHECK_RESULT EQUAL "0") - message(FATAL_ERROR "Could not find wxPython: ${WXCHECK_LOG}") -endif() -list(APPEND _INC_DIRS "$" ${WX_PYTHON_INC} ${wxWidgets_INCLUDE_DIRS}) - -# Populate list of library dirs: -# Set up the include and link dirs for python's setuptools using setup.cfg file -# HDF5: need to extract dirs from paths to libs -foreach(lib IN LISTS HDF5_LIBRARIES) - get_filename_component(d ${lib} DIRECTORY) - list(APPEND _LIB_DIRS ${d}) -endforeach() -list(REMOVE_DUPLICATES _LIB_DIRS) -# And we add sparta and simdb location -# they are in the same folder so we look at libsparta only to find the right location -get_property(_SPARTA_LIB TARGET SPARTA::libsparta PROPERTY IMPORTED_LOCATION) -get_filename_component(SPARTA_LIBDIR ${_SPARTA_LIB} DIRECTORY) -list(APPEND _LIB_DIRS ${SPARTA_LIBDIR} ${wxWidgets_LIBRARY_DIRS}) - -# Those get pushed to setup.cfg file -file(GENERATE - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg - CONTENT -" -[build_ext] -include_dirs=$ -library_dirs=$ -") - -# Wx specific arguments get passed via LDFLAGS and CXXFLAGS in the command -foreach(def IN LISTS wxWidgets_DEFINITIONS) - string(APPEND PYTHON_CFLAGS "-D${def} ") -endforeach() -string(JOIN " " PYTHON_LDFLAGS ${wxWidgets_LIBRARIES}) - -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp - COMMAND ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} LD=${CMAKE_LINKER} CFLAGS=${PYTHON_CFLAGS} LDFLAGS=${PYTHON_LDFLAGS} ${Python3_EXECUTABLE} setup.py build_ext --inplace VERBATIM - COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/setup.py - ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/common.pxd - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/core.pyx - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/helpers.h - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/logsearch/src/log_search.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/logsearch/src/logsearch.pyx - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/PipelineDataCallback.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/Reader.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/SimpleOutputInterface.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/TransactionInterval.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/common.pxd - ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/transactiondb.pyx - ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg - COMMENT "Building pipeViwer and its dependencies" -) -add_custom_target(pipeView ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp) - -add_subdirectory(transactionsearch) - -install(CODE - "execute_process( - COMMAND ${CMAKE_COMMAND} -E env PYTHONUSERBASE=${CMAKE_INSTALL_PREFIX} ${Python3_EXECUTABLE} -m pip install . - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})" -) - -add_subdirectory(argos_dumper) diff --git a/helios/pipeViewer/README.md b/helios/pipeViewer/README.md deleted file mode 100644 index e9c7af2509..0000000000 --- a/helios/pipeViewer/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# Argos: The PipeViewer Transaction Viewer - -## Prerequisites - -Follow the conda environment directions found in the top-level README.md of MAP. - -## Example of Usage - -In this example, you will: -1. Build example core model -1. Generate pipeline database (also known as a `pipeout`) -1. View pipeout using single-cycle layout -1. View pipeout using multi-cycle layout -1. Learn how to create and edit your own multi-cycle layout - -### 1. Build example core model - -There are several ways to build the example core model. First build sparta as described above. Then, pick one of the following: -- In your build directory (ex: `map/sparta/release`), run `make regress` -- In your build directory (ex: `map/sparta/release`), run `make sparta_core_example` -- In example core model directory (ex: `map/sparta/release/example/CoreModel`), run `make` - -### 2. Generate pipeline database - -In the example core model directory (ex: `map/sparta/release/example/CoreModel`), run: -``` -$ ./sparta_core_example -i 1000 -z pipeout_ -``` - -In this example, we run the simulation until 1000 instructions have retired, and we generate a pipeline database corresponding to the entire run. If we wanted to skip a certain number instructions or cycles before collecting pipeline information, we could use the `--debug-on` or `--debug-on-icount` options to specify when to start collecting pipeline information. - -The pipeline database consists of several files, all of which begin with the same prefix. In the example above, we specify the prefix `pipeout_`. - -The database consists of metadata, plus a number of `transactions`. A `transaction` is a piece of data (such as an instruction or a count) that occurs in a specific cycle in a specific location. A `location` can be a pipeline stage, a place in a structure like a queue, a signal, etc. - -### 3. View pipeout using single-cycle layout - -To view the database, you must run Argos and specify both the database prefix and one or more layout files. A `layout` file specifies how the transactions will be visually organized on the screen. - -In the example core model directory, run: -``` -# Linux -$ python3 ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file cpu_layout.alf - -# MacOS -$ pythonw ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file cpu_layout.alf -``` -where `${MAP}` represents the path to your `map` repo. We are using the database with the prefix `pipeout_` and are using the single-cycle layout file in this directory `cpu_layout.alf`. - -This single-cycle view shows only a single cycle at a time. Each box represents a location. In the lower left corner of the window, we see the current cycle. We use controls at the bottom of the window to change the cycle, in any of the following ways: -- Right and left arrow keys will increment/decrement current cycle by 1. -- Buttons at the bottom let you change the current cycle by +1, +3, +10, +30, etc. -- A text box near the buttons lets you specify how much you want to jump; enter the value and click the "jump" button. If the number begins with a `+` or `-` then the cycle changes by this relative amount. If the number does not begin with `+` or `-`, then this specifies the absolute value of the cycle to jump to. -- A `play` button in the lower right of the window allows automatically changing the current cycle over time. - -More controls: you should be able to use normal controls to zoom in/zoom out in the display. For example, `Ctrl +` and `Ctrl -` or `Ctrl` plus mouse scroll should work. - -As we change the current cycle, we can see transactions appear in the appropriate locations. Note that each transaction has a color and begins with a `display ID`. If the visual element is not large enough to hold all the text in the transaction, you can hover your mouse over the location and hover text will show you the entire text of the transaction. - -The `display ID` is used to assign to the transaction both a color and a single letter (case is important). The letter is important in the multi-cycle layout and will be discussed in that section. - -If the transaction represents an instruction, the instruction should have a consistent display ID no matter where the instruction is located. For example, if the instruction's display ID is mapped to a purple W, it should be a purple W no matter what location it is in. (Again, the "W" is mainly used in the multi-cycle layout.) - -Single-cycle layouts can be created using graphical editing tools built into the pipeline viewer, but we do not discuss this here because multi-cycle layouts are much more powerful and versatile and are the preferred method for viewing the pipeline database. - -### 4. View pipeout using multi-cycle layout - -Viewing a multi-cycle layout is done the exact same way as a single-cycle layout. In the example core model directory, run: -``` -# Linux -python3 ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file ${MAP}/helios/pipeViewer/scripts/gen_alf/core.alf - -# MacOS -pythonw ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file ${MAP}/helios/pipeViewer/scripts/gen_alf/core.alf -``` - -In this example, we replace the single-cycle layout file with a multi-cycle layout. You can also specify multiple layout files. Each layout will be displayed in its own window, and the current cycle will be sync'ed up among all the layout specified. For example: - -``` -# Linux -python3 ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file cpu_layout.alf --layout-file ${MAP}/helios/pipeViewer/scripts/gen_alf/core.alf - -# MacOS -pythonw ${MAP}/helios/pipeViewer/pipe_view/argos.py --database pipeout_ --layout-file cpu_layout.alf --layout-file ${MAP}/helios/pipeViewer/scripts/gen_alf/core.alf -``` - -In this example layout, there are three main columns of locations. The left column contains the multi-cycle display; the middle and right columns show only the current cycle. The middle column is a more verbose version of the left column, and the right column contains a "directory" containing the current reorder buffer (ROB) entries. - -As before, you can change the current cycle with the controls at the bottom of the window. - -Let's examine the multi-cycle display (left column). The horizontal axis is time (cycles). At the bottom of the left column is an orange bar containing numbers. These numbers are offsets from the current cycle. For example, let's say the current cycle is 100. Then the column labeled "0" represents cycle 100, whereas the column labeled "-5" represents cycle 100-5 = 95, and the column labeled "20" represents cycle 100+20 = 120. - -The vertical axis represent different locations in the pipeline. Many of these show instructions as they flow through the pipeline, one leter per instruction. You can see instructions propagating diagonally in the display as they go from one pipeline stage to the next. - -Not all locations represent instructions. For example, the "cred" locations represent the number of credits received by that execution unit on that cycle. And in the load-store unit (LSU), locations represent load/store transactions. Note you can mouse over any location to see a pop-up corresponding to that location. - -Note a further detail. For example, look at the ROB entries at the bottom of the multi-cycle display. In the core model example, there are many ROB entries; displaying them all in full would eat up a lot of screen real estate. So we choose to display only the oldest entries in full (labeled `ROB[0]`, `ROB[1]`, etc). Beyond the oldest entries, we display the rest of the entries in a vertically-compressed "mini" format. This still gives a visual representation of the fullness of the ROB, but the letters are no longer visible. However, the transaction colors are still visible. You can still hover your mouse over these "mini" entries to see the full text. - -The middle column shows only the current cycle (corresponding to the "0" column in the multi-cycle column). The locations match the ones in the left column. - -Finally, the right column provides additional info and shows the entire ROB verbosely. You can use the middle and right columns to see at a glance more data about each transaction in the display. - -### 5. Learn how to create and edit your own multi-cycle layout - -Multi-cycle layouts are specified using a simple ad-hoc text format. By convention, these layout specification files have the `.cmd` extension. You can see example `.cmd` files in `${MAP}/helios/pipeViewer/scripts/gen_alf`. This is easily edited; to create your own multi-cycle layout, it's best to copy this example. - -This example includes three `.cmd` files. `core.cmd` is the primary file and includes two others, `vars.cmd` and `core_column.cmd`. `vars.cmd` contains general settings. `core_column.cmd` is the heart of the layout and specifies which locations to display. - -To generate a layout file (extension: `.alf`), in this directory run the script `gen_layouts` which runs the `gen_alf` script. `gen_alf` takes the `.cmd` file and creates the `.alf` layout file. - -See `core_column.cmd` to see how to specify locations. Location paths can be seen in the `*location.data` file when the pipeline database is generated. In our example, the file is `${MAP}/sparta/release/example/CoreModel/pipeout_location.data` file. diff --git a/helios/pipeViewer/argos_dumper/.gitignore b/helios/pipeViewer/argos_dumper/.gitignore deleted file mode 100644 index 4b32446a1b..0000000000 --- a/helios/pipeViewer/argos_dumper/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -InformationWriter/info.txt -data_test_0clock.dat -data_test_0index.bin -data_test_0location.dat -data_test_0record.bin diff --git a/helios/pipeViewer/argos_dumper/ArgosCollection_test.cpp b/helios/pipeViewer/argos_dumper/ArgosCollection_test.cpp deleted file mode 100644 index 8b12048818..0000000000 --- a/helios/pipeViewer/argos_dumper/ArgosCollection_test.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// -*- C++ -*- - -#include "sparta/utils/SpartaTester.hpp" - -#include "sparta/collection/Collectable.hpp" -#include "sparta/collection/PipelineCollector.hpp" - -#include "sparta/simulation/TreeNode.hpp" -#include "sparta/simulation/ClockManager.hpp" -#include "sparta/events/Event.hpp" -#include "sparta/events/EventSet.hpp" -#include "sparta/events/StartupEvent.hpp" - -#include "sparta/log/Tap.hpp" - -TEST_INIT; - -class DummyObject -{ -public: - std::string msg = ""; - -}; - -std::ostream & operator<<(std::ostream & os, const DummyObject & dumb) { - os << dumb.msg; - return os; -} - -class ObjectClk : public sparta::TreeNode -{ -public: - ObjectClk(TreeNode * node, const std::string & name) : - sparta::TreeNode(node, name, "A random pretend head node for tests" ), - pc2_var(1000), - pc1_(this, name + "0_int_manual_collectable"), - pc1_always_close_(this, name + "0_int_manual_collectable_will_close"), - pc2_(this, name + "1_int_local_collectable", &pc2_var), - pc3_(this, name + "2_dummy_collectable", &pc3_dummy), - es_(this), - ev_update_(&es_, "update", CREATE_SPARTA_HANDLER(ObjectClk, updateCollectables)) - { - pc1_.initialize(1000); - pc1_always_close_.initialize(1000); - pc3_dummy.msg = name; - - sparta::StartupEvent(node, CREATE_SPARTA_HANDLER(ObjectClk, startup)); - } - - void startup() { - ev_update_.schedule(1); - } - - void updateCollectables() - { - ++pc2_var; - pc1_.collect(pc2_var); - if(toggle & 1) { - pc1_always_close_.collectWithDuration(pc2_var, 1); - } - ++toggle; - std::stringstream ss; - ss << pc3_dummy.msg << " " << pc2_var; - pc3_dummy.msg = ss.str(); - ev_update_.schedule(1); - } - -private: - DummyObject pc3_dummy; - uint64_t pc2_var; - sparta::collection::Collectable pc1_; - sparta::collection::Collectable pc1_always_close_; - sparta::collection::Collectable pc2_; - sparta::collection::Collectable pc3_; - sparta::EventSet es_; - sparta::Event ev_update_; - uint32_t toggle = 0; -}; - - -int main() -{ - - // Start with a root - sparta::RootTreeNode root_node("root"); - sparta::RootTreeNode root_clks("clocks", - "Clock Tree Root", - root_node.getSearchScope()); - sparta::Scheduler sched; - sparta::ClockManager cm(&sched); - sparta::Clock::Handle root_clk = cm.makeRoot(&root_clks); - sparta::Clock::Handle clk_1000 = cm.makeClock("clk_1000", root_clk, 1000.0); - sparta::Clock::Handle clk_100 = cm.makeClock("clk_100", root_clk, 100.0); - sparta::Clock::Handle clk_10 = cm.makeClock("clk_10", root_clk, 10.0); - cm.normalize(); - - root_node.setClock(root_clk.get()); - - sparta::TreeNode obj1000_tn(&root_node, "obj1000", "obj1000 desc"); - sparta::TreeNode obj100_tn(&root_node, "obj100", "obj100 desc"); - sparta::TreeNode obj10_tn(&root_node, "obj10", "obj10 desc"); - - obj1000_tn.setClock(clk_1000.get()); - obj100_tn.setClock(clk_100.get()); - obj10_tn.setClock(clk_10.get()); - - // Add some objects - ObjectClk obj1000(&obj1000_tn, "level1_0"); - ObjectClk obj100 (&obj100_tn, "level1_1"); - ObjectClk obj10 (&obj10_tn, "level1_2"); - - root_node.enterConfiguring(); - root_node.enterFinalized(); - - // sparta::log::Tap scheduler_debug(sparta::TreeNode::getVirtualGlobalNode(), - // sparta::log::categories::DEBUG, std::cout); - - sparta::collection::PipelineCollector pc("testPipe", 0, root_node.getClock(), &root_node); - - sched.finalize(); - - pc.startCollection(&root_node); - - pc.printMap(); - - sched.run(100000); - pc.stopCollection(); - sched.run(100000); - pc.startCollection(&root_node); - sched.run(100000); - - pc.stopCollection(&root_node); - pc.destroy(); - - root_node.enterTeardown(); - root_clks.enterTeardown(); - - REPORT_ERROR; - return ERROR_CODE; - -} diff --git a/helios/pipeViewer/argos_dumper/CMakeLists.txt b/helios/pipeViewer/argos_dumper/CMakeLists.txt deleted file mode 100644 index 3fad07fd27..0000000000 --- a/helios/pipeViewer/argos_dumper/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -project(Argos_dumper) - -set (CMAKE_CXX_STANDARD 17) -set (CMAKE_CXX_STANDARD_REQUIRED ON) - -# Enable testing -enable_testing () - -add_executable(Argos_dumper ArgosCollection_test.cpp) -add_test (NAME Argos_dumper_RUN COMMAND Argos_dumper) - -target_include_directories(Argos_dumper PRIVATE ${CMAKE_SOURCE_DIR}/pipeViewer/pipe_view) -target_link_libraries (Argos_dumper SPARTA::sparta) - -add_subdirectory(DatabaseDump) diff --git a/helios/pipeViewer/argos_dumper/DatabaseDump/CMakeLists.txt b/helios/pipeViewer/argos_dumper/DatabaseDump/CMakeLists.txt deleted file mode 100644 index ce1aaedfbb..0000000000 --- a/helios/pipeViewer/argos_dumper/DatabaseDump/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -project(Argosdumper) - -set (CMAKE_CXX_STANDARD 17) -set (CMAKE_CXX_STANDARD_REQUIRED ON) - -add_executable(DBDumper Database_dumper.cpp) - -target_include_directories(DBDumper PRIVATE ${CMAKE_SOURCE_DIR}/pipeViewer/pipe_view) -target_link_libraries (DBDumper SPARTA::sparta) - -add_test (NAME DBDumperSmokeTestingDB COMMAND DBDumper db_pipeout/pipeout) diff --git a/helios/pipeViewer/argos_dumper/DatabaseDump/Database_dumper.cpp b/helios/pipeViewer/argos_dumper/DatabaseDump/Database_dumper.cpp deleted file mode 100644 index 22ff2cbd25..0000000000 --- a/helios/pipeViewer/argos_dumper/DatabaseDump/Database_dumper.cpp +++ /dev/null @@ -1,218 +0,0 @@ - -#include "transactiondb/src/Reader.hpp" -#include "transactiondb/src/PipelineDataCallback.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/SpartaTester.hpp" -#include -#include -#include -#include - -/** - * \file Database_dumper.cpp - * \brief dump an pipeViewer database to a human readable format. - * Instructions, run ./ArgosDumper - * the database prefix should be the same prefix passed to the simulator when creating the database - * example - * ./ArgosDumper ../../sim/data_ > out.csv - * - * I recommend dumping the output to an *.csv file. Then in open office you can set the file to - * recognize spaces as new columns. If you do this, then you will have a nice output formatted table - * that should be a little easier to read/manipulate using sort functionality and what not in office. - */ -namespace sparta -{ -namespace pipeViewer -{ - class DumpCallback : public PipelineDataCallback - { - private: - bool merge_transactions = false; - bool sort_by_end_time = false; - - std::unordered_map > continued_transactions; - std::ostringstream& output_buffer; - - // Returns whether the given transaction is split across a heartbeat - bool isContinued(const transaction_t *t) const - { - return (t->flags & CONTINUE_FLAG) != 0; - } - - // Prints a transaction to output_buffer - // This is used for the default sort mode (by transaction ID) when merging transactions - template - void printToBuf(const T* t, void(*print_func)(const T*, std::ostream &)) const - { - print_func(t, output_buffer); - } - - template - void genericTransactionHandler(const T* t, void(*print_func)(const T*, std::ostream &)) - { - // If there's no merging, then we can just print the transaction and be done - if(!merge_transactions) - { - print_func(t, std::cout); - return; - } - - uint16_t loc_id = t->location_ID; - - std::stringstream sstr; - - // This transaction has already been encountered and is split across a heartbeat boundary - if(continued_transactions.count(loc_id)) - { - // Update the saved transaction with the latest end time - auto* cont_trans = static_cast(continued_transactions.at(loc_id).get()); - cont_trans->time_End = t->time_End; - - // If this transaction isn't continued, then it's the last one in the chain. So, we can print it and delete the entry. - if(!isContinued(t)) - { - if(!sort_by_end_time) - { - // This will be out of order with respect to transaction ID, so print it to the buffer instead of stdout - printToBuf(cont_trans, print_func); - } - else - { - // We're sorting by end time, so we can just print directly to stdout - print_func(cont_trans, std::cout); - } - - continued_transactions.erase(loc_id); - } - } - else - { - // This is the first part of a transaction that has been split across a heartbeat boundary - if(isContinued(t)) - { - continued_transactions[loc_id] = std::make_unique(*t); - } - // This transaction isn't split at all - else - { - if(!sort_by_end_time) - { - // This will be out of order with respect to transaction ID, so print it to the buffer instead of stdout - printToBuf(t, print_func); - } - else - { - // We're sorting by end time, so we can just print directly to stdout - print_func(t, std::cout); - } - } - } - - - } - - static void printInst(const instruction_t *t, std::ostream & os) - { - os << std::setbase(10); - os << "*instruction* " << t->transaction_ID << " @ " << t->location_ID << " start: " << t->time_Start << " end: "<time_End; - os << " opcode: " << std::setbase(16) << std::showbase << t->operation_Code << " vaddr: " << t->virtual_ADR; - os << " real_addr: " << t->real_ADR << std::endl; - } - - void foundInstRecord(const instruction_t*t) final - { - genericTransactionHandler(t, &printInst); - } - - static void printMemOp(const memoryoperation_t *t, std::ostream & os) - { - os << std::setbase(10); - os << "*memop* " << t->transaction_ID << " @ " << t->location_ID << " start: " << t->time_Start << " end: "<time_End; - os << std::setbase(16) << std::showbase << " vaddr: " << t->virtual_ADR; - os << " real_addr: " << t->real_ADR << std::endl; - } - - void foundMemRecord(const memoryoperation_t*t) final - { - genericTransactionHandler(t, printMemOp); - } - - static void printPairOp(const pair_t * p, std::ostream & os) { - os << "*pair* @ " << p->location_ID << " "; - for (uint32_t i = 0; i < p->nameVector.size(); ++i) { - os << p->nameVector[i] << "(" << p->stringVector[i] << ") "; - } - os << "start: " << p->time_Start << " end: " << p->time_End; - os << std::endl; - } - - void foundPairRecord(const pair_t * t) final { - genericTransactionHandler(t, printPairOp); - } - - static void printAnnotation(const annotation_t * t, std::ostream & os) - { - os << "*annotation* " << t->transaction_ID << " @ " << t->location_ID << " start: " << t->time_Start << " end: "<time_End; - os << t->annt << std::endl; - } - - void foundAnnotationRecord(const annotation_t*t) final - { - genericTransactionHandler(t, printAnnotation); - } - - public: - DumpCallback(const bool merge, const bool sort, std::ostringstream& buffer) : - merge_transactions(merge), - sort_by_end_time(sort), - output_buffer(buffer) - { - } - - }; - -}//namespace sparta -}//namespace pipeViewer - -void usage() -{ - std::cerr << "Usage: ArgosDumper [-h] [-m] [-s] argos_db_prefix" << std::endl - << "Options:" << std::endl - << "\t-h\t\tPrint usage info" << std::endl - << "\t-m\t\tMerge transactions that were split by a heartbeat interval" << std::endl - << "\t-s\t\tSort output by transaction end time" << std:: endl; -} - -int main(int argc, char ** argv) -{ - std::ostringstream output_buffer; - std::string db_path = "db_pipeout/pipeout"; - if (argc > 1) { - db_path = argv[1]; - } - - const bool merge_transactions = false; - const bool sort_by_end_time = true; - - auto reader = sparta::pipeViewer::Reader::construct(db_path, - merge_transactions, - sort_by_end_time, - output_buffer); - - // Get data - reader.getWindow(reader.getCycleFirst(), reader.getCycleLast()); - - // If we're sorting by transaction ID, then we need to flush the buffer - // In non-merging mode, sorting by transaction ID and end time should be identical - if(merge_transactions && !sort_by_end_time) - { - std::cout << output_buffer.str(); - } - - std::cout << "range: [" << reader.getCycleFirst() << ", " << reader.getCycleLast() << "]" << std::endl; - - // Check indices - std::cout << "Checking indices:" << std::endl; - reader.dumpIndexTransactions(); - -} diff --git a/helios/pipeViewer/argos_dumper/DatabaseDump/db_pipeout/pipeoutsimulation.info b/helios/pipeViewer/argos_dumper/DatabaseDump/db_pipeout/pipeoutsimulation.info deleted file mode 100644 index 63d74fd3b6..0000000000 --- a/helios/pipeViewer/argos_dumper/DatabaseDump/db_pipeout/pipeoutsimulation.info +++ /dev/null @@ -1,5 +0,0 @@ -Pipeline Collection files generated from Simulator sparta_core_example - -Simulation started at: Mon Jun 25 10:36:17 2018 -Simulation ended at: Mon Jun 25 10:36:17 2018 -Heartbeat interval: 0 ticks diff --git a/helios/pipeViewer/argos_dumper/testPipe_layout.alf b/helios/pipeViewer/argos_dumper/testPipe_layout.alf deleted file mode 100644 index 5354685d8a..0000000000 --- a/helios/pipeViewer/argos_dumper/testPipe_layout.alf +++ /dev/null @@ -1,338 +0,0 @@ ---- -- Content: annotation - LocationString: root.obj1000.level1_0.level1_02_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 60) - t_offset: 0 -- Content: annotation - LocationString: root.obj1000.level1_0.level1_01_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 80) - t_offset: 0 -- Content: annotation - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 100) - t_offset: 0 -- Content: annotation - LocationString: root.obj100.level1_1.level1_12_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 200) - t_offset: 0 -- Content: annotation - LocationString: root.obj100.level1_1.level1_11_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 220) - t_offset: 0 -- Content: annotation - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 240) - t_offset: 0 -- Content: annotation - LocationString: root.obj10.level1_2.level1_22_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 340) - t_offset: 0 -- Content: annotation - LocationString: root.obj10.level1_2.level1_21_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 360) - t_offset: 0 -- Content: annotation - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 380) - t_offset: 0 -- Content: clock - LocationString: root.obj100.level1_1.level1_12_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (860, 20) - position: (460, 180) - t_offset: 0 -- Content: clock - LocationString: root.obj10.level1_2.level1_22_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (860, 20) - position: (460, 320) - t_offset: 0 -- Content: clock - LocationString: root.obj1000.level1_0.level1_02_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (860, 20) - position: (460, 40) - t_offset: 0 -- Content: cycle - LocationString: root.obj100.level1_1.level1_12_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (840, 20) - position: (720, 180) - t_offset: 0 -- Content: cycle - LocationString: root.obj1000.level1_0.level1_02_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (840, 20) - position: (720, 40) - t_offset: 0 -- Content: cycle - LocationString: root.obj10.level1_2.level1_22_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (840, 20) - position: (720, 320) - t_offset: 0 -- Content: loc_id - LocationString: root.obj1000.level1_0.level1_02_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 60) - t_offset: 0 -- Content: loc_id - LocationString: root.obj1000.level1_0.level1_01_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 80) - t_offset: 0 -- Content: loc_id - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 100) - t_offset: 0 -- Content: loc_id - LocationString: root.obj100.level1_1.level1_12_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 200) - t_offset: 0 -- Content: loc_id - LocationString: root.obj100.level1_1.level1_11_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 220) - t_offset: 0 -- Content: loc_id - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 240) - t_offset: 0 -- Content: loc_id - LocationString: root.obj10.level1_2.level1_22_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 340) - t_offset: 0 -- Content: loc_id - LocationString: root.obj10.level1_2.level1_21_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 360) - t_offset: 0 -- Content: loc_id - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 380) - t_offset: 0 -- Content: loc - LocationString: root.obj1000.level1_0.level1_02_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 60) - t_offset: 0 -- Content: loc - LocationString: root.obj1000.level1_0.level1_01_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 80) - t_offset: 0 -- Content: loc - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 100) - t_offset: 0 -- Content: loc - LocationString: root.obj100.level1_1.level1_12_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 200) - t_offset: 0 -- Content: loc - LocationString: root.obj100.level1_1.level1_11_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 220) - t_offset: 0 -- Content: loc - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 240) - t_offset: 0 -- Content: loc - LocationString: root.obj10.level1_2.level1_22_dummy_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 340) - t_offset: 0 -- Content: loc - LocationString: root.obj10.level1_2.level1_21_int_local_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 360) - t_offset: 0 -- Content: loc - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 380) - t_offset: 0 -- Content: loc - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 120) - t_offset: 0 -- Content: loc - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 260) - t_offset: 0 -- Content: loc - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (360, 20) - position: (60, 400) - t_offset: 0 -- Content: loc_id - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 120) - t_offset: 0 -- Content: loc_id - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 260) - t_offset: 0 -- Content: loc_id - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (20, 20) - position: (20, 400) - t_offset: 0 -- Content: annotation - LocationString: root.obj1000.level1_0.level1_00_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 120) - t_offset: 0 -- Content: annotation - LocationString: root.obj100.level1_1.level1_10_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 260) - t_offset: 0 -- Content: annotation - LocationString: root.obj10.level1_2.level1_20_int_manual_collectable_will_close - annotation: - caption: - color: (128, 128, 128) - dimensions: (1100, 20) - position: (460, 400) - t_offset: 0 -... diff --git a/helios/pipeViewer/argos_dumper/testPipeline.alf b/helios/pipeViewer/argos_dumper/testPipeline.alf deleted file mode 100644 index 70a8e741cc..0000000000 --- a/helios/pipeViewer/argos_dumper/testPipeline.alf +++ /dev/null @@ -1,282 +0,0 @@ ---- -- Content: auto_color_annotation - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[0] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (145, 28) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[1] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (145, 42) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[2] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (145, 56) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[3] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (145, 70) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[4] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (145, 84) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[0] - auto_color_basis: - caption: myFifthSpartaPipeline[0] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (126, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 28) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[1] - auto_color_basis: - caption: myFifthSpartaPipeline[1] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (126, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 42) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[2] - auto_color_basis: - caption: myFifthSpartaPipeline[2] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (126, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 56) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[3] - auto_color_basis: - caption: myFifthSpartaPipeline[3] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (126, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 70) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.myFifthSpartaPipeline.myFifthSpartaPipeline[4] - auto_color_basis: - caption: myFifthSpartaPipeline[4] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (126, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 84) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.STWR_Pipe.STWR_Pipe[0] - auto_color_basis: - caption: STWR_Pipe[0] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 126) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.STWR_Pipe.STWR_Pipe[1] - auto_color_basis: - caption: STWR_Pipe[1] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 140) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.STWR_Pipe.STWR_Pipe[2] - auto_color_basis: - caption: STWR_Pipe[2] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 154) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.STWR_Pipe.STWR_Pipe[3] - auto_color_basis: - caption: STWR_Pipe[3] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 168) - t_offset: 0 - tooltip: - type: box -- Content: caption - LocationString: top.STWR_Pipe.STWR_Pipe[4] - auto_color_basis: - caption: STWR_Pipe[4] - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (14, 182) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.STWR_Pipe.STWR_Pipe[0] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (154, 126) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.STWR_Pipe.STWR_Pipe[1] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (154, 140) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.STWR_Pipe.STWR_Pipe[2] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (154, 154) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.STWR_Pipe.STWR_Pipe[3] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (154, 168) - t_offset: 0 - tooltip: - type: box -- Content: auto_color_annotation - LocationString: top.STWR_Pipe.STWR_Pipe[4] - auto_color_basis: - caption: - color: (128, 128, 128) - color_basis_type: string_key - dimensions: (120, 14) - on_cycle_changed: - on_init: - on_update: - position: (154, 182) - t_offset: 0 - tooltip: - type: box -... diff --git a/helios/pipeViewer/mypy.ini b/helios/pipeViewer/mypy.ini deleted file mode 100644 index fb13aa556f..0000000000 --- a/helios/pipeViewer/mypy.ini +++ /dev/null @@ -1,31 +0,0 @@ -[mypy] -mypy_path = stubs -disallow_untyped_defs = True -check_untyped_defs = True - -[mypy-Cython.*] -ignore_missing_imports = True - -[mypy-matplotlib.*] -ignore_missing_imports = True - -[mypy-wx] -ignore_missing_imports = True - -[mypy-wx.adv] -ignore_missing_imports = True - -[mypy-wx.grid] -ignore_missing_imports = True - -[mypy-wx.lib.*] -ignore_missing_imports = True - -[mypy-wx.py.*] -ignore_missing_imports = True - -[mypy-daltonize] -ignore_missing_imports = True - -[mypy-AppKit] -ignore_missing_imports = True diff --git a/helios/pipeViewer/pipe_view/.gitignore b/helios/pipeViewer/pipe_view/.gitignore deleted file mode 100644 index d66f281012..0000000000 --- a/helios/pipeViewer/pipe_view/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -doc -*.so diff --git a/helios/pipeViewer/pipe_view/__init__.py b/helios/pipeViewer/pipe_view/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/argos.py b/helios/pipeViewer/pipe_view/argos.py deleted file mode 100755 index 9cfedddf83..0000000000 --- a/helios/pipeViewer/pipe_view/argos.py +++ /dev/null @@ -1,468 +0,0 @@ -#!/usr/bin/env python3 - -# @package argos.py -# @brief Startup file for argos -# -# This file must be run as a script and not imported - -from __future__ import annotations -import argparse -import logging -from logging import info, error, warn -import os -import pstats -import re -import sys -from typing import Dict, List, Optional, Tuple, Union, cast -import wx -import cProfile - -info("currently with debug mode: %s", __debug__) - -# Check Interpreter Version -assert sys.version_info[0] == 3 and sys.version_info[1] >= 6, \ - f'Python interpreter {sys.argv[0]} version ({sys.version}) ' \ - 'is too old. This version is not supported but might still ' \ - 'work if you disable this assertion' - -# Path for built argos cython libraries -added_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'core/lib') -if added_path is not None: - sys.path.insert(0, added_path) # Add temporary search path - -corePath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'core') -sys.path.insert(0, corePath) - -# Argos imports need to go below this line (some depend on the path to core.so) - -from pipe_view.gui import autocoloring # noqa: E402 -from pipe_view.gui.dialogs.select_db_dlg import SelectDatabaseDlg # noqa: E402 -from pipe_view.gui.dialogs.select_layout_dlg import SelectLayoutDlg # noqa: E402, E501 -from pipe_view.model.database import Database # noqa: E402 -from pipe_view.model.settings import ArgosSettings # noqa: E402 -from pipe_view.model.utilities import LogFormatter # noqa: E402 -from pipe_view.model.workspace import Workspace # noqa: E402 - -# End Argos imports - - -def main() -> None: - rc = 0 - - logging.getLogger().setLevel(logging.INFO) - streamHandler = logging.StreamHandler(sys.stderr) - streamHandler.setFormatter(LogFormatter()) - logging.getLogger().addHandler(streamHandler) - - # Regex for finding variable assignments inside a --layout-vars option - # @note Designed to handle " core_num=7 , foo = 8 ,bar= 2, cat =3 " - LAYOUT_VAR_RE = re.compile(r'\s*(?:,?\s*)?(\w+)\s*=\s*(\w+)\s*') - - parser = argparse.ArgumentParser( - description='Argos Viewer\n' - 'Visualize a transaction database collected from a simulation', - epilog='Copyright 2019', - formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - '--layout-file', - '-l', - metavar='LAYOUT_FILE', - type=str, - nargs='?', - action='append', - help='Specifies an Argos layout file to open. Each Layout file opened ' - 'corresponds to a Frame. This option can be specified any number ' - 'of times to open multiple frames' - ) - parser.add_argument( - '--layout-vars', - '-L', - metavar='VARS_DICT', - type=str, - nargs=1, - action='append', - help='Specifies a comma-separated list of "key=value" pairs ' - 'associated with one layout instance. The Nth instance of this ' - 'option corresponds to the Nth instance of --layout-file/-l. ' - 'Example: "argos -l mylayout -L "core_num=2"' - ) - parser.add_argument( - '--layout-geometry', - '-g', - metavar='GEOMETRY', - type=str, - nargs=1, - action='append', - help='Specifies a comma-separated list of geometry associated with ' - 'one layout instance in the form "w,h[,x,y]". The Nth instance ' - 'of this option corresponds to the Nth instance of ' - '--layout-file/-l. Use "0,0,0,0" to minimize the layout. ' - 'Example: "argos -l mylayout -g "800,200,0,0"' - ) - parser.add_argument( - '--resource-dir', - '-R', - metavar='DIR', - type=str, - nargs='*', - help='Specifies an additional directory in which to search for ' - 'resources used by argos layouts (e.g. image files). By default, ' - 'resources are searched for in the same directory as the ' - 'requesting layout file and then in the built-in ' - 'argos_view/resources directory' - ) - parser.add_argument( - '--database', - '-d', - metavar='DATABASE_PREFIX', - type=str, - nargs='?', - help='Selects the database to be opened. If not specified, the user ' - 'will be prompted within the application' - ) - parser.add_argument( - '--start-cycle', - '-c', - metavar='CYCLE_NUM', - type=str, - nargs='?', - help='Selects the initial cycle number in terms of clock set by ' - '(--clock). Overrides --start-tick' - ) - parser.add_argument( - '--start-tick', - '-t', - metavar='TICK_NUM', - type=str, - nargs='?', - help='Selects an initial tick number at which to display the layout(s)' - ) - parser.add_argument( - '--clock', - metavar='CLOCK_NAME', - type=str, - nargs='?', - help='Selects an initial displayed clock for each layout frame. If no ' - 'matching clock is found by this name (case insensitive), a ' - 'default is selected which is usually the root clock' - ) - parser.add_argument( - '--poll', - action='store_true', - help='Update Argos if the database file changes' - ) - parser.add_argument( - '--title-prefix', - metavar='TITLE_PREFIX', - type=str, - nargs='?', - help='Specifies a prefix for the window title' - ) - parser.add_argument( - '--title-override', - metavar='TITLE_OVERRIDE', - type=str, - nargs='?', - help='Overrides the window title with the specified string' - ) - parser.add_argument( - '--title-suffix', - metavar='TITLE_SUFFIX', - type=str, - nargs='?', - help='Specifies a suffix for the window title' - ) - parser.add_argument( - '--debug', - action='store_true', - help='Debugging and Verbose output' - ) - parser.add_argument( - '--profile', - action='store_true', - help='Profile and report where time is being spent' - ) - parser.add_argument( - '--tracemalloc', - action='store_true', - help='trace memory blocks allocated by this program' - ) - parser.add_argument( - '--quiet', - action="store_true", - help='Be less verbose, set the logging level to warning' - ) - - args = parser.parse_args() - - if args.profile and args.tracemalloc: - error("cannot enable profiling and tracemalloc at the same time") - sys.exit(1) - elif args.profile: - info("enabling profiling") - prof = cProfile.Profile(subcalls=True, builtins=True) - prof.enable() - elif args.tracemalloc: - info("enabling tracemalloc profiling") - import tracemalloc - tracemalloc.start() - - # logging setup - # tell the formatter how to identify this one - logging.getLogger().setLevel( - logging.WARNING if args.quiet else - (logging.DEBUG if args.debug else logging.INFO) - ) - - # file output - fileHandler = logging.FileHandler('argos.log') - fileHandler.setLevel(logging.DEBUG) - fileHandler.setFormatter(LogFormatter(isSmoke=False)) - logging.getLogger().addHandler(fileHandler) - - # Argos Application - class ArgosApp(wx.App): - - def OnInit(self) -> bool: - return True - - app = ArgosApp() - - settings = ArgosSettings() - # The user can specify default colorblindness and palette shuffle modes - # with these environment variables - autocoloring.BuildBrushes(settings.palette, settings.palette_shuffle) - - # Preconfigure the workspace with options - # Must be after wx.App is instantiated - ws = Workspace(settings) - - if args.resource_dir is not None: - for rd in args.resource_dir: - ws.AddUserResourceDir(rd) - - # Parse clock - select_clock: Optional[str] - if args.clock is not None: - select_clock = str(args.clock) - else: - select_clock = None - - # Determine start tick - start_tick = None - if args.start_tick is not None: - start_tick = int(args.start_tick) - - start_cycle = None - if args.start_cycle is not None: - start_cycle = int(args.start_cycle) - if start_tick is not None: - logging.warn( - '--start-cycle is overriding ' - '--start-tick because both were specified' - ) - start_tick = 0 - - dlg: wx.Dialog - # Select Database - if args.database is not None: - database_prefix = args.database - else: - # Show database-selection dialog - dlg = SelectDatabaseDlg() - dlg.Centre() - if dlg.ShowModal() == wx.CANCEL: - sys.exit(0) - dlg.Destroy() - - database_prefix = dlg.GetPrefix() - if database_prefix is None: - error('No database selected, exiting Argos') - sys.exit(1) - - # Open the Database - try: - db = Database(database_prefix, args.poll) - except IOError as ex: - error('Error opening pipeout database (prefix) "%s"', database_prefix) - error(ex) - sys.exit(1) - - if args.layout_file is not None: - layout_files = args.layout_file - else: - # Show layout-selection dialog - dlg = SelectLayoutDlg() - dlg.Centre() - if dlg.ShowModal() == wx.CANCEL: - exit(0) - dlg.Destroy() - - layout_files = [dlg.GetFilename()] - # Note: '' or None will cause an empty layout to be created - - layout_vardicts: Optional[List] - if args.layout_vars is not None: - layout_vardicts = [] - for lv in args.layout_vars: - d: Dict[str, str] = {} - layout_vardicts.append(d) - - last_match_size = [0] - - def replace(match: re.Match[str]) -> str: - # group(1) is key, group(2) is value - # group(0) is full string (x=y) - d[str(match.group(1))] = str(match.group(2)) - last_match_size[0] = len(match.group(0)) - return '' - - pos = 0 - lv = lv[0] - while pos < len(lv): - # Disregard result. No replacement is actually done - _, repls = LAYOUT_VAR_RE.subn(replace, lv[pos:], 1) - if repls > 0: - pos += last_match_size[0] - else: - if pos < len(lv): - error("Failed to evaluate layout variables argument " - "'%s' as a comma-separated list of key=value " - "pairs. Successfully parsed '%s'. " - "Unable to parse '%s'", - lv, - lv[:pos], - lv[pos:]) - sys.exit(1) - else: - layout_vardicts = None - - layout_geos: \ - Optional[List[Union[Tuple[int, int], Tuple[int, int, int, int]]]] - if args.layout_geometry is not None: - layout_geos = [] - for lgs in args.layout_geometry: - toks = [x.strip() for x in lgs[0].split(',')] - if len(toks) != 2 and len(toks) != 4: - raise ValueError( - f'Unable to convert layout frame geometry "{lgs}" to a ' - f'sequence of 2 or 4 integers. {len(toks)} tokens found' - ) - - try: - w = int(toks[0]) - h = int(toks[1]) - except ValueError as ex: - raise ValueError( - f'Unable to convert layout frame geometry "{lgs}" to a ' - f'sequence of 2 or 4 integers: {ex}' - ) - - if len(toks) == 4: - try: - x = int(toks[2]) - y = int(toks[3]) - except ValueError as ex: - raise ValueError( - f'Unable to convert layout frame geometry "{lgs}" to ' - f'a sequence of 2 or 4 integers: {ex}' - ) - layout_geos.append((w, h, x, y)) - elif len(toks) == 2: - layout_geos.append((w, h)) - else: - assert False # Should be covered by if statement above - else: - layout_geos = None - - # Launch Layout Frames - assert layout_files is not None - - for layout_idx, lf in enumerate(layout_files): - if lf is None: - lf = '' # Empty layout - - if layout_vardicts is not None and layout_idx < len(layout_vardicts): - loc_vars = layout_vardicts[layout_idx] - else: - loc_vars = {} - - frame = ws.OpenLayoutFrame(lf, - db, - start_tick, - args.poll, - args.title_prefix, - args.title_override, - args.title_suffix, - loc_vars, - norefresh=True) - - refreshed = False - if layout_geos is not None and layout_idx < len(layout_geos): - lg = layout_geos[layout_idx] - # Difference between actual size and window size (e.g. menu bar). - # Use this to offset user's dimensions to line things up as - # expected (no overlap). This may break other systems but they can - # always just turn off geometry specification as a temoprary - # workaround. - # There is complimentary logic in Argos_Menu.OnFrameInfo - wdiff = frame.GetSize()[0] - frame.GetClientSize()[0] - hdiff = frame.GetSize()[1] - frame.GetClientSize()[1] - if len(lg) == 4: - lg = cast(Tuple[int, int, int, int], lg) - if lg == (0, 0, 0, 0): - frame.Iconize() - else: - w, h, x, y = lg - frame.SetRect(wx.Rect(x, y, w - wdiff, h - hdiff)) - elif len(lg) == 2: - lg = cast(Tuple[int, int], lg) - w, h = lg - frame.SetSize((w, h)) - else: - error("layout_geos is not 2 or 4") - sys.exit(-1) - - if select_clock: - if not frame.SetDisplayClock(select_clock, False): - clock_mgr = frame.GetContext().dbhandle.database.clock_manager - warn( - f"Failed to select display clock {select_clock}. " - f'No such clock for frame "{frame.GetTitle()}". ' - '--start-cycle will not be applied if set. ' - "Available clocks are: " - f"{', '.join(c.name for c in clock_mgr.getClocks())}" - ) - elif start_cycle is not None: - frame.GoToCycle(start_cycle) # Implies a frame refresh - refreshed = True - - frame.Show(True) - - if not refreshed: - # Updates metadata so everything is good on the first rendering - frame.GetContext().RefreshFrame() - - # Run until close - app.MainLoop() - - if args.profile: - prof.disable() - overallStats = pstats.Stats(prof) - overallStats = overallStats.sort_stats("cumulative") - overallStats.reverse_order() - overallStats.print_stats() - elif args.tracemalloc: - snapshot = tracemalloc.take_snapshot() - stats = snapshot.statistics('lineno', cumulative=True) - for stat in stats: - info(stat) - - ws.Cleanup() - sys.exit(rc) - - -if __name__ == '__main__': - main() diff --git a/helios/pipeViewer/pipe_view/core/.gitignore b/helios/pipeViewer/pipe_view/core/.gitignore deleted file mode 100644 index 2a15923b50..0000000000 --- a/helios/pipeViewer/pipe_view/core/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/core.cpp diff --git a/helios/pipeViewer/pipe_view/core/__init__.pyi b/helios/pipeViewer/pipe_view/core/__init__.pyi deleted file mode 100644 index 1fe1792182..0000000000 --- a/helios/pipeViewer/pipe_view/core/__init__.pyi +++ /dev/null @@ -1,42 +0,0 @@ -# Stub definitions for the renderer core library -# Cython isn't very good at generating these automatically yet - -from typing import Dict, Optional, Tuple, Union -import wx - -from ..gui.layout_canvas import Layout_Canvas -from ..model.element import Element -from ..model.extension_manager import ExtensionManager - -EXPR_NAMESPACE: dict -debug: function -error: function -info: function - -class Renderer: - def __init__(self) -> None: ... - def drawElements(self, dc: wx.DC, canvas: Layout_Canvas, tick: int) -> None: ... - def drawInfoRectangle(self, - tick: int, - element: Element, - dc: wx.DC, - canvas: Layout_Canvas, - rect: Union[wx.Rect, Tuple[int, int, int, int]], - annotation: Optional[Union[int, str]], - missing_needed_loc: bool, - content_type: str, - auto_color: Tuple[str, str], # type, basis - clip_x: Tuple[int, int], # (start, width) - schedule_settings: Optional[Tuple[int, int]] = None, - short_format: Optional[str] = None) -> None: ... - def parseAnnotationAndGetColor(self, - string_to_display: str, - content_type: str, - field_type: Optional[str] = None, - field_string: Optional[str] = None) -> Tuple[str, wx.Brush]: ... - def setBrushes(self, reason_brushes: Dict[int, wx.Brush], background_brushes: Dict[int, wx.Brush]) -> None: ... - def setExtensions(self, extensions: ExtensionManager) -> None: ... - def setFontFromDC(self, dc: wx.DC) -> None: ... - -def extract_value(s: str, key: str, separators: str = '=:', skip_chars: int = 0, not_found: str = '') -> str: ... -def get_argos_version() -> int: ... diff --git a/helios/pipeViewer/pipe_view/core/src/common.pxd b/helios/pipeViewer/pipe_view/core/src/common.pxd deleted file mode 100644 index 11c67ea62f..0000000000 --- a/helios/pipeViewer/pipe_view/core/src/common.pxd +++ /dev/null @@ -1,32 +0,0 @@ -## @package Common files for transction database interface - -cdef extern from "stdint.h": - - ctypedef long uint8_t - ctypedef long uint16_t - ctypedef long uint32_t - ctypedef long uint64_t - ctypedef long int8_t - ctypedef long int16_t - ctypedef long int32_t - ctypedef long int64_t - -cdef extern from "cstring" namespace "std": - - ctypedef int size_t - - cdef size_t strlen(char*) - -cdef extern from "helpers.h" namespace "pipeViewer": - - ctypedef long ptr_t - -cdef extern from "helpers.h": - - cdef char* PIPEVIEWER_VERSION "_PIPEVIEWER_VERSION" - -cdef extern from "string" namespace "std": - - cdef cppclass string: - string(char* c) # c is const - char* c_str() diff --git a/helios/pipeViewer/pipe_view/core/src/core.pyx b/helios/pipeViewer/pipe_view/core/src/core.pyx deleted file mode 100644 index 952ec2a041..0000000000 --- a/helios/pipeViewer/pipe_view/core/src/core.pyx +++ /dev/null @@ -1,641 +0,0 @@ -# @package PipeViewer drawing loop and support routines - -import sys -import wx -import re -import math -import colorsys -from logging import debug, error, info - -from common cimport * -from libc.stdlib cimport strtoul -from libcpp.unordered_map cimport unordered_map -from cpython.ref cimport PyObject - -cimport cython - -from libcpp cimport bool - -cdef extern from "wx/defs.h": - - ctypedef int wxCoord - -ctypedef unsigned char ChannelType - -cdef extern from "wx/colour.h": - - cdef cppclass wxColour: - wxColour() - - wxColour(ChannelType red, - ChannelType green, - ChannelType blue) - wxColour(unsigned long colRGB) - int GetRGB() - -cdef extern from "wx/bitmap.h": - - cdef cppclass wxBitmap: - wxBitmap() - -cdef extern from "wx/pen.h": - - cdef cppclass wxPen: - wxPen() - wxPen(wxColour colour, # const & - int width) # =1 - - void SetColour(wxColour colour) # const & - - ctypedef wxPen const_wxPen "wxPen const" - -cdef extern from "wx/brush.h": - - cdef cppclass wxBrush: - wxBrush() - wxBrush(wxColour colour) # const & - - void setColour(wxColour col) # const & - - ctypedef wxBrush const_wxBrush "wxBrush const" - -cdef extern from "wx/gdicmn.h": - - cdef cppclass wxSize: - wxSize() - wxSize(int xx, int yy) - - cdef cppclass wxPoint: - wxPoint() - wxPoint(int xx, int yy) - - cdef cppclass wxRect: - wxRect() - wxRect(int xx, int yy, int ww, int hh) - - cdef cppclass wxRect: - wxRect(wxPoint topLeft, # const & - wxPoint bottomRight) # const & - wxRect(wxPoint pt, # const & - wxSize size) # const & - - cdef cppclass wxRegion: - wxRegion(wxCoord x, wxCoord y, wxCoord w, wxCoord h) - wxRegion(wxPoint topLeft, # const & - wxPoint bottomRight) # const & - wxRegion(wxRect) # const & - -cdef extern from "wx/wxchar.h": - - ctypedef unsigned char wxChar - -cdef extern from "wx/string.h": - - cdef cppclass wxString: - wxString() - wxString(wxString) # const & - wxString(const char *) # const - const char * ToAscii() # const - -cdef extern from "wx/font.h": - - cdef cppclass wxFont: - wxFont() - wxFont(const wxFont & font) - - bint IsFixedWidth() # const - wxFont MakeBold() - - ctypedef wxFont const_wxFont "wxFont const" - - cdef enum wxFontWeight: - wxFONTWEIGHT_NORMAL, - wxFONTWEIGHT_LIGHT, - wxFONTWEIGHT_BOLD, - wxFONTWEIGHT_MAX - -cdef extern from "wx/dcgraph.h": - - cdef cppclass wxGCDC: - void SetFont(wxFont font) # const & - void SetPen(wxPen pen) # const & - void SetBrush(wxBrush brush) # const & - void SetBackground(wxBrush brush) # const & - - const_wxFont GetFont() # const - - const_wxBrush GetBackground() # const method - - const_wxPen GetPen() # const method - - void DrawRectangle(wxCoord x, wxCoord y, wxCoord width, wxCoord height) - - void SetClippingRegion(wxCoord x, wxCoord y, wxCoord width, wxCoord height) - - void SetClippingRegion(wxPoint pt, # const & - wxSize sz) # const & - void SetClippingRegion(wxRect rect) # const & - - void SetClippingRegion(wxRegion region) # const & - - void DestroyClippingRegion() - - void GetTextExtent(wxString text, - long * x, long * y) - # long *descent = NULL, - # long *externalLeading = NULL, - # wxFont *theFont = NULL) # const - - void DrawBitmap(wxBitmap bmp, # const & - wxCoord x, - wxCoord y, - bint useMask = false) - void DrawBitmap(wxBitmap bmp, # const & - wxPoint pt, # const & - bint useMask = false) - - void DrawText(wxString text, # const & - wxCoord x, - wxCoord y) - void DrawText(wxString text, # const & - wxPoint pt) # const & - - bint Blit(wxCoord xdest, - wxCoord ydest, - wxCoord width, - wxCoord height, - wxGCDC * source, - wxCoord xsrc, - wxCoord ysrc) # Truncated argument list - - -cdef extern from "helpers.h": - bool wxPyConvertWrappedPtr(PyObject* obj, void **ptr, const wxString& className) - wxGCDC* getDC_wrapped(PyObject* dc) except + - wxFont* getFont_wrapped(PyObject* font) except + - wxBrush* getBrush_wrapped(PyObject* brush) except + - void getTextExtent(wxGCDC* dc, long* char_width, long* char_height) - -def get_argos_version(): - return 1; - -# Extracts a value from a string \a s by its \a key. Key is separated from -# value by 0 or more spaces, then a character in \a separators, followed by 0 or -# more additional spaces. After extracing the value for the given key, the -# first \a skip_chars characters are dropped form the result -# @return Value of first match if there are any matches. Otherwise returns \a not_found -cpdef str extract_value(str s, str key, str separators = '=:', long skip_chars = 0, not_found = ''): - extractor = re.compile(f'{key}\\s*[{separators}]\\s*([^ ]*)') - matches = extractor.findall(s) - if len(matches) == 0: - return not_found - return matches[0][skip_chars:] - -cdef wxGCDC* getDC(dc): - return getDC_wrapped(dc) - -cdef wxFont* getFont(font): - return getFont_wrapped(font) - -cdef wxBrush* getBrush(brush): - return getBrush_wrapped(brush) - -# Color expression namespace -EXPR_NAMESPACE = {'re':re, 'colorsys':colorsys, 'math':math, 'extract':extract_value} - -cdef class Renderer: - - cdef unordered_map[int, wxPen] c_pens_map - cdef dict __reason_brushes - cdef dict __background_brushes - cdef object __extensions - cdef wxFont c_font - cdef wxFont c_bold_font - cdef long c_char_width - cdef long c_char_height - - # Brian Grayson prefers it to be a long time (>500 uops) before we repeat - # the same combination of color and symbol. So it's probably best for these to - # be relatively prime. Least common multiple should be much larger than 300. - cdef int NUM_REASON_COLORS - cdef int NUM_ANNOTATION_COLORS - cdef int NUM_ANNOTATION_SYMBOLS - cdef char * ANNOTATION_SYMBOL_STRING - cdef wxPen HIGHLIGHTED_PEN - cdef wxPen SEARCH_PEN - cdef wxPen HIGHLIGHTED_SEARCH_PEN - - def __cinit__(self, *args): - self.NUM_REASON_COLORS = 16 - self.NUM_ANNOTATION_COLORS = 32 - self.NUM_ANNOTATION_SYMBOLS = 52 - self.ANNOTATION_SYMBOL_STRING = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - self.HIGHLIGHTED_PEN = wxPen(wxColour(255, 0, 0), 2) - self.SEARCH_PEN = wxPen(wxColour(0, 0, 0), 2) - self.HIGHLIGHTED_SEARCH_PEN = wxPen(wxColour(0, 0, 255), 2) - - def __init__(self, *args): - pass - - def __dealloc__(self): - pass - - def __str__(self): - return '' - - def __repr__(self): - return self.__str__() - - def setBrushes(self, reason_brushes, background_brushes): - self.__reason_brushes = reason_brushes - self.__background_brushes = background_brushes - - def setExtensions(self, extensions): - self.__extensions = extensions - - def __fieldStringColorization(self, string_to_display, field_string): - # extract what to base color on - start_idx = string_to_display.find(field_string) - if start_idx != -1: - start_idx += len(field_string) - if start_idx < len(string_to_display) and string_to_display[start_idx] == '{': - # use everything inside brackets - start_idx += 1 - next_open = string_to_display.find('{', start_idx) - next_close = string_to_display.find('}', start_idx) - # while there is an open at a lower position than a close, - # skip the close and open (inside pair) - while next_open < next_close: - next_open = string_to_display.find('{', next_open + 1) - next_close = string_to_display.find('}', next_close + 1) - field_value = string_to_display[start_idx:next_close] - else: - next_idxs = filter(lambda x: x >= 0, (string_to_display.find(',', start_idx + 1), \ - string_to_display.find(' ', start_idx + 1), \ - string_to_display.find('\n', start_idx + 1))) - if not next_idxs: - field_value = string_to_display[start_idx:] - else: - next_idx = min(next_idxs) - field_value = string_to_display[start_idx:next_idx] - try: - field_num = int(field_value, 16) - except ValueError: - field_num = hash(field_value) - string_to_display = chr(self.ANNOTATION_SYMBOL_STRING[field_num % self.NUM_ANNOTATION_SYMBOLS]) - background_idx = field_num % self.NUM_ANNOTATION_COLORS - return string_to_display, self.__background_brushes[background_idx] - elif string_to_display: # not empty, just can't find - return '#', None - else: # empty string - return string_to_display, None - - # @brief Sets color of background brush and parses annotation according to type. - # - def parseAnnotationAndGetColor(self, string_to_display, content_type, field_type = None, field_string = None): - cdef char * c_seq_id_str - cdef unsigned long int c_seq_id - cdef char * c_endptr - - brush = None - - if field_string is None: - field_string = '' - #-------------------------------------------------- - # Choose brush color - # - uop seq ID is first three hex digits - # - string_to_display = str(string_to_display) - if any([content_type == 'auto_color_annotation', - content_type == 'auto_color_anno_notext', - content_type == 'auto_color_anno_nomunge']): - - if field_string: - if field_type == 'string_key' or field_type is None: - if content_type == 'auto_color_anno_nomunge': - # Preserve current annotation - _, brush = self.__fieldStringColorization(string_to_display, field_string) - elif content_type == 'auto_color_anno_notext': - string_to_display = '' # No text displayed - _, brush = self.__fieldStringColorization(string_to_display, field_string) - else: # normal annotation mode (display encoded annotation) - string_to_display, brush = self.__fieldStringColorization(string_to_display, field_string) - - elif field_type == 'python_exp': - try: - # Evaluate the user expression - info_tuple = eval(field_string, {'anno': string_to_display}, EXPR_NAMESPACE) - brush = wx.TheBrushList.FindOrCreateBrush(info_tuple[:3], wx.SOLID) # no guarantees this is fast - if len(info_tuple) > 3: - string_to_display = str(info_tuple[3]) - else: - pass # Preserve current display string - except: - error('Error: expression "%s" raised exception on input "%s":', field_string, string_to_display) - error(sys.exc_info()) - string_to_display = '!' - elif field_type == 'python_func': - func = self.__extensions.GetFunction(field_string) - if func: - try: - info_tuple = func(string_to_display) - brush = wx.TheBrushList.FindOrCreateBrush(info_tuple[:3], wx.SOLID) - if len(info_tuple) > 3: - string_to_display = str(info_tuple[3]) - else: - pass # Preserve current display string - except: - error('Error: function "%s" raised exception on input "%s":', field_string, string_to_display) - error(sys.exc_info()) - string_to_display = '!' - else: - error('Error: function "%s" can not be loaded.', field_string) - if brush is None: - brush = wx.TheBrushList.FindOrCreateBrush(wx.WHITE, wx.SOLID) - return string_to_display, brush - else: - seq_id_str = string_to_display[:3].strip() - if (len(seq_id_str) >= 3 and seq_id_str[0] == 'R'): - #-------------------------------------------------- - # This is a "reason" instead of a "uop" - # - seq_id_str = seq_id_str[1:] # Ignore 'R' - byte_str = seq_id_str.encode('UTF-8') - c_seq_id_str = byte_str - c_seq_id = strtoul(c_seq_id_str, & c_endptr, 16) # hex - - if c_endptr == c_seq_id_str + len(seq_id_str): - #-------------------------------------------------- - # We found a valid seq id, so use the new colorization method - # - - if (content_type != 'auto_color_anno_nomunge'): - string_to_display = string_to_display[2:] - return string_to_display, self.__reason_brushes[c_seq_id % self.NUM_REASON_COLORS] - elif seq_id_str: - #-------------------------------------------------- - # This is not a "reason" and may be a "uop" - # - byte_str = seq_id_str.encode('UTF-8') - c_seq_id_str = byte_str - c_seq_id = strtoul(c_seq_id_str, & c_endptr, 16) # hex - - if c_endptr == c_seq_id_str + len(seq_id_str): - #-------------------------------------------------- - # We found a valid seq id, so use the new colorization method - # - if (content_type != 'auto_color_anno_nomunge'): - c_symbol = self.ANNOTATION_SYMBOL_STRING[c_seq_id % self.NUM_ANNOTATION_SYMBOLS] - string_to_display = chr(c_symbol) + string_to_display[3:] - return string_to_display, self.__background_brushes[c_seq_id % self.NUM_ANNOTATION_COLORS] - else: - if all([content_type == 'caption', - len(string_to_display) >= 3, - string_to_display[:2] == 'C=']): - byte_str = string_to_display[2].encode('UTF-8') - c_seq_id_str = byte_str - c_seq_id = strtoul(c_seq_id_str, & c_endptr, 16) # hex - - if c_endptr != c_seq_id_str: - #-------------------------------------------------- - # We found a valid seq id, so use the new colorization method - # - - string_to_display = string_to_display[4:] - return string_to_display, self.__reason_brushes[c_seq_id % self.NUM_REASON_COLORS] - return string_to_display, wx.TheBrushList.FindOrCreateBrush(wx.WHITE, wx.SOLID) - - def setFontFromDC(self, dc): - py_font = dc.GetFont() - self.c_font = getFont(py_font)[0] - self.c_bold_font = wxFont(self.c_font) - self.c_bold_font.MakeBold() - - if self.c_font.IsFixedWidth(): - getTextExtent(getDC(dc), &self.c_char_width, &self.c_char_height) - - def drawInfoRectangle(self, - tick, - element, - dc, - canvas, - rect, - annotation, - missing_needed_loc, - content_type, - auto_color, # type, basis - clip_x, # (start, width) - schedule_settings = None, - short_format = None): - # schedule_settings: (period_width, 0/1/2 (none/dots/boxed)) - cdef wxGCDC * c_dc = getDC(dc) - - cdef int c_x - cdef int c_y - cdef int c_w - cdef int c_h - cdef int x_offs - - cdef char * c_str - - cdef char c_tmp_char - - cdef char c_symbol - - cdef int c_x_adj - cdef int c_y_adj - - cdef int c_num_chars - cdef int c_content_str_len - cdef wxPen old_pen - - if short_format is None: - short_format = '' - x_offs = 0 - - c_x, c_y, c_w, c_h = rect - - if missing_needed_loc: - # Missing location but required one to display. Show with grey hatched background - string_to_display = '' - brush = wx.TheBrushList.FindOrCreateBrush((160, 160, 160), wx.CROSSDIAG_HATCH) # no guarantees this is fast - highlighted = False - search_result = False - else: - record = canvas.GetTransactionColor(annotation, content_type, auto_color[0], auto_color[1]) - if record: - string_to_display, brush, highlighted, search_result, _, _ = record - else: - string_to_display, brush, highlighted, search_result = canvas.AddColoredTransaction(annotation, content_type, auto_color[0], auto_color[1], tick, element) - - if highlighted or search_result: - old_pen = c_dc.GetPen() - c_dc.SetFont(self.c_bold_font) - c_w -= 1 - c_h -= 1 - c_x += 1 - c_y += 1 - if highlighted and search_result: - c_dc.SetPen(self.HIGHLIGHTED_SEARCH_PEN) - elif search_result: - c_dc.SetPen(self.SEARCH_PEN) - elif highlighted: - c_dc.SetPen(self.HIGHLIGHTED_PEN) - - # Graph C pointer to brush - c_dc.SetBrush(getBrush(brush)[0]) - - if content_type == 'image': - # Draw an image - #dc.DrawBitmap(image, c_x, c_y) - pass - else: # auto - # Draw text clipped to this element - - # Parameters to easily shift the text within a cell. - c_y_adj = 0 - c_x_adj = 1 - # schedule line drawing code: strict cutting of over-flowing elements - if schedule_settings: - # clip long elements - # don't worry about rendering at -20 or less left - if c_x + 30 < clip_x[0]: - x_offs = c_x - clip_x[0] - c_w = c_w + x_offs # can only use upper range - c_x = clip_x[0] - - if c_w > clip_x[1]: - c_w = clip_x[1] + 30 - - # FILL rectangle - c_dc.DrawRectangle(c_x, c_y, c_w, c_h) - - period_width, div_type = schedule_settings - if content_type != 'auto_color_anno_notext' and \ - string_to_display and \ - period_width >= self.c_char_width: - - number_of_divs = c_w / period_width - - # draw text - if div_type == 10 or short_format == 'multi_char': # RULER - byte_str = string_to_display.encode('UTF-8') - c_str = byte_str - if 0 <= -x_offs < self.c_char_width * len(string_to_display): - c_dc.DrawText(wxString(c_str), < wxCoord > c_x + c_x_adj + x_offs, < wxCoord > c_y + c_y_adj) - else: - byte_str = string_to_display[0].encode('UTF-8') - c_str = byte_str - if 0 <= -x_offs < self.c_char_width: - c_dc.DrawText(wxString(c_str), < wxCoord > c_x + c_x_adj + x_offs, < wxCoord > c_y + c_y_adj) - - end_coord = c_x + c_w - period_width / 2.0 - if div_type == 1: - y_coord = c_y + c_y_adj + 5 - curr_x = c_x + c_x_adj + x_offs - 1 + period_width * 1.5 - while curr_x < end_coord: - c_dc.DrawRectangle(< wxCoord > curr_x, < wxCoord > y_coord, 2, 2) - curr_x += period_width - elif div_type == 2: - y_coord = c_y + c_y_adj - curr_x = c_x + c_x_adj + x_offs - if x_offs == 0: - curr_x += period_width - while curr_x < end_coord: - c_dc.DrawText(wxString(c_str), < wxCoord > (curr_x), < wxCoord > (y_coord)) - curr_x += period_width - else: - # regular draw - c_dc.DrawRectangle(c_x, c_y, c_w, c_h) - if content_type != 'auto_color_anno_notext': - string_to_display = self.__truncateString(string_to_display, c_w) - byte_str = string_to_display.encode('UTF-8') - c_str = byte_str - if len(string_to_display): - c_dc.DrawText(wxString(c_str), < wxCoord > c_x + c_x_adj, < wxCoord > c_y + c_y_adj) - if highlighted or search_result: - c_dc.SetPen(old_pen) - c_dc.SetFont(self.c_font) - - def __truncateString(self, string_to_display, c_w): - # Truncate text if possible - if self.c_char_width: - num_chars = int(1 + (c_w / self.c_char_width)) - str_len = len(string_to_display) - if num_chars < str_len: - return string_to_display[:num_chars] - return string_to_display - return string_to_display - - def drawElements(self, dc, canvas, tick): - """ - Draw elements in the list to the given dc - """ - - elements = canvas.GetDrawPairs() # Uses bounds - xoff, yoff = canvas.GetRenderOffsets() - _, reason_brushes = canvas.GetBrushes() - - cdef wxGCDC * c_dc = getDC(dc) - - cdef wxColour c_color - cdef wxPen c_pen - cdef int c_x - cdef int c_y - cdef int c_w - cdef int c_h - cdef int c_yoff = yoff - cdef int c_xoff = xoff - cdef int color_rgb - cdef int64_t c_vis_tick = canvas.GetVisibilityTick() - - needs_brush_purge = False - for pair in elements: - if pair.GetVisibilityTick() != c_vis_tick: - continue # Skip drawing - not visible (probably not on screen) - - string_to_display = pair.GetVal() - e = pair.GetElement() - - # hack: probably should be handled during update calls - if e.NeedsDatabase() and e.BrushCacheNeedsPurging(): - e.SetBrushesWerePurged() - needs_brush_purge = True - - if not e.ShouldDraw(): - continue - - dr = e.GetDrawRoutine() - if dr: - # Custom routine. - dr(e, pair, dc, canvas, tick) - continue - - (c_x, c_y), (c_w, c_h) = e.GetProperty('position'), e.GetProperty('dimensions') - (c_x, c_y) = (c_x - c_xoff, c_y - c_yoff) - - color = e.GetProperty('color') - content_type = e.GetProperty('Content') - - c_color = wxColour(int(color[0]), int(color[1]), int(color[2])) - - color_rgb = c_color.GetRGB() - if self.c_pens_map.count(color_rgb): - c_dc.SetPen(self.c_pens_map[color_rgb]) - else: - c_pen = wxPen(c_color, 1) - self.c_pens_map[color_rgb] = c_pen - c_dc.SetPen(c_pen) - - c_dc.SetClippingRegion(c_x, c_y, c_w, c_h) - self.drawInfoRectangle(tick, e, dc, canvas, - (c_x, c_y, c_w, c_h), - string_to_display, - pair.IsMissingLocation(), - content_type, - (e.GetProperty('color_basis_type'), e.GetProperty('auto_color_basis')), - (c_x, c_w)) - c_dc.DestroyClippingRegion() - - if needs_brush_purge: - canvas.PurgeBrushCache() # will take care of new render call diff --git a/helios/pipeViewer/pipe_view/core/src/helpers.h b/helios/pipeViewer/pipe_view/core/src/helpers.h deleted file mode 100644 index 8ed8bd1208..0000000000 --- a/helios/pipeViewer/pipe_view/core/src/helpers.h +++ /dev/null @@ -1,74 +0,0 @@ -/*! - * \file helpers.h - * \brief Helpers for Cython wrappers - */ - -#ifndef __HELPERS_H__ -#define __HELPERS_H__ - -#include -#include -#include - -#include "wxPython/sip.h" - -#ifdef HAVE_SSIZE_T - #undef HAVE_SSIZE_T -#endif -#include "wxPython/wxpy_api.h" - -#include "wx/dcgraph.h" - -#ifndef ARGOS_VERSION -#define ARGOS_VERSION unknown -#endif - -class wxConversionException : public std::exception { - private: - std::string msg_; - - public: - wxConversionException(const wxString& class_name) { - msg_ = "Failed to convert object to type " + class_name.ToStdString(); - } - - const char* what() const noexcept final { - return msg_.c_str(); - } -}; - -template -inline T* getWrappedWXType(PyObject* py_type, const wxString& class_name) { - T* c_type; - if(!wxPyConvertWrappedPtr(py_type, reinterpret_cast(&c_type), class_name)) { - throw wxConversionException(class_name); - } - return c_type; -} - -#define UNWRAP_WX(type, obj) \ - static const wxString CLASS_TYPE(#type); \ - return getWrappedWXType(obj, CLASS_TYPE); - -inline wxGCDC* getDC_wrapped(PyObject* dc) { - UNWRAP_WX(wxGCDC, dc) -} - -inline wxFont* getFont_wrapped(PyObject* font) { - UNWRAP_WX(wxFont, font) -} - -inline wxBrush* getBrush_wrapped(PyObject* brush) { - UNWRAP_WX(wxBrush, brush) -} - -inline void getTextExtent(wxGCDC* dc, long* char_width, long* char_height) { - static const wxString M("m"); - wxCoord width = 0; - wxCoord height = 0; - dc->GetTextExtent(M, &width, &height); - *char_width = width; - *char_height = height; -} - -#endif // #ifndef __HELPERS_H__ diff --git a/helios/pipeViewer/pipe_view/doc/Doxyfile b/helios/pipeViewer/pipe_view/doc/Doxyfile deleted file mode 100644 index b822d17d4d..0000000000 --- a/helios/pipeViewer/pipe_view/doc/Doxyfile +++ /dev/null @@ -1,1800 +0,0 @@ -# Doxyfile 1.8.0 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" "). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# http://www.gnu.org/software/libiconv for the list of possible encodings. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or sequence of words) that should -# identify the project. Note that if you do not use Doxywizard you need -# to put quotes around the project name if it contains spaces. - -PROJECT_NAME = "Argos Viewer" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer -# a quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = - -# With the PROJECT_LOGO tag one can specify an logo or icon that is -# included in the documentation. The maximum height of the logo should not -# exceed 55 pixels and the maximum width should not exceed 200 pixels. -# Doxygen will copy the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, -# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, -# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English -# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, -# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, -# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful if your file system -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like regular Qt-style comments -# (thus requiring an explicit @brief command for a brief description.) - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then Doxygen will -# interpret the first line (until the first dot) of a Qt-style -# comment as the brief description. If set to NO, the comments -# will behave just like regular Qt-style comments (thus requiring -# an explicit \brief command for a brief description.) - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding -# "class=itcl::class" will allow you to use the command class in the -# itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for -# Java. For instance, namespaces will be presented as packages, qualified -# scopes will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources only. Doxygen will then generate output that is more tailored for -# Fortran. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for -# VHDL. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given extension. -# Doxygen has a built-in mapping, but you can override or extend it using this -# tag. The format is ext=language, where ext is a file extension, and language -# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, -# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make -# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C -# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions -# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all -# comments according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you -# can mix doxygen, HTML, and XML commands with Markdown formatting. -# Disable only in case of backward compatibilities issues. - -MARKDOWN_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also makes the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. -# Doxygen will parse them like normal C++ but will assume all classes use public -# instead of private inheritance when no explicit protection keyword is present. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate getter -# and setter methods for a property. Setting this option to YES (the default) -# will make doxygen replace the get and set methods by a property in the -# documentation. This will only work if the methods are indeed getting or -# setting a simple type. If this is not the case, or you want to show the -# methods anyway, you should set this option to NO. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and -# unions are shown inside the group in which they are included (e.g. using -# @ingroup) instead of on a separate page (for HTML and Man pages) or -# section (for LaTeX and RTF). - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and -# unions with only public data fields will be shown inline in the documentation -# of the scope in which they are defined (i.e. file, namespace, or group -# documentation), provided this scope is documented. If set to NO (the default), -# structs, classes, and unions are shown on a separate page (for HTML and Man -# pages) or section (for LaTeX and RTF). - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum -# is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically -# be useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. - -TYPEDEF_HIDES_STRUCT = NO - -# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to -# determine which symbols to keep in memory and which to flush to disk. -# When the cache is full, less often used symbols will be written to disk. -# For small to medium size projects (<1000 input files) the default value is -# probably good enough. For larger projects a too small cache size can cause -# doxygen to be busy swapping symbols to and from disk most of the time -# causing a significant performance penalty. -# If the system has enough physical memory increasing the cache will improve the -# performance by keeping more symbols in memory. Note that the value works on -# a logarithmic scale so increasing the size by one will roughly double the -# memory usage. The cache size is given by this formula: -# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, -# corresponding to a cache size of 2^16 = 65536 symbols. - -SYMBOL_CACHE_SIZE = 0 - -# Similar to the SYMBOL_CACHE_SIZE the size of the symbol lookup cache can be -# set using LOOKUP_CACHE_SIZE. This cache is used to resolve symbols given -# their name and scope. Since this can be an expensive process and often the -# same symbol appear multiple times in the code, doxygen keeps a cache of -# pre-resolved symbols. If the cache is too small doxygen will become slower. -# If the cache is too large, memory is wasted. The cache size is given by this -# formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range is 0..9, the default is 0, -# corresponding to a cache size of 2^16 = 65536 symbols. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal scope will be included in the documentation. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base -# name of the file that contains the anonymous namespace. By default -# anonymous namespaces are hidden. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen -# will list include files with double quotes in the documentation -# rather than with sharp brackets. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen -# will sort the (brief and detailed) documentation of class members so that -# constructors and destructors are listed first. If set to NO (the default) -# the constructors will appear in the respective orders defined by -# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. -# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO -# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the -# hierarchy of group names into alphabetical order. If set to NO (the default) -# the group names will appear in their defined order. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to -# do proper type resolution of all parameters of a function it will reject a -# match between the prototype and the implementation of a member function even -# if there is only one candidate or it is obvious which candidate to choose -# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen -# will still accept a match between prototype and implementation in such cases. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or macro consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and macros in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. The default is NO. - -SHOW_DIRECTORIES = NO - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. -# This will remove the Files entry from the Quick Index and from the -# Folder Tree View (if specified). The default is YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the -# Namespaces page. -# This will remove the Namespaces entry from the Quick Index -# and from the Folder Tree View (if specified). The default is YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command , where is the value of -# the FILE_VERSION_FILTER tag, and is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. The create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. -# You can optionally specify a file name after the option, if omitted -# DoxygenLayout.xml will be used as the name of the layout file. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files -# containing the references data. This must be a list of .bib files. The -# .bib extension is automatically appended if omitted. Using this command -# requires the bibtex tool to be installed. See also -# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style -# of the bibliography can be controlled using LATEX_BIB_STYLE. To use this -# feature you need bibtex and perl available in the search path. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_NO_PARAMDOC option can be enabled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = ../ - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is -# also the default input encoding. Doxygen uses libiconv (or the iconv built -# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for -# the list of possible encodings. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh -# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py -# *.f90 *.f *.for *.vhd *.vhdl - -FILE_PATTERNS = *.py - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. -# If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. -# Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. -# The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty or if -# non of the patterns match the file name, INPUT_FILTER is applied. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) -# and it is also possible to disable source filtering for a specific pattern -# using *.ext= (so without naming a filter). This option only has effect when -# FILTER_SOURCE_FILES is enabled. - -FILTER_SOURCE_PATTERNS = - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. -# Otherwise they will link to the documentation. - -REFERENCES_LINK_SOURCE = YES - -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. Note that when using a custom header you are responsible -# for the proper inclusion of any scripts and style sheets that doxygen -# needs, which is dependent on the configuration options used. -# It is advised to generate a default header using "doxygen -w html -# header.html footer.html stylesheet.css YourConfigFile" and then modify -# that header. Note that the header is subject to change so you typically -# have to redo this when upgrading to a newer version of doxygen or when -# changing the value of configuration settings such as GENERATE_TREEVIEW! - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# style sheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that -# the files will be copied as-is; there are no commands or markers available. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. -# Doxygen will adjust the colors in the style sheet and background images -# according to this color. Hue is specified as an angle on a colorwheel, -# see http://en.wikipedia.org/wiki/Hue for more information. -# For instance the value 0 represents red, 60 is yellow, 120 is green, -# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. -# The allowed range is 0 to 359. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of -# the colors in the HTML output. For a value of 0 the output will use -# grayscales only. A value of 255 will produce the most vivid colors. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to -# the luminance component of the colors in the HTML output. Values below -# 100 gradually make the output lighter, whereas values above 100 make -# the output darker. The value divided by 100 is the actual gamma applied, -# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, -# and 100 does not change the gamma. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting -# this to NO can help when comparing the output of multiple runs. - -HTML_TIMESTAMP = YES - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. For this to work a browser that supports -# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox -# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). - -HTML_DYNAMIC_SECTIONS = NO - -# If the GENERATE_DOCSET tag is set to YES, additional index files -# will be generated that can be used as input for Apple's Xcode 3 -# integrated development environment, introduced with OSX 10.5 (Leopard). -# To create a documentation set, doxygen will generate a Makefile in the -# HTML output directory. Running make will produce the docset in that -# directory and running "make install" will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find -# it at startup. -# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. - -GENERATE_DOCSET = NO - -# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the -# feed. A documentation feed provides an umbrella under which multiple -# documentation sets from a single provider (such as a company or product suite) -# can be grouped. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that -# should uniquely identify the documentation set bundle. This should be a -# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen -# will append .docset to the name. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING -# is used to encode HtmlHelp index (hhk), content (hhc) and project file -# content. - -CHM_INDEX_ENCODING = - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated -# that can be used as input for Qt's qhelpgenerator to generate a -# Qt Compressed Help (.qch) of the generated HTML documentation. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can -# be used to specify the file name of the resulting .qch file. -# The path specified is relative to the HTML output folder. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#namespace - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#virtual-folders - -QHP_VIRTUAL_FOLDER = doc - -# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to -# add. For more information please see -# http://doc.trolltech.com/qthelpproject.html#custom-filters - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see -# -# Qt Help Project / Custom Filters. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's -# filter section matches. -# -# Qt Help Project / Filter Attributes. - -QHP_SECT_FILTER_ATTRS = - -# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can -# be used to specify the location of Qt's qhelpgenerator. -# If non-empty doxygen will try to run qhelpgenerator on the generated -# .qhp file. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files -# will be generated, which together with the HTML files, form an Eclipse help -# plugin. To install this plugin and make it available under the help contents -# menu in Eclipse, the contents of the directory containing the HTML and XML -# files needs to be copied into the plugins directory of eclipse. The name of -# the directory within the plugins directory should be the same as -# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before -# the help appears. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have -# this name. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) -# at top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. Since the tabs have the same information as the -# navigation tree you can set this option to NO if you already set -# GENERATE_TREEVIEW to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. -# If the tag value is set to YES, a side panel will be generated -# containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). -# Windows users are probably better off using the HTML help feature. -# Since the tree basically has the same information as the tab index you -# could consider to set DISABLE_INDEX to NO when enabling this option. - -GENERATE_TREEVIEW = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values -# (range [0,1..20]) that doxygen will group on one line in the generated HTML -# documentation. Note that a value of 0 will completely suppress the enum -# values from appearing in the overview section. - -ENUM_VALUES_PER_LINE = 4 - -# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, -# and Class Hierarchy pages using a tree view instead of an ordered list. - -USE_INLINE_TREES = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open -# links to external symbols imported via tag files in a separate window. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of Latex formulas included -# as images in the HTML documentation. The default is 10. Note that -# when you change the font size after a successful doxygen run you need -# to manually remove any form_*.png images from the HTML output directory -# to force them to be regenerated. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are -# not supported properly for IE 6.0, but are supported on all modern browsers. -# Note that when changing this option you need to delete any form_*.png files -# in the HTML output before the changes have effect. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax -# (see http://www.mathjax.org) which uses client side Javascript for the -# rendering instead of using prerendered bitmaps. Use this if you do not -# have LaTeX installed or if you want to formulas look prettier in the HTML -# output. When enabled you may also need to install MathJax separately and -# configure the path to it using the MATHJAX_RELPATH option. - -USE_MATHJAX = NO - -# When MathJax is enabled you need to specify the location relative to the -# HTML output directory using the MATHJAX_RELPATH option. The destination -# directory should contain the MathJax.js script. For instance, if the mathjax -# directory is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to -# the MathJax Content Delivery Network so you can quickly see the result without -# installing MathJax. -# However, it is strongly recommended to install a local -# copy of MathJax from http://www.mathjax.org before deployment. - -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest - -# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension -# names that should be enabled during MathJax rendering. - -MATHJAX_EXTENSIONS = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box -# for the HTML output. The underlying search engine uses javascript -# and DHTML and should work on any modern browser. Note that when using -# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets -# (GENERATE_DOCSET) there is already a search function so this one should -# typically be disabled. For large projects the javascript based search engine -# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. - -SEARCHENGINE = YES - -# When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a PHP enabled web server instead of at the web client -# using Javascript. Doxygen will generate the search PHP script and index -# file to put on the web server. The advantage of the server -# based approach is that it scales better to large projects and allows -# full text search. The disadvantages are that it is more difficult to setup -# and does not have live searching capabilities. - -SERVER_BASED_SEARCH = NO - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. -# Note that when enabling USE_PDFLATEX this option is only used for -# generating bitmaps for formulas in the HTML output, but not in the -# Makefile that is written to the output directory. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4 - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for -# the generated latex document. The footer should contain everything after -# the last chapter. If it is left blank doxygen will generate a -# standard footer. Notice: only use this tag if you know what you are doing! - -LATEX_FOOTER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = YES - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -# If LATEX_SOURCE_CODE is set to YES then doxygen will include -# source code with syntax highlighting in the LaTeX output. -# Note that which sources are shown also depends on other settings -# such as SOURCE_BROWSER. - -LATEX_SOURCE_CODE = NO - -# The LATEX_BIB_STYLE tag can be used to specify the style to use for the -# bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See -# http://en.wikipedia.org/wiki/BibTeX for more info. - -LATEX_BIB_STYLE = plain - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load style sheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. -# This is useful -# if you want to understand what is going on. -# On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_DEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# pointed to by INCLUDE_PATH will be searched when a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition that -# overrules the definition found in the source code. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all references to function-like macros -# that are alone on a line, have an all uppercase name, and do not end with a -# semicolon, because these will confuse the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. For each -# tag file the location of the external documentation should be added. The -# format of a tag file without this location is as follows: -# -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths -# or URLs. Note that each tag file must have a unique name (where the name does -# NOT include the path). If a tag file is not located in the directory in which -# doxygen is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option also works with HAVE_DOT disabled, but it is recommended to -# install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see -# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the -# documentation. The MSCGEN_PATH tag allows you to specify the directory where -# the mscgen tool resides. If left empty the tool is assumed to be found in the -# default search path. - -MSCGEN_PATH = - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is -# allowed to run in parallel. When set to 0 (the default) doxygen will -# base this on the number of processors available in the system. You can set it -# explicitly to a value larger than 0 to get control over the balance -# between CPU load and processing speed. - -DOT_NUM_THREADS = 0 - -# By default doxygen will use the Helvetica font for all dot files that -# doxygen generates. When you want a differently looking font you can specify -# the font name using DOT_FONTNAME. You need to make sure dot is able to find -# the font, which can be done by putting it in a standard location or by setting -# the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the -# directory containing the font. - -DOT_FONTNAME = Helvetica - -# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. -# The default size is 10pt. - -DOT_FONTSIZE = 10 - -# By default doxygen will tell dot to use the Helvetica font. -# If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to -# set the path where dot can find it. - -DOT_FONTPATH = - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If the UML_LOOK tag is enabled, the fields and methods are shown inside -# the class node. If there are many fields or methods and many nodes the -# graph may become too big to be useful. The UML_LIMIT_NUM_FIELDS -# threshold limits the number of items for each type to make the size more -# managable. Set this to 0 for no limit. Note that the threshold may be -# exceeded by 50% before the limit is enforced. - -UML_LIMIT_NUM_FIELDS = 10 - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT options are set to YES then -# doxygen will generate a call dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable call graphs -# for selected functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then -# doxygen will generate a caller dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable caller -# graphs for selected functions only using the \callergraph command. - -CALLER_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will generate a graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are svg, png, jpg, or gif. -# If left blank png will be used. If you choose svg you need to set -# HTML_FILE_EXTENSION to xhtml in order to make the SVG files -# visible in IE 9+ (other browsers do not have this requirement). - -DOT_IMAGE_FORMAT = png - -# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to -# enable generation of interactive SVG images that allow zooming and panning. -# Note that this requires a modern browser other than Internet Explorer. -# Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you -# need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files -# visible. Older versions of IE do not have SVG support. - -INTERACTIVE_SVG = NO - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the -# \mscfile command). - -MSCFILE_DIRS = - -# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of -# nodes that will be shown in the graph. If the number of nodes in a graph -# becomes larger than this value, doxygen will truncate the graph, which is -# visualized by representing a node as a red box. Note that doxygen if the -# number of direct children of the root node in a graph is already larger than -# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note -# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. - -DOT_GRAPH_MAX_NODES = 50 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that the size of a graph can be further restricted by -# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not -# seem to support this out of the box. Warning: Depending on the platform used, -# enabling this option may lead to badly anti-aliased labels on the edges of -# a graph (i.e. they become hard to read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES diff --git a/helios/pipeViewer/pipe_view/gui/__init__.py b/helios/pipeViewer/pipe_view/gui/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/gui/argos_menu.py b/helios/pipeViewer/pipe_view/gui/argos_menu.py deleted file mode 100644 index fb09911190..0000000000 --- a/helios/pipeViewer/pipe_view/gui/argos_menu.py +++ /dev/null @@ -1,1554 +0,0 @@ -from __future__ import annotations -import wx -import wx.adv -import os -import logging -from typing import Optional, cast, Tuple, Union, TYPE_CHECKING - -from ..misc.version import get_version -from ..model.layout import Layout -from ..model.schedule_element import ScheduleLineElement -from .hover_preview import HoverPreviewOptionsDialog -from .dialogs.element_propsdlg import ElementTypeSelectionDialog -from .dialogs.layout_varsdlg import LayoutVariablesDialog -from .dialogs.watchlist_dialog import WatchListDlg -from .dialogs.console_dialog import ConsoleDlg -from .dialogs.select_layout_dlg import SelectLayoutDlg -from .dialogs.translate_elements_dlg import TranslateElementsDlg -from .dialogs.view_settings_dlg import ViewSettingsDialog -from .dialogs.shortcut_help import ShortcutHelp - -if TYPE_CHECKING: - from .layout_frame import Layout_Frame - -# Name each ID by ID_MENU_SUBMENU_etc... -ID_FILE_NEW = wx.NewId() -ID_FILE_OPEN = wx.NewId() -ID_FILE_CLOSE = wx.NewId() -ID_FILE_QUIT = wx.NewId() -ID_FILE_SAVE = wx.NewId() -ID_FILE_SAVEAS = wx.NewId() -ID_FILE_OPTIONS = wx.NewId() - -ID_HELP_ABOUT = wx.NewId() - -ID_SHORTCUT_HELP = wx.NewId() - -# Undo/Redo menu string templates. -# Render as 'Undo [num available] (next action)\thotkey' -UNDO_FMT = 'Undo {} ({} remaining)\tCTRL+Z' -REDO_FMT = 'Redo {} ({} until current)\tCTRL+Y' -REDO_ALL_FMT = 'Redo All ({})\tCTRL+SHIFT+Y' - - -# The menubar displayed within each Layout Frame that will have options to -# interact with a Layout, Layout Context, Workspace, etc. -class Argos_Menu(wx.MenuBar): - - # Set up all the menus and embedded sub-menus, - # with all their bindings/callbacks - def __init__(self, - frame: Layout_Frame, - layout: Layout, - update_enabled: bool) -> None: - this_script_filename = os.path.join(os.getcwd(), __file__) - - self.__parent = frame - self.__layout = layout - wx.MenuBar.__init__(self) - accelentries = [] - - self.__selection = self.__parent.GetCanvas().GetSelectionManager() - self.__selection.AddUndoRedoHook(self.__OnUndoRedo) - - # keeps track of the last location a graph was imported from - self.__last_loaded_graph_dir = None - self.__shortcut_help_dlg: Optional[ShortcutHelp] = None - - # Setting up the menu(s). - filemenu = wx.Menu() - self.__editmenu = wx.Menu() - - # sub menus of edit - selectmenu = wx.Menu() - arrangemenu = wx.Menu() - snappingmenu = wx.Menu() - - toolsmenu = wx.Menu() - viewmenu = wx.Menu() - dbmenu = wx.Menu() - helpmenu = wx.Menu() - - # Accessibility sub menu - accessibilitymenu = wx.Menu() - - # File - - menuNew = filemenu.Append(ID_FILE_NEW, - "&New Layout", - " Open a blank new frame") - menuOpen = filemenu.Append(ID_FILE_OPEN, - "&Open Layout", - " Open a layout file in a new frame") - - menuSaveAs = filemenu.Append(ID_FILE_SAVEAS, - "&Save Layout As\tCTRL+SHIFT+S", - " Save this layout to a new file") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, ord('S'), menuSaveAs.GetId()) - ) - - menuSave = filemenu.Append(ID_FILE_SAVE, - "&Save Layout\tCTRL+S", - " Save this layout to disk") - accelentries.append((wx.ACCEL_CTRL, ord('S'), menuSave.GetId())) - - filemenu.AppendSeparator() - - menuOptions = filemenu.Append(ID_FILE_OPTIONS, - "&Preferences (not implemented)", - " User Preferences") - menuOptions.Enable(False) - - filemenu.AppendSeparator() - - menuClose = filemenu.Append(ID_FILE_CLOSE, - "&Close Frame\tCTRL+W", - " Close this Frame") - accelentries.append((wx.ACCEL_CTRL, ord('W'), menuClose.GetId())) - - menuQuit = filemenu.Append(ID_FILE_QUIT, - "&Quit Argos\tCTRL+Q", - " Quit Argos Entirely") - accelentries.append((wx.ACCEL_CTRL, ord('Q'), menuQuit.GetId())) - - # Settings - self.menuMoveSnap = snappingmenu.Append(wx.NewId(), - "Snap during Move ops", - kind=wx.ITEM_CHECK) - self.menuMDominant = snappingmenu.Append( - wx.NewId(), - "Snap Dominant", - "Selection snaps as a group, based on Element beneath mouse", - wx.ITEM_RADIO - ) - self.menuMEach = snappingmenu.Append( - wx.NewId(), - "Snap Each", - "Each Element in selection freely snaps on its own", - wx.ITEM_RADIO - ) - snappingmenu.AppendSeparator() - self.menuResizeSnap = snappingmenu.Append(wx.NewId(), - "Snap during Resize ops", - kind=wx.ITEM_CHECK) - self.menuRDominant = snappingmenu.Append(wx.NewId(), - "Snap dominant", - "", - wx.ITEM_RADIO) - # Don't have a clue how to make this work smoothly. - self.menuRDominant.Enable(False) - self.menuREach = snappingmenu.Append(wx.NewId(), - "Snap each", - "", - wx.ITEM_RADIO) - - snappingmenu.Check(self.menuMoveSnap.GetId(), True) - snappingmenu.Check(self.menuResizeSnap.GetId(), True) - snappingmenu.Check(self.menuREach.GetId(), True) - - # Select - - menuSelectAll = selectmenu.Append( - wx.NewId(), - "Select All\tCTRL+A", - " Select all elements in the layout" - ) - accelentries.append((wx.ACCEL_CTRL, ord('A'), menuSelectAll.GetId())) - - menuInvertSelection = selectmenu.Append( - wx.NewId(), - "Invert Selection\tCTRL+I", - " Toggle selection state of all elements" - ) - accelentries.append( - (wx.ACCEL_CTRL, ord('I'), menuInvertSelection.GetId()) - ) - - # Arrange - - menuColumnUp = arrangemenu.Append(wx.NewId(), - "Column from top\tCTRL+ALT+UP", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT, wx.WXK_UP, menuColumnUp.GetId()) - ) - - menuColumnDown = arrangemenu.Append( - wx.NewId(), - "Column from bottom\tCTRL+ALT+DOWN", - "" - ) - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT, wx.WXK_DOWN, menuColumnDown.GetId()) - ) - - menuRowLeft = arrangemenu.Append(wx.NewId(), - "Row from leftmost\tCTRL+ALT+LEFT", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT, wx.WXK_LEFT, menuRowLeft.GetId()) - ) - - menuRowRight = arrangemenu.Append(wx.NewId(), - "Row from rightmost\tCTRL+ALT+RIGHT", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT, wx.WXK_RIGHT, menuRowRight.GetId()) - ) - - arrangemenu.AppendSeparator() - menuAlignTop = arrangemenu.Append(wx.NewId(), - "Align top edges\tCTRL+SHIFT+UP", - "Can result in overlap") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, wx.WXK_UP, menuAlignTop.GetId()) - ) - - menuAlignBottom = arrangemenu.Append( - wx.NewId(), - "Align bottom edges\tCTRL+SHIFT+DOWN", - "Can result in overlap" - ) - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - wx.WXK_DOWN, - menuAlignBottom.GetId()) - ) - - menuAlignLeft = arrangemenu.Append(wx.NewId(), - "Align left edges\tCTRL+SHIFT+LEFT", - "Can result in overlap") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - wx.WXK_LEFT, - menuAlignLeft.GetId()) - ) - - menuAlignRight = arrangemenu.Append( - wx.NewId(), - "Align right edges\tCTRL+SHIFT+RIGHT", - "Can result in overlap" - ) - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - wx.WXK_RIGHT, - menuAlignRight.GetId()) - ) - - arrangemenu.AppendSeparator() - menuFlipHoriz = arrangemenu.Append(wx.NewId(), - "Flip Horizontally\tHOME", - "") - accelentries.append( - (wx.ACCEL_NORMAL, wx.WXK_HOME, menuFlipHoriz.GetId()) - ) - - menuFlipVert = arrangemenu.Append(wx.NewId(), - "Flip Vertically\tPGDN", - "") - accelentries.append( - (wx.ACCEL_NORMAL, wx.WXK_PAGEDOWN, menuFlipVert.GetId()) - ) - - arrangemenu.AppendSeparator() - menuMoveToTop = arrangemenu.Append(wx.NewId(), - "Move to Front\tCTRL+SHIFT+PGUP", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - wx.WXK_PAGEUP, - menuMoveToTop.GetId()) - ) - - menuMoveToBottom = arrangemenu.Append(wx.NewId(), - "Move to Back\tCTRL+SHIFT+PGDN", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - wx.WXK_PAGEDOWN, - menuMoveToBottom.GetId()) - ) - - menuMoveUp = arrangemenu.Append(wx.NewId(), - "Move Forward\tCTRL+ALT+SHIFT+PGUP", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT | wx.ACCEL_SHIFT, - wx.WXK_PAGEUP, - menuMoveUp.GetId()) - ) - - menuMoveDown = arrangemenu.Append(wx.NewId(), - "Move Backward\tCTRL+ALT+SHIFT+PGDN", - "") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_ALT | wx.ACCEL_SHIFT, - wx.WXK_PAGEDOWN, - menuMoveDown.GetId()) - ) - - # Edit - self.menuEditMode = self.__editmenu.Append(wx.NewId(), - "Edit mode\tCTRL+M", - kind=wx.ITEM_CHECK) - # Checkable menu items don't get toggled by keyboard shortcuts. - # So, we assign the shortcut to a different ID/handler that toggles - # the menu item then calls the appropriate handler. - menuEditModeKeyShortcutId = wx.NewId() - accelentries.append( - (wx.ACCEL_CTRL, ord('M'), menuEditModeKeyShortcutId) - ) - - self.__editmenu.Check(self.menuEditMode.GetId(), False) - self.__editmenu.AppendSeparator() - - self.menuUndo = self.__editmenu.Append(wx.NewId(), - UNDO_FMT.format('', 0), - " Undo last action") - accelentries.append((wx.ACCEL_CTRL, ord('Z'), self.menuUndo.GetId())) - self.menuUndo.Enable(False) - - self.menuRedo = self.__editmenu.Append(wx.NewId(), - REDO_FMT.format('', 0), - " Redo last action") - accelentries.append((wx.ACCEL_CTRL, ord('Y'), self.menuRedo.GetId())) - self.menuRedo.Enable(False) - - self.menuRedoAll = self.__editmenu.Append(wx.NewId(), - REDO_ALL_FMT.format(0), - " Redo all") - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, - ord('Y'), - self.menuRedoAll.GetId()) - ) - self.menuRedoAll.Enable(False) - - self.__editmenu.AppendSeparator() - self.__editmenu.AppendSubMenu(selectmenu, "Select") - self.__editmenu.AppendSubMenu(arrangemenu, "Arrange") - self.__editmenu.AppendSubMenu(snappingmenu, "Snapping") - self.__editmenu.AppendSeparator() - menuNewEl = self.__editmenu.Append(wx.NewId(), - "New &Element\tCTRL+E", - " Create a new element") - accelentries.append((wx.ACCEL_CTRL, ord('E'), menuNewEl.GetId())) - - menuCloneEl = self.__editmenu.Append(wx.NewId(), - "&Duplicate Elements\tCTRL+D", - " Clones the selected elements") - accelentries.append((wx.ACCEL_CTRL, ord('D'), menuCloneEl.GetId())) - - menuDeleteEl = self.__editmenu.Append(wx.NewId(), - "Delete Elements\tDel", - " Deletes the selected elements") - accelentries.append( - (wx.ACCEL_NORMAL, wx.WXK_DELETE, menuDeleteEl.GetId()) - ) - - self.__editmenu.AppendSeparator() - menuTranslate = self.__editmenu.Append(wx.NewId(), - "Translate Selection...\tT", - " Moves a set of elements") - accelentries.append((wx.ACCEL_NORMAL, ord('T'), menuTranslate.GetId())) - - menuAvg = self.__editmenu.Append( - wx.NewId(), - "Average Dimensions\tSHIFT+A", - " Averages out the dimensions of each element" - ) - accelentries.append((wx.ACCEL_SHIFT, ord('A'), menuAvg.GetId())) - - menuIndRight = self.__editmenu.Append(wx.NewId(), - "Add Spacing Right\tALT+END", - " Spaces out the elements") - accelentries.append((wx.ACCEL_ALT, wx.WXK_END, menuIndRight.GetId())) - - menuIndLeft = self.__editmenu.Append(wx.NewId(), - "Add Spacing Left\tALT+HOME", - " Spaces out the elements") - accelentries.append((wx.ACCEL_ALT, wx.WXK_HOME, menuIndLeft.GetId())) - - menuIndUp = self.__editmenu.Append(wx.NewId(), - "Add Spacing Up\tALT+PGUP", - " Spaces out the elements") - accelentries.append((wx.ACCEL_ALT, wx.WXK_PAGEUP, menuIndUp.GetId())) - - menuIndDown = self.__editmenu.Append(wx.NewId(), - "Add Spacing Down\tALT+PGDN ", - " Spaces out the elements") - accelentries.append( - (wx.ACCEL_ALT, wx.WXK_PAGEDOWN, menuIndDown.GetId()) - ) - - menuSubtIndRight = self.__editmenu.Append( - wx.NewId(), - "Remove Spacing Right\tCTRL+END", - " Spaces out the elements" - ) - accelentries.append( - (wx.ACCEL_CTRL, wx.WXK_END, menuSubtIndRight.GetId()) - ) - - menuSubtIndLeft = self.__editmenu.Append( - wx.NewId(), - "Remove Spacing Left\tCTRL+HOME", - " Spaces out the elements" - ) - accelentries.append( - (wx.ACCEL_CTRL, wx.WXK_HOME, menuSubtIndLeft.GetId()) - ) - - menuSubtIndUp = self.__editmenu.Append(wx.NewId(), - "Remove Spacing Up\tCTRL+PGUP", - " Spaces out the elements") - accelentries.append( - (wx.ACCEL_CTRL, wx.WXK_PAGEUP, menuSubtIndUp.GetId()) - ) - - menuSubtIndDown = self.__editmenu.Append( - wx.NewId(), - "Remove Spacing Down\tCTRL+PGDN ", - " Spaces out the elements" - ) - accelentries.append( - (wx.ACCEL_CTRL, wx.WXK_PAGEDOWN, menuSubtIndDown.GetId()) - ) - - # Database - menuShowLocations = dbmenu.Append( - wx.NewId(), - "Open Locations List...\tCTRL+L", - "List of locations which can be referenced in this database" - ) - accelentries.append( - (wx.ACCEL_CTRL, ord('L'), menuShowLocations.GetId()) - ) - - menuSearch = dbmenu.Append( - wx.NewId(), - "Search Database\tCTRL+F", - "Search for strings in the database annotation." - ) - accelentries.append((wx.ACCEL_CTRL, ord('F'), menuSearch.GetId())) - - self.menuPollDB = dbmenu.Append(wx.NewId(), - "Poll Database For Updates\tCTRL+U", - kind=wx.ITEM_CHECK) - menuPollDBKeyShortcutId = wx.NewId() - accelentries.append((wx.ACCEL_CTRL, ord('U'), menuPollDBKeyShortcutId)) - if update_enabled: - self.menuPollDB.Check() - - menuRefreshDB = dbmenu.Append(wx.NewId(), "Refresh Database\tF5") - accelentries.append( - (wx.ACCEL_NORMAL, wx.WXK_F5, menuRefreshDB.GetId()) - ) - - # list of objects in edit menu that shouldn't be toggled between edit - # and playback modes - self.__no_toggle_in_edit_mode = [self.menuEditMode.GetId()] - - # \todo Show clocks info list - # \todo Show db .info file - self.menuViewSettings = viewmenu.Append( - wx.NewId(), - "Settings...", - "View and change display settings." - ) - # View - self.menuShuffleColors = accessibilitymenu.Append( - wx.NewId(), - "Shuffle Color Map", - "Shuffle the color map so that it doesn't produce a smooth " - "gradient.", - kind=wx.ITEM_CHECK - ) - accessibilitymenu.AppendSeparator() - self.menuDefaultColors = accessibilitymenu.Append( - wx.NewId(), - "Default Color Map", - "Use the default color map.", - kind=wx.ITEM_RADIO - ) - self.menuDeuteranopiaColors = accessibilitymenu.Append( - wx.NewId(), - "Deuteranopia", - "Use a color map adjusted for deuteranopia.", - kind=wx.ITEM_RADIO - ) - self.menuProtanopiaColors = accessibilitymenu.Append( - wx.NewId(), - "Protanopia", - "Use a color map adjusted for protanopia.", - kind=wx.ITEM_RADIO - ) - self.menuTritanopiaColors = accessibilitymenu.Append( - wx.NewId(), - "Tritanopia", - "Use a color map adjusted for tritanopia.", - kind=wx.ITEM_RADIO - ) - - viewmenu.AppendSubMenu(accessibilitymenu, "Accessibility") - - self.menuHoverPreview = viewmenu.Append( - wx.NewId(), - "Hover Preview\tCTRL+H", - "Displays annotation when mouse is over layout element.", - kind=wx.ITEM_CHECK - ) - menuHoverPreviewKeyShortcutId = wx.NewId() - accelentries.append( - (wx.ACCEL_CTRL, ord('H'), menuHoverPreviewKeyShortcutId) - ) - - viewmenu.Check(self.menuHoverPreview.GetId(), True) - self.__parent.SetHoverPreview( - viewmenu.IsChecked(self.menuHoverPreview.GetId()) - ) - - self.menuHoverOptions = viewmenu.Append( - wx.NewId(), - "Hover Options", - "Change what is shown in hover preview." - ) - - menuScheduleStyle = viewmenu.Append( - wx.NewId(), - "Schedule Element Style", - "Set attributes about schedule layout element." - ) - - self.menuElementSettings = viewmenu.Append( - wx.NewId(), - "Element Settings", - "Set attributes about layout element." - ) - menuWatchlist = viewmenu.Append( - wx.NewId(), - "Transaction Watches", - "Brings up a list of pinned hover view transactions." - ) - - self.menuElementSettings.Enable(False) - - menuLayoutVariables = viewmenu.Append( - wx.NewId(), - "Layout Location String Variables", - "Show values of variables used in the layout." - ) - - menuConsole = viewmenu.Append( - wx.NewId(), - "Python Console", - "Open a Python console." - ) - - menuFindElement = viewmenu.Append( - wx.NewId(), - "Search Layout\tCTRL+SHIFT+F", - "Search for elements in this layout." - ) - accelentries.append( - (wx.ACCEL_CTRL | wx.ACCEL_SHIFT, ord('F'), menuFindElement.GetId()) - ) - - self.menuToggleControls = viewmenu.Append( - wx.NewId(), - "Show navigation controls\tALT+C", - "Displays navigation controls at the bottom of the window", - kind=wx.ITEM_CHECK - ) - menuToggleControlsKeyShortcutId = wx.NewId() - accelentries.append( - (wx.ACCEL_ALT, ord('C'), menuToggleControlsKeyShortcutId) - ) - - viewmenu.Check(self.menuToggleControls.GetId(), True) - - # Help - - menuInfo = helpmenu.Append( - wx.NewId(), - "&Information", - "Show information about the current frame and database." - ) - menuShortcutsHelp = helpmenu.Append(wx.NewId(), - "Shortcuts", - "Show shortcut information") - menuAbout = helpmenu.Append(ID_HELP_ABOUT, - "&About", - "Information about this program") - - # Creating the menubar. - self.Append(filemenu, "&File") - self.Append(self.__editmenu, "&Edit") - self.Append(toolsmenu, "&Tools") - self.Append(dbmenu, "&Database") - self.Append(viewmenu, "&View") - self.Append(helpmenu, "&Help") - - # Creating the toolbar - self.__edit_toolbar = wx.ToolBar(self.__parent, - style=wx.TB_FLAT | wx.TB_HORIZONTAL, - name='Edit Toolbar') - self.__edit_toolbar.Show(False) - - # File operations - self.toolbarNew = self.__edit_toolbar.AddTool( - wx.NewId(), - 'New', - wx.ArtProvider.GetBitmap(wx.ART_NEW), - shortHelp='New layout' - ) - self.toolbarOpen = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Open', - wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN), - shortHelp='Open layout...' - ) - self.toolbarSave = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Save', - wx.ArtProvider.GetBitmap(wx.ART_FILE_SAVE), - shortHelp='Save current layout' - ) - self.toolbarSaveAs = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Save As', - wx.ArtProvider.GetBitmap(wx.ART_FILE_SAVE_AS), - shortHelp='Save current layout as...' - ) - self.toolbarEditMode = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Stop Editing', - wx.ArtProvider.GetBitmap(wx.ART_CLOSE), - shortHelp='Leave edit mode' - ) - - self.__edit_toolbar.AddSeparator() - - # Undo/Redo - self.toolbarUndo = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Undo', - wx.ArtProvider.GetBitmap(wx.ART_UNDO), - shortHelp='Undo' - ) - - redo_icon = wx.ArtProvider.GetBitmap(wx.ART_REDO, wx.ART_TOOLBAR) - self.toolbarRedo = self.__edit_toolbar.AddTool(wx.NewId(), - 'Redo', - redo_icon, - shortHelp='Redo') - - self.__edit_toolbar.AddSeparator() - - # Add/Duplicate/Delete Elements - self.toolbarAddElement = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Add Element', - wx.ArtProvider.GetBitmap(wx.ART_PLUS), - shortHelp='Add an element' - ) - self.toolbarCloneElement = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Duplicate Element', - wx.ArtProvider.GetBitmap(wx.ART_COPY), - shortHelp='Duplicate an element' - ) - self.toolbarDeleteElement = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Delete Element(s)', - wx.ArtProvider.GetBitmap(wx.ART_MINUS), - shortHelp='Delete selected element(s)' - ) - - self.__edit_toolbar.AddSeparator() - - def get_resource_bitmap(resource_path: str) -> wx.Bitmap: - return wx.Bitmap( - os.path.join( - os.path.dirname(os.path.dirname(this_script_filename)), - "resources", - resource_path - ), - wx.BITMAP_TYPE_PNG - ) - - # Translate - translate_icon = get_resource_bitmap("Actions-transform-move-icon.png") - self.toolbarTranslate = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Translate', - translate_icon, - shortHelp='Translate element(s)' - ) - - self.__edit_toolbar.AddSeparator() - - # Cursor location - self.toolbarCursorLocation = wx.TextCtrl( - self.__edit_toolbar, - wx.NewId(), - style=wx.TE_READONLY | wx.TE_LEFT - ) - self.toolbarCursorLocation.SetSize((110, -1)) - self.__edit_toolbar.AddControl(self.toolbarCursorLocation) - - self.__edit_toolbar.AddSeparator() - - # Move Front/Forward/Backward/Back - move_to_front_icon = get_resource_bitmap("shape-move-front.png") - self.toolbarMoveToTop = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Move To Front', - move_to_front_icon, - shortHelp='Move element(s) to the front' - ) - - move_forward_icon = get_resource_bitmap("shape-move-forward.png") - self.toolbarMoveUp = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Move Forward', - move_forward_icon, - shortHelp='Move element(s) forward' - ) - - move_backward_icon = get_resource_bitmap("shape-move-backward.png") - self.toolbarMoveDown = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Move Backward', - move_backward_icon, - shortHelp='Move element(s) backward' - ) - - move_to_back_icon = get_resource_bitmap("shape-move-back.png") - self.toolbarMoveToBottom = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Move To Back', - move_to_back_icon, - shortHelp='Move elements(s) to the back' - ) - - self.__edit_toolbar.AddSeparator() - - # Alignment - align_left_icon = get_resource_bitmap("shape-align-left.png") - self.toolbarAlignLeft = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Align Left', - align_left_icon, - shortHelp='Align element(s) to their left edge' - ) - - align_right_icon = get_resource_bitmap("shape-align-right.png") - self.toolbarAlignRight = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Align Right', - align_right_icon, - shortHelp='Align element(s) to their right edge' - ) - - align_top_icon = get_resource_bitmap("shape-align-top.png") - self.toolbarAlignTop = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Align Top', - align_top_icon, - shortHelp='Align element(s) to their top edge' - ) - - align_bottom_icon = get_resource_bitmap("shape-align-bottom.png") - self.toolbarAlignBottom = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Align Bottom', - align_bottom_icon, - shortHelp='Align element(s) to their bottom edge' - ) - - self.__edit_toolbar.AddSeparator() - - # Flipping - flip_horizontal_icon = get_resource_bitmap("shape-flip-horizontal.png") - self.toolbarFlipHoriz = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Horizontal Flip', - flip_horizontal_icon, - shortHelp='Flip element(s) horizontally' - ) - - flip_vertical_icon = get_resource_bitmap("shape-flip-vertical.png") - self.toolbarFlipVert = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Vertical Flip', - flip_vertical_icon, - shortHelp='Flip element(s) vertically' - ) - - self.__edit_toolbar.AddSeparator() - - # Stacking - self.toolbarColumnUp = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Make Column Up', - wx.ArtProvider.GetBitmap(wx.ART_GO_UP), - shortHelp='Make column from bottom' - ) - self.toolbarColumnDown = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Make Column Down', - wx.ArtProvider.GetBitmap(wx.ART_GO_DOWN), - shortHelp='Make column from top' - ) - self.toolbarRowLeft = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Make Row Left', - wx.ArtProvider.GetBitmap(wx.ART_GO_BACK), - shortHelp='Make row from rightmost' - ) - self.toolbarRowRight = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Make Row Right', - wx.ArtProvider.GetBitmap(wx.ART_GO_FORWARD), - shortHelp='Make row from leftmost' - ) - - self.__edit_toolbar.AddSeparator() - - # Padding - add_space_up_icon = get_resource_bitmap("add-space-up.png") - self.toolbarIndUp = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Add Spacing Up', - add_space_up_icon, - shortHelp='Add spacing up' - ) - - add_space_left_icon = get_resource_bitmap("add-space-left.png") - self.toolbarIndLeft = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Add Spacing Left', - add_space_left_icon, - shortHelp='Add spacing left' - ) - - add_space_right_icon = get_resource_bitmap("add-space-right.png") - self.toolbarIndRight = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Add Spacing Right', - add_space_right_icon, - shortHelp='Add spacing right' - ) - - add_space_down_icon = get_resource_bitmap("add-space-down.png") - self.toolbarIndDown = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Add Spacing Down', - add_space_down_icon, - shortHelp='Add spacing down' - ) - - sub_space_up_icon = get_resource_bitmap("sub-space-up.png") - self.toolbarSubtIndUp = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Remove Spacing Up', - sub_space_up_icon, - shortHelp='Remove spacing up' - ) - - sub_space_left_icon = get_resource_bitmap("sub-space-left.png") - self.toolbarSubtIndLeft = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Remove Spacing Left', - sub_space_left_icon, - shortHelp='Remove spacing left' - ) - - sub_space_right_icon = get_resource_bitmap("sub-space-right.png") - self.toolbarSubtIndRight = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Remove Spacing Right', - sub_space_right_icon, - shortHelp='Remove spacing right' - ) - - sub_space_down_icon = get_resource_bitmap("sub-space-down.png") - self.toolbarSubtIndDown = self.__edit_toolbar.AddTool( - wx.NewId(), - 'Remove Spacing Down', - sub_space_down_icon, - shortHelp='Remove spacing down' - ) - - self.__edit_toolbar.Realize() - - # Disable some menus by default (edit mode) - edit_mode = False - # refresh enabling/disabling of items - self.SetEditModeSettings(edit_mode) - - self.hover_options_dialog: Optional[HoverPreviewOptionsDialog] = None - - # Event Bindings - self.__parent.Bind(wx.EVT_MENU, self.OnNew, menuNew) - self.__parent.Bind(wx.EVT_TOOL, self.OnNew, self.toolbarNew) - self.__parent.Bind(wx.EVT_MENU, self.OnOpen, menuOpen) - self.__parent.Bind(wx.EVT_TOOL, self.OnOpen, self.toolbarOpen) - self.__parent.Bind(wx.EVT_MENU, self.OnSave, menuSave) - self.__parent.Bind(wx.EVT_TOOL, self.OnSave, self.toolbarSave) - self.__parent.Bind(wx.EVT_MENU, self.OnSaveAs, menuSaveAs) - self.__parent.Bind(wx.EVT_TOOL, self.OnSaveAs, self.toolbarSaveAs) - self.__parent.Bind(wx.EVT_MENU, self.OnClose, menuClose) - self.__parent.Bind(wx.EVT_MENU, self.OnExit, menuQuit) - self.__parent.Bind(wx.EVT_MENU, self.OnNewElement, menuNewEl) - self.__parent.Bind(wx.EVT_TOOL, - self.OnNewElement, - self.toolbarAddElement) - self.__parent.Bind(wx.EVT_MENU, self.OnCloneElement, menuCloneEl) - self.__parent.Bind(wx.EVT_TOOL, - self.OnCloneElement, - self.toolbarCloneElement) - self.__parent.Bind(wx.EVT_MENU, self.OnDeleteElement, menuDeleteEl) - self.__parent.Bind(wx.EVT_TOOL, - self.OnDeleteElement, - self.toolbarDeleteElement) - self.__parent.Bind(wx.EVT_MENU, self.OnSelectAll, menuSelectAll) - self.__parent.Bind(wx.EVT_MENU, - self.OnInvertSelection, - menuInvertSelection) - self.__parent.Bind(wx.EVT_MENU, self.OnFrameInfo, menuInfo) - self.__parent.Bind(wx.EVT_MENU, - self.OnShortcutsHelp, - menuShortcutsHelp) - self.__parent.Bind(wx.EVT_MENU, self.OnAbout, menuAbout) - self.__parent.Bind(wx.EVT_MENU, self.OnRowLeft, menuRowLeft) - self.__parent.Bind(wx.EVT_TOOL, self.OnRowLeft, self.toolbarRowLeft) - self.__parent.Bind(wx.EVT_MENU, self.OnRowRight, menuRowRight) - self.__parent.Bind(wx.EVT_TOOL, self.OnRowRight, self.toolbarRowRight) - self.__parent.Bind(wx.EVT_MENU, self.OnColumnUp, menuColumnUp) - self.__parent.Bind(wx.EVT_TOOL, self.OnColumnUp, self.toolbarColumnUp) - self.__parent.Bind(wx.EVT_MENU, self.OnColumnDown, menuColumnDown) - self.__parent.Bind(wx.EVT_TOOL, - self.OnColumnDown, - self.toolbarColumnDown) - self.__parent.Bind(wx.EVT_MENU, self.OnAlignTop, menuAlignTop) - self.__parent.Bind(wx.EVT_TOOL, self.OnAlignTop, self.toolbarAlignTop) - self.__parent.Bind(wx.EVT_MENU, self.OnAlignBottom, menuAlignBottom) - self.__parent.Bind(wx.EVT_TOOL, - self.OnAlignBottom, - self.toolbarAlignBottom) - self.__parent.Bind(wx.EVT_MENU, self.OnAlignLeft, menuAlignLeft) - self.__parent.Bind(wx.EVT_TOOL, - self.OnAlignLeft, - self.toolbarAlignLeft) - self.__parent.Bind(wx.EVT_MENU, self.OnAlignRight, menuAlignRight) - self.__parent.Bind(wx.EVT_TOOL, - self.OnAlignRight, - self.toolbarAlignRight) - self.__parent.Bind(wx.EVT_MENU, - self.OnLocationsList, - menuShowLocations) - self.__parent.Bind(wx.EVT_MENU, self.OnSearch, menuSearch) - self.__parent.Bind(wx.EVT_MENU, self.OnToggleUpdate, self.menuPollDB) - self.__parent.Bind(wx.EVT_MENU, - self.OnToggleUpdateKey, - id=menuPollDBKeyShortcutId) - self.__parent.Bind(wx.EVT_MENU, self.OnRefreshDB, menuRefreshDB) - self.__parent.Bind(wx.EVT_MENU, self.OnMDominant, self.menuMDominant) - self.__parent.Bind(wx.EVT_MENU, self.OnMEach, self.menuMEach) - self.__parent.Bind(wx.EVT_MENU, self.OnMoveSnap, self.menuMoveSnap) - self.__parent.Bind(wx.EVT_MENU, self.OnResizeSnap, self.menuResizeSnap) - self.__parent.Bind(wx.EVT_MENU, self.OnREach, self.menuREach) - self.__parent.Bind(wx.EVT_MENU, self.OnRDominant, self.menuRDominant) - self.__parent.Bind(wx.EVT_MENU, self.OnTranslate, menuTranslate) - self.__parent.Bind(wx.EVT_TOOL, - self.OnTranslate, - self.toolbarTranslate) - self.__parent.Bind(wx.EVT_MENU, self.OnAvg, menuAvg) - self.__parent.Bind(wx.EVT_MENU, self.OnIndUp, menuIndUp) - self.__parent.Bind(wx.EVT_TOOL, self.OnIndUp, self.toolbarIndUp) - self.__parent.Bind(wx.EVT_MENU, self.OnIndDown, menuIndDown) - self.__parent.Bind(wx.EVT_TOOL, self.OnIndDown, self.toolbarIndDown) - self.__parent.Bind(wx.EVT_MENU, self.OnIndRight, menuIndRight) - self.__parent.Bind(wx.EVT_TOOL, self.OnIndRight, self.toolbarIndRight) - self.__parent.Bind(wx.EVT_MENU, self.OnIndLeft, menuIndLeft) - self.__parent.Bind(wx.EVT_TOOL, self.OnIndLeft, self.toolbarIndLeft) - self.__parent.Bind(wx.EVT_MENU, self.OnSubtIndUp, menuSubtIndUp) - self.__parent.Bind(wx.EVT_TOOL, - self.OnSubtIndUp, - self.toolbarSubtIndUp) - self.__parent.Bind(wx.EVT_MENU, self.OnSubtIndDown, menuSubtIndDown) - self.__parent.Bind(wx.EVT_TOOL, - self.OnSubtIndDown, - self.toolbarSubtIndDown) - self.__parent.Bind(wx.EVT_MENU, self.OnSubtIndRight, menuSubtIndRight) - self.__parent.Bind(wx.EVT_TOOL, - self.OnSubtIndRight, - self.toolbarSubtIndRight) - self.__parent.Bind(wx.EVT_MENU, self.OnSubtIndLeft, menuSubtIndLeft) - self.__parent.Bind(wx.EVT_TOOL, - self.OnSubtIndLeft, - self.toolbarSubtIndLeft) - self.__parent.Bind(wx.EVT_MENU, self.OnFlipHoriz, menuFlipHoriz) - self.__parent.Bind(wx.EVT_TOOL, - self.OnFlipHoriz, - self.toolbarFlipHoriz) - self.__parent.Bind(wx.EVT_MENU, self.OnFlipVert, menuFlipVert) - self.__parent.Bind(wx.EVT_TOOL, self.OnFlipVert, self.toolbarFlipVert) - self.__parent.Bind(wx.EVT_MENU, self.OnMoveToTop, menuMoveToTop) - self.__parent.Bind(wx.EVT_TOOL, - self.OnMoveToTop, - self.toolbarMoveToTop) - self.__parent.Bind(wx.EVT_MENU, self.OnMoveToBottom, menuMoveToBottom) - self.__parent.Bind(wx.EVT_TOOL, - self.OnMoveToBottom, - self.toolbarMoveToBottom) - self.__parent.Bind(wx.EVT_MENU, self.OnMoveUp, menuMoveUp) - self.__parent.Bind(wx.EVT_TOOL, self.OnMoveUp, self.toolbarMoveUp) - self.__parent.Bind(wx.EVT_MENU, self.OnMoveDown, menuMoveDown) - self.__parent.Bind(wx.EVT_TOOL, self.OnMoveDown, self.toolbarMoveDown) - self.__parent.Bind(wx.EVT_MENU, self.OnEditMode, self.menuEditMode) - self.__parent.Bind(wx.EVT_TOOL, - self.OnEditModeKey, - self.toolbarEditMode) - self.__parent.Bind(wx.EVT_MENU, - self.OnEditModeKey, - id=menuEditModeKeyShortcutId) - self.__parent.Bind(wx.EVT_MENU, self.OnUndo, self.menuUndo) - self.__parent.Bind(wx.EVT_TOOL, self.OnUndo, self.toolbarUndo) - self.__parent.Bind(wx.EVT_MENU, self.OnRedo, self.menuRedo) - self.__parent.Bind(wx.EVT_TOOL, self.OnRedo, self.toolbarRedo) - self.__parent.Bind(wx.EVT_MENU, self.OnRedoAll, self.menuRedoAll) - self.__parent.Bind(wx.EVT_MENU, - self.OnHoverPreview, - self.menuHoverPreview) - self.__parent.Bind(wx.EVT_MENU, - self.OnHoverPreviewKey, - id=menuHoverPreviewKeyShortcutId) - self.__parent.Bind(wx.EVT_MENU, - self.OnHoverOptions, - self.menuHoverOptions) - self.__parent.Bind(wx.EVT_MENU, - self.OnElementSettings, - self.menuElementSettings) - self.__parent.Bind(wx.EVT_MENU, - self.OnChooseScheduleStyle, - menuScheduleStyle) - self.__parent.Bind(wx.EVT_MENU, - self.OnShowLayoutVariables, - menuLayoutVariables) - self.__parent.Bind(wx.EVT_MENU, self.OnWatchList, menuWatchlist) - self.__parent.Bind(wx.EVT_MENU, self.OnConsole, menuConsole) - self.__parent.Bind(wx.EVT_MENU, self.OnFindElement, menuFindElement) - self.__parent.Bind(wx.EVT_MENU, - self.OnToggleControls, - self.menuToggleControls) - self.__parent.Bind(wx.EVT_MENU, - self.OnToggleControlsKey, - id=menuToggleControlsKeyShortcutId) - self.__parent.Bind(wx.EVT_MENU, - self.OnShuffleColors, - self.menuShuffleColors) - self.__parent.Bind(wx.EVT_MENU, - self.OnColorMapChange, - self.menuDefaultColors) - self.__parent.Bind(wx.EVT_MENU, - self.OnColorMapChange, - self.menuDeuteranopiaColors) - self.__parent.Bind(wx.EVT_MENU, - self.OnColorMapChange, - self.menuProtanopiaColors) - self.__parent.Bind(wx.EVT_MENU, - self.OnColorMapChange, - self.menuTritanopiaColors) - self.__parent.Bind(wx.EVT_MENU, - self.OnViewSettings, - self.menuViewSettings) - - self.__accel_tbl = wx.AcceleratorTable(accelentries) - - workspace = self.__parent.GetWorkspace() - colorblindness_option = workspace.GetPalette() - palette_shuffle_option = workspace.GetColorShuffleState() - - if colorblindness_option == 'default': - self.menuDefaultColors.Check() - elif colorblindness_option == 'd': - self.menuDeuteranopiaColors.Check() - elif colorblindness_option == 'p': - self.menuProtanopiaColors.Check() - elif colorblindness_option == 't': - self.menuTritanopiaColors.Check() - else: - raise ValueError( - 'Invalid value specified for ARGOS_COLORBLINDNESS_MODE: ' - f'{colorblindness_option}' - ) - - if palette_shuffle_option == 'default': - self.menuShuffleColors.Check(False) - elif palette_shuffle_option == 'shuffled': - self.menuShuffleColors.Check() - else: - raise ValueError( - 'Invalid value specified for ARGOS_PALETTE_SHUFFLE_MODE: ' - f'{palette_shuffle_option}' - ) - - def OnFlipHoriz(self, evt: wx.CommandEvent) -> None: - self.__selection.Flip(self.__selection.RIGHT) - - def OnFlipVert(self, evt: wx.CommandEvent) -> None: - self.__selection.Flip(self.__selection.TOP) - - def OnMoveToTop(self, evt: wx.CommandEvent) -> None: - self.__selection.MoveToTop() - - def OnMoveToBottom(self, evt: wx.CommandEvent) -> None: - self.__selection.MoveToBottom() - - def OnMoveUp(self, evt: wx.CommandEvent) -> None: - self.__selection.MoveUp() - - def OnMoveDown(self, evt: wx.CommandEvent) -> None: - self.__selection.MoveDown() - - def OnAvg(self, evt: wx.CommandEvent) -> None: - self.__selection.Average() - - def OnIndUp(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.BOTTOM) - - def OnIndDown(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.TOP) - - def OnIndLeft(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.RIGHT) - - def OnIndRight(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.LEFT) - - def OnSubtIndUp(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.TOP, True) - - def OnSubtIndDown(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.BOTTOM, True) - - def OnSubtIndLeft(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.LEFT, True) - - def OnSubtIndRight(self, evt: wx.CommandEvent) -> None: - self.__selection.Indent(self.__selection.RIGHT, True) - - def OnRowLeft(self, evt: wx.CommandEvent) -> None: - self.__selection.Stack('left') - - def OnColumnUp(self, evt: wx.CommandEvent) -> None: - self.__selection.Stack('top') - - def OnRowRight(self, evt: wx.CommandEvent) -> None: - self.__selection.Stack('right') - - def OnColumnDown(self, evt: wx.CommandEvent) -> None: - self.__selection.Stack('bottom') - - def OnAlignTop(self, evt: wx.CommandEvent) -> None: - self.__selection.Align('top') - - def OnAlignBottom(self, evt: wx.CommandEvent) -> None: - self.__selection.Align('bottom') - - def OnAlignLeft(self, evt: wx.CommandEvent) -> None: - self.__selection.Align('left') - - def OnAlignRight(self, evt: wx.CommandEvent) -> None: - self.__selection.Align('right') - - def OnMoveSnap(self, evt: wx.CommandEvent) -> None: - if not self.menuMoveSnap.IsChecked(): - self.__selection.SetSnapMode('freemove') - else: - print(self.menuMDominant.IsChecked()) - if self.menuMDominant.IsChecked(): - self.__selection.SetSnapMode('mdominant') - else: - self.__selection.SetSnapMode('meach') - - def OnMDominant(self, evt: wx.CommandEvent) -> None: - if self.menuMoveSnap.IsChecked(): - self.__selection.SetSnapMode('mdominant') - - def OnMEach(self, evt: wx.CommandEvent) -> None: - if self.menuMoveSnap.IsChecked(): - self.__selection.SetSnapMode('meach') - - def OnResizeSnap(self, evt: wx.CommandEvent) -> None: - if not self.menuResizeSnap.IsChecked(): - self.__selection.SetSnapMode('freesize') - else: - if self.menuRDominant.IsChecked(): - self.__selection.SetSnapMode('rdominant') - else: - self.__selection.SetSnapMode('reach') - - def OnTranslate(self, evt: wx.CommandEvent) -> None: - if self.__selection.GetSelection(): - # Translation is done within the dialog - dlg = TranslateElementsDlg(self.__parent) - try: - dlg.ShowModal() - finally: - dlg.Destroy() - - def OnRDominant(self, evt: wx.CommandEvent) -> None: - if self.menuResizeSnap.IsChecked(): - self.__selection.SetSnapMode('rdominant') - - def OnREach(self, evt: wx.CommandEvent) -> None: - if self.menuResizeSnap.IsChecked(): - self.__selection.SetSnapMode('reach') - - # sets settings based on status of edit mode - def SetEditModeSettings(self, menuEditBool: bool) -> None: - self.menuEditMode.Check(menuEditBool) - items_to_change = self.__editmenu.GetMenuItems() - for item in items_to_change: - if not item.GetId() in self.__no_toggle_in_edit_mode: - item.Enable(menuEditBool) - - self.__UpdateUndoRedoItems() - - dialog = self.__parent.GetCanvas().GetDialog() - self.menuElementSettings.Enable(menuEditBool) - if menuEditBool: - self.SetBackgroundColour('YELLOW') - self.__parent.SetAcceleratorTable(self.__accel_tbl) - dialog.Show() - dialog.SetFocus() - dialog.Raise() - else: - self.SetBackgroundColour('GRAY') - self.__parent.SetAcceleratorTable(wx.NullAcceleratorTable) - dialog.Show(False) - - def OnRefreshDB(self, evt: wx.CommandEvent) -> None: - self.__parent.ForceDBUpdate() - - def OnToggleUpdate(self, evt: wx.CommandEvent) -> None: - poll_mode = self.menuPollDB.IsChecked() - self.__parent.SetPollMode(poll_mode) - - def OnToggleUpdateKey(self, evt: wx.CommandEvent) -> None: - self.menuPollDB.Check(not self.menuPollDB.IsChecked()) - self.OnToggleUpdate(evt) - - def OnEditMode(self, evt: wx.CommandEvent) -> None: - edit_mode = self.menuEditMode.IsChecked() - self.__parent.SetEditMode(edit_mode) - - def OnEditModeKey(self, evt: wx.CommandEvent) -> None: - self.menuEditMode.Check(not self.menuEditMode.IsChecked()) - self.OnEditMode(evt) - - def OnUndo(self, evt: Optional[wx.CommandEvent] = None) -> None: - self.__selection.Undo() - - def OnRedo(self, evt: Optional[wx.CommandEvent] = None) -> None: - self.__selection.Redo() - - def OnRedoAll(self, evt: Optional[wx.CommandEvent] = None) -> None: - self.__selection.RedoAll() - - def OnHoverPreview(self, evt: wx.CommandEvent) -> None: - hover_preview = self.menuHoverPreview.IsChecked() - self.__parent.SetHoverPreview(hover_preview) - self.__parent.Refresh() - - def OnHoverPreviewKey(self, evt: wx.CommandEvent) -> None: - self.menuHoverPreview.Check(not self.menuHoverPreview.IsChecked()) - self.OnHoverPreview(evt) - - def OnHoverOptions(self, evt: wx.CommandEvent) -> None: - self.hover_options_dialog = HoverPreviewOptionsDialog( - self, - self.__parent.GetCanvas().GetHoverPreview() - ) - self.hover_options_dialog.ShowWindowModal() - - def OnViewSettings(self, evt: wx.CommandEvent) -> None: - settings = self.__parent.GetSettings() - with ViewSettingsDialog(self.__parent, settings) as view_settings_dlg: - view_settings_dlg = cast(ViewSettingsDialog, view_settings_dlg) - if view_settings_dlg.ShowModal() == wx.ID_OK: - new_settings = view_settings_dlg.GetSettings() - if new_settings: - self.__parent.UpdateSettings(new_settings) - settings.save() - - def OnElementSettings(self, evt: wx.CommandEvent) -> None: - dialog = self.__parent.GetCanvas().GetDialog() - dialog.Show() - dialog.SetFocus() - dialog.Raise() - - def OnConsole(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowDialog('console', ConsoleDlg) - - def OnWatchList(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowDialog('watchlist', WatchListDlg) - - def OnChooseScheduleStyle(self, evt: wx.CommandEvent) -> None: - dl = ScheduleLineElement.DRAW_LOOKUP - dialog = wx.SingleChoiceDialog( - self, - 'Global Style for Schedule Lines.', - '', - list(dl.keys()), - wx.CHOICEDLG_STYLE - ) - if dialog.ShowModal() == wx.ID_OK: - self.__parent.GetCanvas().SetScheduleLineStyle( - dl[dialog.GetStringSelection()] - ) - dialog.Destroy() - - # Show dialog containing menu layout variables - def OnShowLayoutVariables(self, evt: wx.CommandEvent) -> None: - self.__parent.GetContext().UpdateLocationVariables() - dialog = LayoutVariablesDialog(self, - wx.NewId(), - 'Layout location Variables', - self.__parent.GetContext()) - dialog.Show() - - # Handle clicking of Open Locations List - def OnLocationsList(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowLocationsList() - - # Handle clicking of Search menu - def OnSearch(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowSearch() - - def OnFindElement(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowFindElement() - - def OnToggleControls(self, evt: wx.CommandEvent) -> None: - self.__parent.ShowNavigationControls( - self.menuToggleControls.IsChecked() - ) - - def OnToggleControlsKey(self, evt: wx.CommandEvent) -> None: - self.menuToggleControls.Check(not self.menuToggleControls.IsChecked()) - self.OnToggleControls(evt) - - def OnColorMapChange(self, evt: wx.CommandEvent) -> None: - if self.menuDefaultColors.IsChecked(): - palette = 'default' - elif self.menuDeuteranopiaColors.IsChecked(): - palette = 'd' - elif self.menuProtanopiaColors.IsChecked(): - palette = 'p' - elif self.menuTritanopiaColors.IsChecked(): - palette = 't' - else: - raise RuntimeError('None of the color menu options are checked!') - - self.__parent.GetWorkspace().SetPalette(palette) - - def OnShuffleColors(self, evt: wx.CommandEvent) -> None: - if self.menuShuffleColors.IsChecked(): - shuffle_state = 'shuffled' - else: - shuffle_state = 'default' - self.__parent.GetWorkspace().SetColorShuffleState(shuffle_state) - - # Placeholder method - def OnNew(self, evt: wx.CommandEvent) -> None: - context = self.__parent.GetContext() - self.__parent.GetWorkspace().OpenLayoutFrame( - None, - context.dbhandle.database, - context.GetHC(), - self.menuPollDB.IsChecked(), - self.__parent.GetTitlePrefix(), - self.__parent.GetTitleOverride(), - self.__parent.GetTitleSuffix(), - None - ) - - def OnOpen(self, evt: wx.CommandEvent) -> None: - context = self.__parent.GetContext() - - layout = self.__parent.GetContext().GetLayout() - if layout is not None: - default = layout.GetFilename() - else: - default = None - dlg = SelectLayoutDlg(default) - dlg.Centre() - if dlg.ShowModal() == wx.CANCEL: - return - dlg.Destroy() - - lf = dlg.GetFilename() - self.__parent.GetWorkspace().OpenLayoutFrame( - lf, - context.dbhandle.database, - context.GetHC(), - self.menuPollDB.IsChecked(), - self.__parent.GetTitlePrefix(), - self.__parent.GetTitleOverride(), - self.__parent.GetTitleSuffix(), - None - ) - - # 'Saving' means saving the Layout to file, nothing else is currently - # preserved about a session (no user preferences, current selection, HC) - def OnSave(self, evt: wx.CommandEvent) -> None: - self.__parent.Save() - - # Handle saving the file to a selected path - def OnSaveAs(self, evt: Optional[wx.CommandEvent] = None) -> None: - self.__parent.SaveAs() - - # Show information about this frame - def OnFrameInfo(self, evt: wx.CommandEvent) -> None: - layout_file = self.__layout.GetFilename() - if layout_file is None: - layout_file = '' - else: - layout_file = f'"{layout_file}"' - - # There is a diferent between client size and screen size (GetSize). - # Both are too small and do not include border. - # Enlarge the apparent size so that the reported size includes the - # border and can correctly be used as a command line geometry input. - # There is complimentary logic in argos.py - w, h = self.__parent.GetSize() - wdiff = w - self.__parent.GetClientSize()[0] - hdiff = h - self.__parent.GetClientSize()[1] - w += wdiff - h += hdiff - x, y = self.__parent.GetPosition() - - context = self.__parent.GetContext() - assert context.dbhandle.database.dbmodule.__file__ is not None - message = \ - f'Version:{get_version()}\n\n' \ - f'Layout:\n{layout_file}\n\n' \ - 'Database:\n' \ - f'"{context.dbhandle.database.filename}" ' \ - f'(v{context.dbhandle.api.getFileVersion()})\n\n' \ - f'Layout Elements: {len(self.__layout.GetElements())}\n\n' \ - f'Frame Geometry (w,h,x,y): {w},{h},{x},{y}\n\n' \ - 'Reader Library:\n' \ - f'{os.path.dirname(context.dbhandle.database.dbmodule.__file__)}' - - dlg = wx.MessageDialog(self, - message, - "Argos Frame-Specific Information", - wx.OK) - dlg.ShowModal() - dlg.Destroy() - - def OnShortcutsHelp(self, evt: wx.CommandEvent) -> None: - if not self.__shortcut_help_dlg: - self.__shortcut_help_dlg = ShortcutHelp(self.__parent, - ID_SHORTCUT_HELP) - - def OnNewElement(self, evt: wx.CommandEvent) -> None: - if self.__parent.GetCanvas().GetInputDecoder().GetEditMode(): - type_dialog = ElementTypeSelectionDialog(self.__parent.GetCanvas()) - type_dialog.Center() - if type_dialog.ShowModal() == wx.ID_OK: - self.__selection.GenerateElement(self.__layout, - type_dialog.GetSelection()) - - def OnCloneElement(self, evt: wx.CommandEvent) -> None: - if self.__parent.GetCanvas().GetInputDecoder().GetEditMode(): - self.__selection.PrepNextCopy() - self.__selection.GenerateDuplicateSelection(self.__layout, delta=5) - - def OnDeleteElement(self, evt: wx.CommandEvent) -> None: - if self.__parent.GetCanvas().GetInputDecoder().GetEditMode(): - self.__selection.Delete(self.__layout) - - def OnSelectAll(self, evt: wx.CommandEvent) -> None: - if self.__parent.GetCanvas().GetInputDecoder().GetEditMode(): - self.__selection.SelectEntireLayout() - - def OnRelease(self, evt: wx.CommandEvent) -> None: - # Allow clearing selection in edit mode - self.__selection.Clear() - - def OnInvertSelection(self, evt: wx.CommandEvent) -> None: - if self.__parent.GetCanvas().GetInputDecoder().GetEditMode(): - self.__selection.InvertSelection(self.__layout) - - # Show about dialog - def OnAbout(self, evt: wx.CommandEvent) -> None: - # @todo Forward to workspace! - info = wx.adv.AboutDialogInfo() - info.SetName('Argos') - info.SetDescription( - 'PipeViewer. This tool visualizes an Argos pipeline database and ' - 'allows the editing of custom layout files' - ) - info.SetCopyright("""Copyright 2019""") - wx.adv.AboutBox(info) # Show modal about box - - # Handle exit menu event - def OnClose(self, evt: wx.CommandEvent) -> None: - self.__parent._HandleClose() - - # Handle exit menu event - def OnExit(self, evt: wx.CommandEvent) -> None: - logging.info('OnExit') - self.__parent.GetWorkspace().Exit() - - def GetEditToolbar(self) -> wx.ToolBar: - return self.__edit_toolbar - - def ShowEditToolbar(self, show: bool) -> None: - self.__edit_toolbar.Show(show) - - def UpdateMouseLocation(self, - pos: Union[Tuple[int, int], wx.Point]) -> None: - self.toolbarCursorLocation.SetValue(f'{pos}') - - # Undo/Redo hook from selection manager - def __OnUndoRedo(self) -> None: - self.__UpdateUndoRedoItems() - - # Update the undo and redo items based on the number of undos/redos tracked - # in the selection manager - def __UpdateUndoRedoItems(self) -> None: - editmode = self.__parent.GetCanvas().GetInputDecoder().GetEditMode() - undos = self.__selection.NumUndos() - redos = self.__selection.NumRedos() - self.menuUndo.SetItemLabel( - UNDO_FMT.format(self.__selection.GetNextUndoDesc(), undos) - ) - undo_enabled = editmode and (undos != 0) - self.menuUndo.Enable(undo_enabled) - self.__edit_toolbar.EnableTool(self.toolbarUndo.GetId(), undo_enabled) - self.menuRedo.SetItemLabel( - REDO_FMT.format(self.__selection.GetNextRedoDesc(), redos) - ) - redo_enabled = editmode and (redos != 0) - self.menuRedo.Enable(redo_enabled) - self.__edit_toolbar.EnableTool(self.toolbarRedo.GetId(), redo_enabled) - self.menuRedoAll.SetItemLabel(REDO_ALL_FMT.format(redos)) - self.menuRedoAll.Enable(editmode and (redos != 0)) diff --git a/helios/pipeViewer/pipe_view/gui/autocoloring.py b/helios/pipeViewer/pipe_view/gui/autocoloring.py deleted file mode 100644 index e2a3185a1d..0000000000 --- a/helios/pipeViewer/pipe_view/gui/autocoloring.py +++ /dev/null @@ -1,254 +0,0 @@ -# @package Argos auto-coloring color map - -from __future__ import annotations -import wx -import colorsys # For easy HSL to RGB conversion -import numpy as np -from typing import Dict, Set, Tuple - -try: - from daltonize import daltonize - HAS_DALTONIZE = True -except ModuleNotFoundError: - HAS_DALTONIZE = False - - -class BrushRepository: - _COLORBLIND_MODES = set(('d', 'p', 't')) - _PALETTES = set(('default',)) | _COLORBLIND_MODES - _SHUFFLE_MODES = set(('default', 'shuffled')) - - def __init__(self) -> None: - self.palette = 'default' - self.shuffle_mode = 'default' - self.brushes: Dict[str, Dict[str, Dict[int, wx.Brush]]] = {} - for shuffle_mode in self._SHUFFLE_MODES: - self.brushes[shuffle_mode] = {} - - @classmethod - def get_supported_palettes(cls) -> Set[str]: - return cls._PALETTES - - @classmethod - def get_supported_shuffle_modes(cls) -> Set[str]: - return cls._SHUFFLE_MODES - - def __getitem__(self, idx: int) -> wx.Brush: - return self.as_dict()[idx] - - def as_dict(self) -> Dict[int, wx.Brush]: - return self.brushes[self.shuffle_mode][self.palette] - - def __len__(self) -> int: - return len(self.as_dict()) - - def validate_shuffle_mode(self, shuffle_mode: str) -> None: - if shuffle_mode not in self._SHUFFLE_MODES: - raise ValueError( - 'Shuffle mode shoud be one of the following values: ' - f'{self._SHUFFLE_MODES}' - ) - - def validate_palette(self, palette: str) -> None: - if not HAS_DALTONIZE and palette in self._COLORBLIND_MODES: - raise NotImplementedError( - 'Daltonize is not installed.' - '`pip install daltonize` to enable colorblindness modes.' - ) - if palette not in self._PALETTES: - raise ValueError( - 'Palette mode should be one of the following values: ' - f'{self._PALETTES}' - ) - - def generate_shuffled_palette( - self, - shuffle_mode: str, - color_dict: Dict[int, Tuple[float, float, float]] - ) -> Dict[int, Tuple[float, float, float]]: - self.validate_shuffle_mode(shuffle_mode) - if shuffle_mode == 'default': - return color_dict - elif shuffle_mode == 'shuffled': - # Shuffle the colors around - new_dict = { - (3 * key) % len(color_dict): value - for key, value in color_dict.items() - } - # Make sure the shuffle didn't leave out any of the original keys - assert set(new_dict.keys()) == \ - set(range(min(color_dict.keys()), - max(color_dict.keys()) + 1)) - return new_dict - else: - raise NotImplementedError( - f'Shuffle mode {shuffle_mode} has not been implemented.' - ) - - def create_brush_dict( - self, - color_dict: Dict[int, Tuple[float, float, float]] - ) -> Dict[int, wx.Brush]: - return { - key: wx.Brush([int(255 * x) for x in value]) - for key, value in color_dict.items() - } - - def generate_all_brushes( - self, - color_dict: Dict[int, Tuple[float, float, float]] - ) -> None: - shuffle_idx = 0 - shuffle_idx_to_mode = {} - shuffled_color_dicts = {} - - for shuffle_mode in self._SHUFFLE_MODES: - # Keep track of which y-index maps to which shuffle mode - shuffle_idx_to_mode[shuffle_idx] = shuffle_mode - - shuffled_color_dicts[shuffle_mode] = \ - self.generate_shuffled_palette(shuffle_mode, color_dict) - - # The default palettes don't get daltonized, so we can go ahead and - # create brushes for them here - self.brushes[shuffle_mode]['default'] = \ - self.create_brush_dict(shuffled_color_dicts[shuffle_mode]) - - shuffle_idx += 1 - - if HAS_DALTONIZE: - for palette in self._COLORBLIND_MODES: - # Create an image containing 1 RGB pixel for each color in each - # shuffled dict - colors = np.zeros( - (len(self._SHUFFLE_MODES), len(color_dict), 3) - ) - for shf_idx, shf_mode in enumerate(self._SHUFFLE_MODES): - for p_idx, color in shuffled_color_dicts[shf_mode].items(): - colors[shf_idx, p_idx] = color - - # Convert the colors to the requested colorblindness mode - daltonized_colors = daltonize.daltonize(colors, palette) - - # Go back through the image and get the new pixel colors - new_dict: Dict[str, Dict[int, Tuple[float, float, float]]] = {} - - for shuffle_idx in range(len(self._SHUFFLE_MODES)): - for color_idx in range(len(color_dict)): - shuffle_mode = shuffle_idx_to_mode[shuffle_idx] - if shuffle_mode not in new_dict: - new_dict[shuffle_mode] = {} - new_dict[shuffle_mode][color_idx] = \ - daltonized_colors[shuffle_idx, color_idx] - - for shuffle_mode in self._SHUFFLE_MODES: - self.brushes[shuffle_mode][palette] = \ - self.create_brush_dict(new_dict[shuffle_mode]) - - def set_shuffle_mode(self, shuffle_mode: str) -> None: - self.validate_shuffle_mode(shuffle_mode) - self.shuffle_mode = shuffle_mode - - def set_palette(self, palette: str) -> None: - self.validate_palette(palette) - self.palette = palette - - -BACKGROUND_BRUSHES = BrushRepository() -REASON_BRUSHES = BrushRepository() - - -def SetPalettes(palette: str) -> None: - BACKGROUND_BRUSHES.set_palette(palette) - REASON_BRUSHES.set_palette(palette) - - -def SetShuffleModes(mode: str) -> None: - BACKGROUND_BRUSHES.set_shuffle_mode(mode) - REASON_BRUSHES.set_shuffle_mode(mode) - - -# call after wx.App init -def BuildBrushes(colorblindness_mode: str, shuffle_mode: str) -> None: - global BACKGROUND_BRUSHES - global REASON_BRUSHES - # Map of background colors based on annotation content - # The first pass used pretty saturated colors, which was too dark, and - # hard to read on some monitors. I've altered it to use three different - # sequences of pastels, and hand-adjusted some to try to look better on - # the screen. - bgrayson - # There are 32 colors used right now, and 16 different shades makes the - # gradations too small, so let's do three passes of unequal sizes: - # 11, 11, and 10. - # The lightness of 0.9 is too light, so using 0.6, 0.7, and 0.8 - # I manually adjusted the last color in the first two slices to 10.5 to - # make it more distinguished. - BACKGROUND_COLORS = { - # -------------------------------------------------- - # New colorization method (based on sequence ID) - # - see transaction_viewer/argos_view/core/src/renderer.pyx - # - note: number of colors is set in renderer.pyx, in the - # variable NUM_ANNOTATION_COLORS - # - note2: if changes are made in renderer.pyx, then you must type - # "make" in transaction_viewer - # - 0: colorsys.hls_to_rgb(0 / 32.0, 0.75, 1), - 1: colorsys.hls_to_rgb(1 / 32.0, 0.75, 1), - 2: colorsys.hls_to_rgb(2 / 32.0, 0.75, 1), - 3: colorsys.hls_to_rgb(3 / 32.0, 0.75, 1), - 4: colorsys.hls_to_rgb(4 / 32.0, 0.75, 1), - 5: colorsys.hls_to_rgb(5 / 32.0, 0.75, 1), - 6: colorsys.hls_to_rgb(6 / 32.0, 0.75, 1), - 7: colorsys.hls_to_rgb(7 / 32.0, 0.75, 1), - 8: colorsys.hls_to_rgb(8 / 32.0, 0.75, 1), - 9: colorsys.hls_to_rgb(9 / 32.0, 0.75, 1), - 10: colorsys.hls_to_rgb(10 / 32.0, 0.75, 1), - - 11: colorsys.hls_to_rgb(11 / 32.0, 0.75, 1), - 12: colorsys.hls_to_rgb(12 / 32.0, 0.75, 1), - 13: colorsys.hls_to_rgb(13 / 32.0, 0.75, 1), - 14: colorsys.hls_to_rgb(14 / 32.0, 0.75, 1), - 15: colorsys.hls_to_rgb(15 / 32.0, 0.75, 1), - 16: colorsys.hls_to_rgb(16 / 32.0, 0.75, 1), - 17: colorsys.hls_to_rgb(17 / 32.0, 0.75, 1), - 18: colorsys.hls_to_rgb(18 / 32.0, 0.75, 1), - 19: colorsys.hls_to_rgb(19 / 32.0, 0.75, 1), - 20: colorsys.hls_to_rgb(20 / 32.0, 0.75, 1), - 21: colorsys.hls_to_rgb(21 / 32.0, 0.75, 1), - - 22: colorsys.hls_to_rgb(22 / 32.0, 0.75, 1), - 23: colorsys.hls_to_rgb(23 / 32.0, 0.75, 1), - 24: colorsys.hls_to_rgb(24 / 32.0, 0.75, 1), - 25: colorsys.hls_to_rgb(25 / 32.0, 0.75, 1), - 26: colorsys.hls_to_rgb(26 / 32.0, 0.75, 1), - 27: colorsys.hls_to_rgb(27 / 32.0, 0.75, 1), - 28: colorsys.hls_to_rgb(28 / 32.0, 0.75, 1), - 29: colorsys.hls_to_rgb(29 / 32.0, 0.75, 1), - 30: colorsys.hls_to_rgb(30 / 32.0, 0.75, 1), - 31: colorsys.hls_to_rgb(31 / 32.0, 0.75, 1), - } - - REASON_COLORS = { - 0: colorsys.hls_to_rgb(1.0, 1.0, 1.0), - 1: colorsys.hls_to_rgb(1 / 16.0, 0.75, 1), - 2: colorsys.hls_to_rgb(2 / 16.0, 0.75, 1), - 3: colorsys.hls_to_rgb(3 / 16.0, 0.75, 1), - 4: colorsys.hls_to_rgb(4 / 16.0, 0.75, 1), - 5: colorsys.hls_to_rgb(5 / 16.0, 0.75, 1), - 6: colorsys.hls_to_rgb(6 / 16.0, 0.75, 1), - 7: colorsys.hls_to_rgb(7 / 16.0, 0.75, 1), - 8: colorsys.hls_to_rgb(8 / 16.0, 0.75, 1), - 9: colorsys.hls_to_rgb(9 / 16.0, 0.75, 1), - 10: colorsys.hls_to_rgb(10 / 16.0, 0.75, 1), - - 11: colorsys.hls_to_rgb(11 / 16.0, 0.75, 1), - 12: colorsys.hls_to_rgb(12 / 16.0, 0.75, 1), - 13: colorsys.hls_to_rgb(13 / 16.0, 0.75, 1), - 14: colorsys.hls_to_rgb(14 / 16.0, 0.75, 1), - 15: colorsys.hls_to_rgb(15 / 16.0, 0.75, 1), - } - - BACKGROUND_BRUSHES.generate_all_brushes(BACKGROUND_COLORS) - REASON_BRUSHES.generate_all_brushes(REASON_COLORS) - SetPalettes(colorblindness_mode) - SetShuffleModes(shuffle_mode) diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/__init__.py b/helios/pipeViewer/pipe_view/gui/dialogs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/console_dialog.py b/helios/pipeViewer/pipe_view/gui/dialogs/console_dialog.py deleted file mode 100644 index 0cf8579fce..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/console_dialog.py +++ /dev/null @@ -1,62 +0,0 @@ - -from __future__ import annotations -from typing import TYPE_CHECKING -import wx -from wx.py.shell import Shell - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -# This class displays a python console -class ConsoleDlg(wx.Frame): - def __init__(self, parent: Layout_Frame) -> None: - self.__layout_frame = parent - self.__context = parent.GetContext() - - # create GUI - wx.Frame.__init__( - self, - parent, - -1, - 'Python Console', - size=(500, 300), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU) - ) - - menu_bar = wx.MenuBar() - menu = wx.Menu() - menu.Append(wx.NewId(), - '&Run Script\tAlt-R', - 'Run a script in the console.') - menu_bar.Append(menu, '&Shell') - self.SetMenuBar(menu_bar) - - self.__shell = Shell(self) - - # set up what user is given to work with - self.__shell.interp.locals = {'context': self.__context} - - # Hide instead of closing - self.Bind(wx.EVT_CLOSE, lambda evt: self.Hide()) - self.Bind(wx.EVT_MENU, self.OnRunScript) - - def OnRunScript(self, evt: wx.MenuEvent) -> None: - # Loop until user saves or cancels - dlg = wx.FileDialog(self, - 'Run Python Script', - wildcard='Python Scripts (*.py)|*.py', - style=wx.FD_OPEN | wx.FD_CHANGE_DIR) - dlg.ShowModal() - ret = dlg.GetReturnCode() - fp = dlg.GetPath() - dlg.Destroy() - - if ret == wx.ID_CANCEL: - return - self.__shell.write(f'Running Script... {fp}') - self.__shell.run(f'execfile(\'{fp}\')') diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/element_propsdlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/element_propsdlg.py deleted file mode 100644 index 96a6899ca1..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/element_propsdlg.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import annotations -import wx - -from ...model import element_types as eltypes -from ..widgets.element_property_list import ElementPropertyList -from ..font_utils import ScaleFont - -from typing import List, Optional, Tuple, Union, TYPE_CHECKING - -if TYPE_CHECKING: - from ...model.element import Element - from ..layout_canvas import Layout_Canvas - from ..layout_frame import Layout_Frame - from ..selection_manager import Selection_Mgr - - -# The GUI-side window for editing the properties of an Element -class Element_PropsDlg(wx.Frame): - # The constructor - def __init__(self, parent: Layout_Frame, id: int, title: str) -> None: - size = (500, 300) - self.__parent = parent - wx.Frame.__init__( - self, - parent, - id, - "Element Properties Editor", - size=size, - style=(wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.CLIP_CHILDREN | - wx.STAY_ON_TOP) - ) - - # used for temporary display of error messages - self.__timer = wx.Timer(self) - self.Bind(wx.EVT_TIMER, self.Draw, self.__timer) - self.SetBackgroundColour("WHITE") - self.CreateStatusBar() - - self.__fnt_location = wx.Font(ScaleFont(12), - wx.NORMAL, - wx.NORMAL, - wx.NORMAL) - self.SetFont(self.__fnt_location) - - self.__sizer = wx.BoxSizer(wx.VERTICAL) - self.__list = ElementPropertyList(self, id) - - self.__sizer.Add(self.__list, 1, wx.EXPAND) - self.SetSizerAndFit(self.__sizer) - self.SetAutoLayout(True) - - self.Show(True) - self.Move((200, 200)) # better than center - - self.Bind(wx.EVT_CLOSE, self.Hide) - self.Bind(wx.EVT_KEY_DOWN, self.__OnKeyDown) - - # Since each layout_frame has 1 element_propsdlg and their lifetimes are - # tied together, here we override the normal behaviour from clicking the - # close button, and simply hide the window from view, while keeping the - # object around. The next time the user want to see this dialog, rather - # than re-initializing or instantiating a new one, we Show this one and - # re-assign one or more Elements to it - def Hide(self, event: Optional[wx.CommandEvent] = None) -> bool: - return self.Show(False) - - def GetCanvas(self) -> Layout_Canvas: - return self.__parent.GetCanvas() - - # Gotta make sure the StatusBar displays the correct message - def Draw(self, evt: Optional[wx.TimerEvent] = None) -> None: - self.__timer.Stop() - font = self.GetStatusBar().GetFont() - font.SetWeight(wx.NORMAL) - self.GetStatusBar().SetFont(font) - self.GetStatusBar().SetBackgroundColour(wx.WHITE) - self.SetStatusText(f'{self.__list.GetNumberOfElements()} elements') - - def Refresh( - self, - eraseBackground: bool = True, - rect: Optional[Union[Tuple[int, int, int, int], wx.Rect]] = None - ) -> None: - self.__list.Refresh() - self.Layout() - self.Draw() - - # If the validation of setting an Element property fails / raises an - # error, this method will be told to update the status bar accordingly - def ShowError(self, error: Union[Exception, str]) -> None: - self.SetStatusText(str(error)) - font = self.GetStatusBar().GetFont() - font.SetWeight(wx.BOLD) - self.GetStatusBar().SetFont(font) - self.GetStatusBar().SetBackgroundColour(wx.RED) - self.__timer.Start(2200) - - # Simple & self-explanatory, but will eventually need to support - # multiple selections - # @note This must be the only method through which elements on which the - # dialog should operate can be set - def SetElements(self, - elements: List[Element], - sel_mgr: Selection_Mgr) -> None: - self.__list.SetElements(elements, sel_mgr) - self.Fit() # Resize window to fit contents - - # Key handler - def __OnKeyDown(self, evt: wx.KeyEvent) -> None: - ctrl = evt.ControlDown() - if evt.GetKeyCode() == ord('Z') and ctrl: - self.__parent.GetCanvas().GetSelectionManager().Undo() - elif evt.GetKeyCode() == ord('Y') and ctrl and evt.ShiftDown(): - self.__parent.GetCanvas().GetSelectionManager().RedoAll() - elif evt.GetKeyCode() == ord('Y') and ctrl: - self.__parent.GetCanvas().GetSelectionManager().Redo() - else: - evt.Skip() - - -# Type selection for element. -# Kind of fits in this module. It's small code so I'd rather not give it its -# own module. -class ElementTypeSelectionDialog(wx.Dialog): - - def __init__(self, parent: Layout_Canvas) -> None: - wx.Dialog.__init__(self, - parent, - wx.NewId(), - 'Select an Element Type', - size=(200, 70)) - - self.creatables = list(eltypes.creatables.keys()) - self.__drop_down = wx.ComboBox(self, - wx.NewId(), - choices=self.creatables, - style=wx.TE_PROCESS_ENTER) - # assume always one creatable - self.__drop_down.SetStringSelection(self.creatables[0]) - self.__drop_down.SetEditable(False) - done = wx.Button(self, wx.NewId(), 'Done') - sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(self.__drop_down, 1, 0, 0) - sizer.Add(done, 1, 0, 0) - self.SetSizer(sizer) - self.Bind(wx.EVT_TEXT_ENTER, self.OnDone, self.__drop_down) - self.Bind(wx.EVT_BUTTON, self.OnDone, done) - self.__type_string = '' - self.__drop_down.SetFocus() - - def GetSelection(self) -> str: - return self.__type_string - - def OnDone(self, evt: wx.CommandEvent) -> None: - idx = int(self.__drop_down.GetSelection()) - self.__type_string = self.creatables[idx] - # close off selection - self.EndModal(wx.ID_OK) diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/find_element_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/find_element_dlg.py deleted file mode 100644 index 1d11ebd960..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/find_element_dlg.py +++ /dev/null @@ -1,174 +0,0 @@ - - -from __future__ import annotations -import wx -from ..widgets.element_list import ElementList -from typing import Dict, List, Optional, Set, TYPE_CHECKING - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -# FindElementDialog is a window that enables the user to enter a string, -# conduct a search and jump to a location and element based on the result. -# It gets its data from search_handle.py -class FindElementDialog(wx.Frame): - START_COLUMN = 0 - LOCATION_COLUMN = 1 - ANNOTATION_COLUMN = 2 - - INITIAL_SEARCH = 0 - - def __init__(self, parent: Layout_Frame) -> None: - self.__layout_frame = parent - self.__context = parent.GetContext() - self.__full_results: List[Dict[str, str]] = [] - # initialize graphical part - title = f'Find Element in {self.__layout_frame.ComputeTitle()}' - wx.Frame.__init__(self, parent, - -1, - title, - size=(900, 600), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU)) - - main_sizer = wx.BoxSizer(wx.VERTICAL) - self.SetSizer(main_sizer) - - self.__search_sizer = wx.BoxSizer(wx.HORIZONTAL) - main_sizer.Add(self.__search_sizer, 0, wx.EXPAND, 5) - - choices: Set[str] = set() - for el in self.__context.GetElements(): - choices.update(el._properties.keys()) - self.__choices: List[str] = sorted(list(choices)) - - lbl_find = wx.StaticText(self, -1, "Elements where: ") - self.__drop_content = wx.ComboBox( - self, - choices=self.__choices, - size=(150, -1), - style=wx.CB_DROPDOWN | wx.CB_READONLY) - self.__drop_content.SetToolTip( - 'Select a content option on which to search' - ) - DEFAULT_CHOICE = 'name' - if DEFAULT_CHOICE in self.__choices: - self.__drop_content.SetValue(DEFAULT_CHOICE) - else: - self.__drop_content.SetSelection(0) - - lbl_comparison = wx.StaticText(self, -1, "contains") - self.__txt_input = wx.TextCtrl(self, -1, "") - self.__btn_submit = wx.Button(self, -1, "Find") - self.__search_sizer.Add(lbl_find, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.RIGHT, - 4) - self.__search_sizer.Add(self.__drop_content, 0, 0, 4) - self.__search_sizer.Add(lbl_comparison, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT, - 4) - self.__search_sizer.Add(self.__txt_input, 1, wx.EXPAND, 4) - self.__search_sizer.Add(self.__btn_submit, 0, 0, 4) - - self.__results_box = ElementList(self, - parent.GetCanvas(), - name='listbox', - properties=['']) - main_sizer.Add(self.__results_box, 1, wx.EXPAND, 5) - - # bind to events - self.__btn_submit.Bind(wx.EVT_BUTTON, self.OnSearch) - self.__txt_input.Bind(wx.EVT_KEY_DOWN, self.OnKeyPress) - - self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, - self.OnClickElement, - self.__results_box) - # Hide instead of closing - self.Bind(wx.EVT_CLOSE, lambda evt: self.Hide()) - - # defines how the dialog should pop up - def Show(self, show: bool = True) -> bool: - res = wx.Frame.Show(self, show) - self.Raise() - self.FocusQueryBox() - return res - - def FocusQueryBox(self) -> None: - self.__txt_input.SetFocus() - - # Sets the location in the location box - # @pre Requires there be no filters created because this means that the - # original search (which defines location) cannot be replaced. Has no - # effect if there are filters. - def SetSearchLocation(self, loc: str) -> None: - self.__txt_input.SetValue(loc) - - # callback that listens for enter being pressed to initiate search - def OnKeyPress(self, evt: wx.KeyEvent) -> None: - if evt.GetKeyCode() == wx.WXK_RETURN: - self.OnSearch(None) - else: - evt.Skip() - - def OnSearch(self, evt: Optional[wx.CommandEvent]) -> None: - self.__results_box.Clear() - self.__full_results = [] - self.__results_box.RefreshAll() - - prop = self.__drop_content.GetValue() - term = self.__txt_input.GetValue().lower() - - matches = [] - for el in self.__context.GetElements(): - # Check for match - if el.HasProperty(prop): - prop_val = el.GetProperty(prop) - if prop_val is not None and term in str(prop_val).lower(): - matches.append(el) - - # Assemble properties based on matches - properties_set: Set[str] = set() - for m in matches: - properties_set.update(m._properties.keys()) - properties = sorted(list(properties_set)) - - # If no properties found, assume no matches and terminate the search - if len(properties) == 0: - return - - self.__results_box.SetProperties(properties) - - # Add matches to the results table - for m in matches: - entry = {} - for p in properties: - if m.HasProperty(p): - entry[p] = str(m.GetProperty(p)) - else: - entry[p] = '' - - self.__full_results.append(entry) - self.__results_box.Add(m, entry) - - self.__results_box.FitColumns() - - # Attempts to select the element in the associated layout - def OnClickElement(self, evt: wx.ListEvent) -> None: - element = self.__results_box.GetElement(evt.GetIndex()) - if element is not None: - canvas = self.__layout_frame.GetCanvas() - if canvas.GetInputDecoder().GetEditMode(): - sel_mgr = canvas.GetSelectionManager() - sel_mgr.ClearSelection() - sel_mgr.Add(element) - else: - # Non-edit mode - pair = self.__context.GetElementPair(element) - canvas.GetSelectionManager().SetPlaybackSelected(pair) - self.__layout_frame.Refresh() diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/layout_exit_dialog.py b/helios/pipeViewer/pipe_view/gui/dialogs/layout_exit_dialog.py deleted file mode 100644 index 9e8b58beba..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/layout_exit_dialog.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations -from typing import Any, Callable, Optional, TYPE_CHECKING -import wx -import os - -import wx.lib.platebtn as platebtn -from ..font_utils import ScaleFont - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -# Prompts the user if there are unsaved changes when trying to close a layout -class LayoutExitDialog(wx.Dialog): - def __init__(self, frame: Layout_Frame) -> None: - super().__init__(frame, -1, title='Unsaved Changes to this Layout') - - layout = frame.GetContext().GetLayout() - assert layout is not None - layout_name = layout.GetFilename() - if layout_name is not None: - layout_name = os.path.split(layout_name)[-1] - else: - layout_name = '' - - fnt_filename = wx.Font(ScaleFont(13), - wx.NORMAL, - wx.NORMAL, - wx.FONTWEIGHT_BOLD) - - lbl_filename = wx.StaticText(self, -1, layout_name) - lbl_filename.SetFont(fnt_filename) - - BTN_ICON_SIZE = 16 - bmp_discard = wx.ArtProvider.GetBitmap(wx.ART_DELETE, - wx.ART_MESSAGE_BOX, - (BTN_ICON_SIZE, BTN_ICON_SIZE)) - bmp_save = wx.ArtProvider.GetBitmap(wx.ART_FILE_SAVE, - wx.ART_MESSAGE_BOX, - (BTN_ICON_SIZE, BTN_ICON_SIZE)) - bmp_return = wx.ArtProvider.GetBitmap(wx.ART_GO_BACK, - wx.ART_MESSAGE_BOX, - (BTN_ICON_SIZE, BTN_ICON_SIZE)) - - STYLE = platebtn.PB_STYLE_SQUARE - btn_discard = platebtn.PlateButton(self, - wx.ID_DELETE, - ' Discard Changes ', - bmp_discard, - style=STYLE) - btn_discard.SetBackgroundColour((240, 80, 80)) - btn_save = platebtn.PlateButton(self, - wx.ID_SAVE, - ' Save ', - bmp_save, - style=STYLE) - btn_save.SetBackgroundColour((170, 170, 170)) - btn_return = platebtn.PlateButton(self, - wx.ID_BACKWARD, - ' Back to Editing ', - bmp_return, - style=STYLE) - btn_return.SetBackgroundColour((170, 170, 170)) - - button_sizer = wx.BoxSizer(wx.HORIZONTAL) - button_sizer.Add(btn_discard, 0, wx.ALL, 4) - button_sizer.Add((50, 1), 0) - button_sizer.Add(btn_save, 0, wx.ALL, 4) - button_sizer.Add(btn_return, 0, wx.ALL, 4) - - message_sizer = wx.BoxSizer(wx.VERTICAL) - message_sizer.Add((1, 10), 0) - message_sizer.Add( - wx.StaticText(self, - -1, - 'Layout has been modified since last save!'), - 0, - wx.ALL, - 8 - ) - message_sizer.Add((1, 1), 0) - message_sizer.Add(lbl_filename, 0, wx.ALL, 8) - message_sizer.Add((1, 20), 0) - - bmp = wx.ArtProvider.GetBitmap(wx.ART_WARNING, - wx.ART_MESSAGE_BOX, - (64, 64)) - upper_sizer = wx.BoxSizer(wx.HORIZONTAL) - upper_sizer.Add(wx.StaticBitmap(self, - -1, - bmp, - (0, 0), - (bmp.GetWidth(), bmp.GetHeight())), - 0, - wx.ALL, - 10) - upper_sizer.Add(message_sizer, 1, wx.EXPAND) - - main_sizer = wx.BoxSizer(wx.VERTICAL) - main_sizer.Add(upper_sizer, 1, wx.EXPAND) - main_sizer.Add(button_sizer, 0, wx.EXPAND | wx.ALL, 5) - - self.SetSizer(main_sizer) - self.Layout() - self.Fit() - - self.__result: Optional[int] = None - - def GenAssigner(val: int) -> Callable: - def Assigner(*args: Any) -> None: - self.__result = val - self.EndModal(val) - return Assigner - - btn_discard.Bind(wx.EVT_BUTTON, GenAssigner(wx.ID_DELETE)) - btn_save.Bind(wx.EVT_BUTTON, GenAssigner(wx.ID_SAVE)) - btn_return.Bind(wx.EVT_BUTTON, GenAssigner(wx.ID_CANCEL)) - self.Bind(wx.EVT_CLOSE, GenAssigner(wx.ID_CANCEL)) - - # Show modal dialog. - # @return wx.ID_DELETE to discard changes, wx.ID_SAVE to save & quit, and - # wx.ID_CANCEL to return - def ShowModal(self) -> int: - super().ShowModal() - return self.GetReturnCode() - - def GetReturnCode(self) -> int: - assert self.__result is not None - return self.__result diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/layout_varsdlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/layout_varsdlg.py deleted file mode 100644 index 3c560d11fc..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/layout_varsdlg.py +++ /dev/null @@ -1,178 +0,0 @@ -from __future__ import annotations -import wx -from typing import Dict, List, Optional, Tuple, Union, TYPE_CHECKING - -import wx.lib.mixins.listctrl as listmix -from ..font_utils import ScaleFont - -if TYPE_CHECKING: - from ...model.layout_context import Layout_Context - - -# A helper class for building a list of variables and values in separate -# columns -class VarsListCtrl(wx.ListCtrl, - listmix.ListCtrlAutoWidthMixin, - listmix.TextEditMixin): - - # The Constructor - # Calls Populate() which actually sets up the ListCtrl - def __init__(self, - parent: LayoutVariablesDialog, - ID: int, - variables: Optional[Dict[str, str]] = None) -> None: - wx.ListCtrl.__init__(self, - parent, - ID, - style=(wx.LC_REPORT | - wx.BORDER_NONE | - wx.LC_SORT_ASCENDING)) - - self.__parent = parent - listmix.ListCtrlAutoWidthMixin.__init__(self) - self.__vars = variables if variables is not None else {} - self.__keys: List[str] = [] # index in list maps to variable name - self.__is_init = False - self.Populate() - listmix.TextEditMixin.__init__(self) - - # Convenience method for forwarding calls to Populate(), which clears - # and reconstitutes the table with up-to-date values - def Refresh( - self, - eraseBackground: bool = True, - rect: Optional[Union[wx.Rect, Tuple[int, int, int, int]]] = None - ) -> None: - self.Populate() - - # Returns the number of items in the dialog - def GetNumItems(self) -> int: - return self.GetItemCount() - - # Allows users to edit the second column (values) - def OpenEditor(self, col: int, row: int) -> None: - if col == 1: - super().OpenEditor(col, row) - - # Get's called when something is edited in the ListCtrl (in the GUI - # window, by the user) - def SetItem(self, - index: int, - col: int, - data: str, - imageId: int = -1) -> bool: - # hopefully whatever the user input is valid... (data will pass - # through the validation steps on the Element side) - if not self.__is_init: - var = self.__keys[index] - self.__vars[var] = data - - # go ahead and update the text fields on display and redraw the canvas - retcode = self.__UpdateItem(index) - - if not self.__is_init: - old_cursor = self.__parent.GetCursor() - new_cursor = wx.Cursor(wx.CURSOR_WAIT) - wx.SetCursor(new_cursor) - wx.SafeYield() - - try: - # self.__parent.GetContext().ReValueAll() - self.__parent.GetContext().ReSortAll() - self.__parent.GetContext().GoToHC() - self.__parent.GetContext().RefreshFrame() - except Exception as ex: - raise ex - finally: - wx.SetCursor(old_cursor) - wx.SafeYield() - return retcode - - # Used for creating the columns and rows and dumping in initial values - # (or current values every time the user changes the Element being edited) - def Populate(self) -> None: - # for normal, simple columns, you can add them like this: - self.DeleteAllColumns() - self.DeleteAllItems() - self.InsertColumn(0, 'Variable') - self.InsertColumn(5, 'Value', wx.LIST_FORMAT_LEFT) - - self.__keys = [] - - index = 0 - self.__is_init = True - for var in sorted(self.__vars.keys()): - self.__keys.append(var) - self.InsertItem(index, str(var)) - self.__UpdateItem(index) - index += 1 - self.__is_init = False - - self.SetColumnWidth(0, wx.LIST_AUTOSIZE) - self.SetColumnWidth(1, 100) - self.currentItem = 0 - - # Returns a wx color (or string equivalent) based on row index - def GetItemBackgroundColour(self, index: int) -> str: - if index % 2 != 0: - return "white" - return "gray" - - # Updates an item based on the current elements - # @param index Index of parameter - def __UpdateItem(self, index: int) -> bool: - # Default background color - self.SetItemBackgroundColour(index, - self.GetItemBackgroundColour(index)) - - if not self.__vars: - # Set to no value. Whole window should be disabled - return super().SetItem(index, 1, '') - - val = self.__vars[self.__keys[index]] - - return super().SetItem(index, 1, val) - - -# The GUI-side window for editing the properties of an Element -class LayoutVariablesDialog(wx.Frame): - - # The constructor - def __init__(self, - parent: wx.Window, - id: int, - title: str, - layout_context: Layout_Context) -> None: - self.__layout_context = layout_context - size = (600, 100) - self.__parent = parent - wx.Frame.__init__(self, - parent, - id, - title + " properties dialog", - size, - style=(wx.RESIZE_BORDER | - wx.SYSTEM_MENU | - wx.CAPTION | - wx.CLOSE_BOX | - wx.CLIP_CHILDREN)) - - self.__fnt_location = wx.Font(ScaleFont(12), - wx.NORMAL, - wx.NORMAL, - wx.NORMAL) - self.SetFont(self.__fnt_location) - - # work could be done to make these prettier - self.__sizer = wx.BoxSizer(wx.VERTICAL) - self.__list = VarsListCtrl( - self, - id, - variables=layout_context.GetLocationVariables() - ) - self.__sizer.Add(self.__list, 1, wx.EXPAND) - self.SetSizer(self.__sizer) - self.SetAutoLayout(True) - - def GetContext(self) -> Layout_Context: - return self.__layout_context diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/location_window.py b/helios/pipeViewer/pipe_view/gui/dialogs/location_window.py deleted file mode 100644 index 52c247ef5f..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/location_window.py +++ /dev/null @@ -1,367 +0,0 @@ -from __future__ import annotations -import fnmatch -from ..dialogs.element_propsdlg import ElementTypeSelectionDialog -import logging -import wx -import wx.lib.gizmos -from typing import Optional, Tuple, cast, TYPE_CHECKING - -from ..font_utils import ScaleFont - -if TYPE_CHECKING: - from ..dialogs.element_propsdlg import Element_PropsDlg - from ..layout_frame import Layout_Frame - from ...model.location_manager import LocationTree - - -class LocationWindow(wx.Frame): - ''' - Window showing all available locations in particular database - ''' - - # ID of node column - COL_NODE = 0 - - # ID of clock column - COL_CLOCK = 1 - - # ID of full-path column - COL_PATH = 2 - - def __init__(self, - parent: Layout_Frame, - elpropsdlg: Element_PropsDlg) -> None: - ''' - @param parent Parent Layout_Frame - @param elpropsdlg Element Properties Dialog for this frame - ''' - self.__layout_frame = parent - self.__el_props_dlg = elpropsdlg - - self.__db = parent.GetContext().dbhandle.database - self.__tree_dict: LocationTree = {} - - # Create Controls - - title = f"Locations for {self.__db.filename}" - wx.Frame.__init__(self, - parent, - -1, - title, - size=(1025, 600), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU)) - - self.__fnt_small = wx.Font(ScaleFont(12), - wx.NORMAL, - wx.NORMAL, - wx.NORMAL) - - static_heading = wx.StaticText(self, -1, "Showing all locations for:") - static_filename = wx.StaticText(self, -1, self.__db.filename) - static_filename.SetFont(self.__fnt_small) - filter_label = wx.StaticText(self, -1, "Filter") - self.__filter_ctrl = wx.TextCtrl(self, -1) - - # TODO style had wx.TR_EXTENDED but seemed missing from the API - self.__tree_ctrl = wx.lib.gizmos.treelistctrl.TreeListCtrl( - self, - -1, - size=wx.Size(-1, 100), - style=(wx.TR_DEFAULT_STYLE | - wx.TR_FULL_ROW_HIGHLIGHT | - wx.TR_MULTIPLE | - wx.TR_HIDE_ROOT) - ) - self.__tree_ctrl.AddColumn('Node') - self.__tree_ctrl.SetColumnWidth(self.COL_NODE, 400) - self.__tree_ctrl.AddColumn('Clock') - self.__tree_ctrl.SetColumnWidth(self.COL_CLOCK, 100) - self.__tree_ctrl.AddColumn('Full Path') - self.__tree_ctrl.SetColumnWidth(self.COL_PATH, 500) - - tree = self.__db.location_manager.location_tree - self.__root = self.__tree_ctrl.AddRoot('') - self.__use_filter = False - self.__SetLocationTree(tree) - - self.__btn_create_element = wx.Button(self, -1, "Create Element(s)") - self.__btn_create_element.SetToolTip( - 'Creates a new element for each of the selected locations in the ' - 'tree' - ) - self.__btn_set = wx.Button(self, -1, "Set Location to Selected") - self.__btn_set.SetToolTip( - 'Sets the location string for all selected elements to the ' - 'selected location in the tree. Generally, the selected location ' - 'should be a leaf node' - ) - - # Prepare - - self.__UpdateButtons() - - # Bindings - - self.__tree_ctrl.Bind(wx.EVT_TREE_SEL_CHANGED, self.__OnTreeSelChanged) - self.__btn_create_element.Bind(wx.EVT_BUTTON, self.__OnCreateElements) - self.__btn_set.Bind(wx.EVT_BUTTON, self.__OnSetLocation) - self.__filter_ctrl.Bind(wx.EVT_TEXT, self.__OnFilterChanged) - # Hide instead of closing - self.Bind(wx.EVT_CLOSE, lambda evt: self.Hide()) - self.__tree_ctrl.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.__OnExpandItem) - - # Layout - - button_sizer = wx.BoxSizer(wx.HORIZONTAL) - button_sizer.Add(self.__btn_create_element, 0, wx.RIGHT, 2) - button_sizer.Add(self.__btn_set, 0) - button_sizer.Add((1, 1), 1, wx.EXPAND) - - sz = wx.BoxSizer(wx.VERTICAL) - sz.Add((1, 3), 0) - sz.Add(static_heading, 0, wx.EXPAND | wx.ALL, 2) - sz.Add(static_filename, 0, wx.EXPAND | wx.ALL, 4) - filter_sizer = wx.BoxSizer(wx.HORIZONTAL) - filter_sizer.Add((2, 1), 0) - filter_sizer.Add(filter_label, 0, wx.EXPAND | wx.ALL) - filter_sizer.Add(self.__filter_ctrl, 0, wx.EXPAND | wx.ALL) - sz.Add(filter_sizer, 0, wx.EXPAND | wx.ALL) - sz.Add(self.__tree_ctrl, 2, wx.EXPAND | wx.ALL, 1) - sz.Add(button_sizer, 0, wx.EXPAND | wx.ALL, 4) - self.SetSizer(sz) - self.Bind(wx.EVT_ACTIVATE, self.__OnActivate) - - # Fire a text event to load the filter for the first time - wx.PostEvent(self.__filter_ctrl.GetEventHandler(), - wx.PyCommandEvent(wx.EVT_TEXT.typeId, self.GetId())) - - def __OnActivate(self, evt: wx.ActivateEvent) -> None: - ''' - Sets the focus on the filter text control when the window is shown - ''' - self.__filter_ctrl.SetFocus() - - def __SetLocationTree(self, tree: LocationTree) -> None: - ''' - Set the location tree dictionary for this tree view and populate the - top-level entities - ''' - self.__tree_ctrl.DeleteChildren(self.__root) - self.__tree_dict = {} - for key in sorted(tree.keys(), key=len): - child = self.__tree_ctrl.AppendItem(self.__root, key) - self.__tree_ctrl.SetItemText(child, key, self.COL_PATH) - self.__tree_dict[key] = child - - # set clock column - clock_id = self.__db.location_manager.getLocationInfo(key, {})[2] - if clock_id != self.__db.location_manager.NO_CLOCK: - clk = self.__db.clock_manager.getClockDomain(clock_id) - self.__tree_ctrl.SetItemText(child, clk.name, self.COL_CLOCK) - else: - self.__tree_ctrl.SetItemText(child, - '', - self.COL_CLOCK) - if tree[key] != {}: - self.__tree_ctrl.SetItemHasChildren(child, True) - - def __OnExpandItem(self, evt: wx.TreeEvent) -> None: - ''' - This intelligently populates the tree as its nodes are expanded instead - of doing it all at once - reduces time to open the popup - ''' - item = evt.GetItem() - path = self.__tree_ctrl.GetItemText(item, self.COL_PATH) - if path: - curdict = self.__loc_tree - for token in path.split('.'): - curdict = curdict[token] - self.SetTree(curdict, item, path) - - def SetTree(self, - tree: LocationTree, - item: Optional[wx.TreeItemId] = None, - path: str = "") -> None: - ''' - Adds every member at the top level of location dictionary "tree" to - node "item" - ''' - if item is None: - item = self.__root - # Sort keys by length ascending then string-comparison alphabetically - for k, v in sorted(tree.items(), key=len): - if not path: - child_path = k - else: - child_path = path + '.' + k - - if child_path in self.__tree_dict: - child = self.__tree_dict[child_path] - else: - child = self.__tree_ctrl.AppendItem(item, k) - self.__tree_ctrl.SetItemText(child, child_path, self.COL_PATH) - self.__tree_dict[child_path] = child - - # set clock column - clock_id = self.__db.location_manager.getLocationInfoNoVars(child_path)[2] # noqa: E501 - if clock_id != self.__db.location_manager.NO_CLOCK: - clk = self.__db.clock_manager.getClockDomain(clock_id) - self.__tree_ctrl.SetItemText(child, - clk.name, - self.COL_CLOCK) - else: - self.__tree_ctrl.SetItemText(child, - '', - self.COL_CLOCK) - - # Leaf nodes aren't expandable - if v: - self.__tree_ctrl.SetItemHasChildren(child, True) - - def __RecursiveExpand(self, - node: wx.TreeItemId, - limit: int, - level: int = 0) -> None: - ''' - Recursively expand tree nodes up to a given level - ''' - if node != self.__root: - self.__tree_ctrl.Expand(node) - - (cur_node, cookie) = self.__tree_ctrl.GetFirstChild(node) - while cur_node and cur_node.IsOk(): - if level < limit: - self.__RecursiveExpand(cur_node, limit, level + 1) - cur_node = self.__tree_ctrl.GetNextSibling(cur_node) - - def __GenerateFilteredTree(self, - tree: LocationTree, - path: str = '') -> LocationTree: - ''' - Generates a filtered tree to populate the location tree control - ''' - subtree = {} - for k, v in sorted(tree.items(), key=len): - if not path: - child_path = k - else: - child_path = path + '.' + k - - # Add the child to the subtree if the filter matches - if fnmatch.fnmatch(child_path, self.__filter): - subtree[k] = self.__GenerateFilteredTree(v, child_path) - # Otherwise, if it isn't a leaf node, then it might be the parent - # of a match - elif v != {}: - # So, check its children - result = self.__GenerateFilteredTree(v, child_path) - # If it isn't empty, then it has children that match, so we add - # it to the subtree - if result != {}: - subtree[k] = result - - return subtree - - def __OnFilterChanged(self, evt: wx.CommandEvent) -> None: - ''' - Handles changes to the filter - ''' - self.__loc_tree = self.__db.location_manager.location_tree - # If a filter was specified - if self.__filter_ctrl.GetValue(): - self.__use_filter = True - self.__filter = self.__filter_ctrl.GetValue() - # Generate a filtered ttee - self.__loc_tree = self.__GenerateFilteredTree(self.__loc_tree) - else: - self.__use_filter = False - # Setup the tree control with the (possibly) filtered tree - self.__SetLocationTree(self.__loc_tree) - # Expand everything up to the last level of the filter - if self.__use_filter: - self.__RecursiveExpand(self.__root, self.__filter.count('.')) - - def __OnTreeSelChanged(self, evt: wx.TreeEvent) -> None: - ''' - Handles selection changes in the location tree - Updates button enable/caption states based on selection - ''' - self.__UpdateButtons() - - def __OnCreateElements(self, evt: wx.CommandEvent) -> None: - ''' - Handles Clicks on "create elements" button - Sets the selected location string as the location for the entire - selection - ''' - sels = self.__tree_ctrl.GetSelections() - assert sels, ('Create Elements Location button should not be enabled ' - 'if there were no locations selected') - - canvas = self.__layout_frame.GetCanvas() - sel_mgr = canvas.GetSelectionManager() - layout = self.__layout_frame.GetContext().GetLayout() - assert layout is not None - canvas_visible = self.__layout_frame.GetCanvas().GetVisibleArea() - - sel_mgr.Clear() - - # Place in middle of the screen - last_el_pos = (int(canvas_visible[0] + canvas_visible[2] / 2), - int(canvas_visible[1] + canvas_visible[3] / 2)) - - if not canvas.GetInputDecoder().GetEditMode(): - # go into edit mode - self.__layout_frame.SetEditMode(True) - type_dialog = ElementTypeSelectionDialog(canvas) - type_dialog.ShowModal() - type_dialog.Center() - for item in sels: - # Create and add to selection - e = sel_mgr.GenerateElement(layout, - type_dialog.GetSelection(), - add_to_selection=True) - e.SetProperty('position', last_el_pos) - - loc_str = self.__tree_ctrl.GetItemText(item, self.COL_PATH) - e.SetProperty('LocationString', loc_str) - dims = cast(Tuple[int, int], e.GetProperty('dimensions')) - - # Stack elements by default - last_el_pos = (last_el_pos[0], last_el_pos[1] + dims[1]) - - # Need to reflect new selection in element properties dialog - self.__el_props_dlg.Refresh() - - def __OnSetLocation(self, evt: wx.CommandEvent) -> None: - ''' - Handles Clicks on "set location" button - Sets the selected location string as the location for the entire - selection - ''' - sels = self.__tree_ctrl.GetSelections() - assert len(sels) == 1, ('Set Location button should not be enabled if ' - 'there was not exactly one location selected') - item = sels[0] - els = self.__layout_frame.GetCanvas().GetSelectionManager().GetSelection() # noqa: E501 - loc_str = self.__tree_ctrl.GetItemText(item, self.COL_PATH) - logging.debug('Setting all selection location to "%s"', loc_str) - for e in els: - e.SetProperty('LocationString', loc_str) - - # Elements changed. Need to redraw and reflect changes in element - # properties dialog - self.__layout_frame.Refresh() - self.__el_props_dlg.Refresh() - - def __UpdateButtons(self) -> None: - ''' - Enables buttons based on the selection state - ''' - sels = self.__tree_ctrl.GetSelections() - self.__btn_set.Enable(len(sels) == 1) - self.__btn_create_element.Enable(len(sels) > 0) diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/search_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/search_dlg.py deleted file mode 100644 index 8fcc8cf687..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/search_dlg.py +++ /dev/null @@ -1,384 +0,0 @@ -from __future__ import annotations -import sys -import wx -from functools import partial -from ..widgets.transaction_list import TransactionList -from ..widgets.location_entry import LocationEntry -from typing import List, Optional, Tuple, TypedDict, TYPE_CHECKING - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -class SearchResult(TypedDict): - start: int - location: int - annotation: str - - -# SearchDialog is a window that enables the user to enter a string, conduct a -# search and jump to a location and transaction based on the result. It gets -# its data from search_handle.py -class SearchDialog(wx.Frame): - START_COLUMN = 0 - LOCATION_COLUMN = 1 - ANNOTATION_COLUMN = 2 - - INITIAL_SEARCH = 0 - - def __init__(self, parent: Layout_Frame) -> None: - self.__context = parent.GetContext() - self.__canvas = parent.GetCanvas() - self.__search_handle = self.__context.searchhandle - self.__full_results: List[SearchResult] = [] - self.__filters: List[SearchFilter] = [] - # initialize graphical part - wx.Frame.__init__(self, - parent, - -1, - 'Search', - size=(700, 600), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU)) - - main_sizer = wx.BoxSizer(wx.VERTICAL) - self.__filter_sizer = wx.BoxSizer(wx.VERTICAL) - main_sizer.Add(self.__filter_sizer, 0, wx.EXPAND, 5) - - # -- Initial Search Settings -- - self.__initial_search = wx.Panel(self, wx.NewId()) - sizer = wx.BoxSizer(wx.VERTICAL) - sizerbuttons = wx.BoxSizer(wx.HORIZONTAL) - self.__input = wx.TextCtrl(self.__initial_search, wx.NewId()) - sizerbuttons.Add(self.__input, 2) - search = wx.Button(self.__initial_search, wx.NewId(), 'Search') - sizerbuttons.Add(search, 1) - sizer.Add(sizerbuttons, 0, wx.EXPAND) - - sizer.Add(wx.StaticText(self.__initial_search, - wx.NewId(), - 'Location:')) - - location_tree = self.__context.dbhandle.database.location_manager.location_tree # noqa: E501 - - self.__location_to_search = LocationEntry(self.__initial_search, - 'top', - location_tree) - sizer.Add(self.__location_to_search, 0, wx.EXPAND) - - self.__use_regex = wx.CheckBox(self.__initial_search, - wx.NewId(), - 'Use Regex') - self.__use_regex.SetValue(False) - sizer.Add(self.__use_regex) - - self.__exclude_locations_not_shown = wx.CheckBox( - self.__initial_search, - wx.NewId(), - 'Exclude Locations Not Shown' - ) - self.__exclude_locations_not_shown.SetValue(False) - sizer.Add(self.__exclude_locations_not_shown) - - self.__from_current_tick = wx.CheckBox(self.__initial_search, - wx.NewId(), - 'Search From Current Tick') - self.__from_current_tick.SetValue(True) - sizer.Add(self.__from_current_tick) - - self.__colorize = wx.CheckBox( - self.__initial_search, - wx.NewId(), - 'Colorize Results (note: disregards color_basis)' - ) - self.__colorize.SetValue(True) - sizer.Add(self.__colorize) - self.__filter_sizer.Add(self.__initial_search, 4, wx.EXPAND) - - # Filter editor - filter_editor = wx.Panel(self, wx.NewId(), style=wx.BORDER_SUNKEN) - filter_sizer = wx.BoxSizer(wx.HORIZONTAL) - entry_text = wx.StaticText(filter_editor, - wx.NewId(), - 'Filter by (string): \n(annotation)') - filter_sizer.Add(entry_text, 2, wx.ALIGN_LEFT) - self.__new_filter_query = wx.TextCtrl(filter_editor, wx.NewId()) - filter_sizer.Add(self.__new_filter_query, 4, wx.ALIGN_LEFT) - add_filter = wx.Button(filter_editor, wx.NewId(), 'Add Filter') - filter_sizer.Add(add_filter, 2, wx.ALIGN_RIGHT) - filter_editor.SetSizer(filter_sizer) - - self.__filter_sizer.Add(filter_editor, 0, wx.EXPAND) - self.__filter_sizer.Hide(1) - - # Everything else - self.__results_box = TransactionList(self, - parent.GetCanvas(), - name='listbox') - - main_sizer.Add(self.__results_box, 1, wx.EXPAND) - self.__initial_search.SetSizer(sizer) - self.SetSizer(main_sizer) - - # bind to events - search.Bind(wx.EVT_BUTTON, self.OnSearch) - add_filter.Bind(wx.EVT_BUTTON, self.__OnAddFilter) - self.__input.Bind(wx.EVT_KEY_DOWN, self.OnKeyPress) - self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, - self.OnClickTransaction, - self.__results_box) - # Hide instead of closing - self.Bind(wx.EVT_CLOSE, lambda evt: self.Hide()) - - # defines how the dialog should pop up - def Show(self, show: bool = True) -> bool: - res = wx.Frame.Show(self, show) - self.Raise() - self.FocusQueryBox() - return res - - def FocusQueryBox(self) -> None: - self.__input.SetFocus() - - # Sets the location in the location box - # @pre Requires there be no filters created because this means that the - # original search (which defines location) cannot be replaced. Has no - # effect if there are filters. - def SetSearchLocation(self, loc: str) -> None: - if not self.__filters: - self.__location_to_search.SetValue(loc) - - # callback that listens for enter being pressed to initiate search - def OnKeyPress(self, evt: wx.KeyEvent) -> None: - if evt.GetKeyCode() == wx.WXK_RETURN: - self.OnSearch(None) - else: - evt.Skip() - - def __AddResult(self, entry: SearchResult) -> None: - self.__results_box.Add(entry) - self.__context.AddSearchResult(entry) - - def __ClearResults(self) -> None: - self.__results_box.Clear() - self.__context.ClearSearchResults() - - def __UpdateSearchHighlighting(self) -> None: - self.__canvas.UpdateTransactionHighlighting() - self.__context.RedrawHighlightedElements() - self.__canvas.FullUpdate() - - def ApplyFilters(self) -> None: - # full N time complexity for any call because not doing incrementally - # to speed up, keep track of currently already applied filters - self.__results_box.Colorize(self.__colorize.GetValue()) - self.__ClearResults() - self.__results_box.RefreshAll() - for entry in self.__full_results: - for filter in self.__filters: - if entry['annotation'].find(filter.query) == -1: - break - else: - # we have a result - self.__AddResult(entry) - - self.__UpdateSearchHighlighting() - - def OnSearch(self, evt: Optional[wx.CommandEvent]) -> None: - wx.BeginBusyCursor() - query = self.__input.GetValue() - dialog = wx.ProgressDialog( - 'Progress', - 'Searching Database...', - 100, # Maximum - parent=self, - style=wx.PD_CAN_ABORT | wx.PD_REMAINING_TIME - ) - - # Callback adapter to forward to wx.ProgressDialog.Update while - # ignoring some args - def progress_update(percent: int, - num_results: int, - info: str) -> Tuple[bool, bool]: - return dialog.Update(percent, info) - - if self.__use_regex.GetValue(): - mode = 'regex' - else: - mode = 'string' - - if self.__from_current_tick.GetValue(): - start_tick = self.__context.hc - else: - start_tick = None # indicate to use db start - - try: - results = self.__search_handle.Search(mode, - query, - start_tick, - -1, # end_tick - [], # locations - progress_update) - except Exception: - raise - finally: - dialog.Close() - dialog.Destroy() - - # Limit to search size - SEARCH_LIMIT = 10000 # @todo Move this into Search function - - self.__results_box.Colorize(self.__colorize.GetValue()) - self.__ClearResults() - del self.__full_results - self.__full_results = [] - self.__results_box.RefreshAll() - visible_locations = self.__context.GetVisibleLocations() - - location_root_search = self.__location_to_search.GetValue() - if location_root_search is None: - location_root_search = "[]" - truncated = False - - def loc_str(loc_id: int) -> str: - loc = self.__context.dbhandle.database.location_manager.getLocationString(loc_id) # noqa: E501 - loc = str(loc).translate({ord(c): None for c in '[]'}) - return loc - - for start, end, loc_id, annotation in results: - if loc_str(loc_id).startswith(location_root_search) and \ - (not self.__exclude_locations_not_shown.GetValue() or - loc_id in visible_locations): - entry: SearchResult = { - 'start': start, - 'location': loc_id, - 'annotation': annotation - } - self.__full_results.append(entry) - self.__AddResult(entry) - if len(self.__full_results) == SEARCH_LIMIT: - truncated = True - break - - self.__results_box.FitColumns() - self.__filter_sizer.Hide(self.INITIAL_SEARCH) - self.__filter_sizer.Show(1) # current index of filter editor - self.__AddFilter(SearchFilter(query, - True, - location=location_root_search, - num_results=len(self.__full_results))) - self.Layout() - - self.__UpdateSearchHighlighting() - - wx.EndBusyCursor() - - if truncated is True: - msg = f'Truncated search results to first {SEARCH_LIMIT} ' \ - f'results of {len(results)} total' - # Do not truncate here. This contains other locations. - print(msg, file=sys.stderr) - wx.MessageBox(msg) - - # deletes filter and updates results. - # index is indexed on filters not sizers - # e.g. (0 is first filter, not search) - def __OnRemoveFilter(self, - evt: wx.CommandEvent, - obj: SearchFilter) -> None: - index = self.__filters.index(obj) - self.__RemoveFilter(index) - self.ApplyFilters() - self.Layout() - - def __RemoveFilter(self, index: int) -> None: - filter_obj = self.__filters.pop(index) - panel = filter_obj.GetPanel() - if panel is not None: - self.__filter_sizer.Detach(panel) - panel.Destroy() - # never use this filter object again - - def OnClickTransaction(self, evt: wx.ListEvent) -> None: - transaction = self.__results_box.GetTransaction(evt.GetIndex()) - start_loc = transaction.get('start') - if start_loc: - self.__context.GoToHC(start_loc) - - def __OnResetSearch(self, evt: wx.CommandEvent) -> None: - while self.__filters: - self.__RemoveFilter(0) - - self.__filter_sizer.Show(self.INITIAL_SEARCH, True) - # hide filter addition textbox - self.__filter_sizer.Hide(1) - self.Layout() - self.FocusQueryBox() - - def __AddFilter(self, filter_object: SearchFilter) -> None: - panel, remove_button = filter_object.MakePanel(parent=self) - index = len(self.__filters) - self.__filter_sizer.Insert(index, panel, 0, wx.EXPAND) - self.__filters.append(filter_object) - # bind buttom - if filter_object.initial: - panel.Bind(wx.EVT_BUTTON, self.__OnResetSearch) - else: - panel.Bind(wx.EVT_BUTTON, partial(self.__OnRemoveFilter, - obj=filter_object)) - - def __OnAddFilter(self, evt: wx.CommandEvent) -> None: - query = self.__new_filter_query.GetValue() - if query: - self.__AddFilter(SearchFilter(query, False)) - self.ApplyFilters() - self.Layout() - self.__new_filter_query.Clear() - - -# stores the settings for each filter applied -class SearchFilter: - def __init__(self, - query: str, - initial: bool = False, - regex: bool = False, - location: str = '', - num_results: Optional[int] = None) -> None: - # string used for search - self.query = query - self.initial = initial - self.location = location - self.num_results = num_results - self.__panel: Optional[wx.Panel] = None - self.__remove: Optional[wx.Button] = None - - # Get panel, none if no panel - def GetPanel(self) -> Optional[wx.Panel]: - return self.__panel - - # make visual portion - def MakePanel(self, parent: SearchDialog) -> Tuple[wx.Panel, wx.Button]: - if self.__panel is None: - self.__panel = wx.Panel(parent, wx.NewId(), style=wx.BORDER_SUNKEN) - sizer = wx.BoxSizer(wx.HORIZONTAL) - if self.initial: - ratio = 2 - string = f'Initial Query: {self.query}\n' \ - f'Location: {self.location}' - if self.num_results is not None: - string += f'\nResults: {self.num_results}' - button_string = 'Reset Search' - else: - ratio = 1 - string = 'Filter by: '+self.query - button_string = 'Remove' - info = wx.StaticText(self.__panel, -1, string) - sizer.Add(info, 4, wx.ALIGN_LEFT) - self.__remove = wx.Button(self.__panel, wx.NewId(), button_string) - sizer.Add(self.__remove, ratio, wx.ALIGN_RIGHT) - self.__panel.SetSizer(sizer) - assert self.__remove is not None - return self.__panel, self.__remove diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/select_db_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/select_db_dlg.py deleted file mode 100644 index 2cb84b33c6..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/select_db_dlg.py +++ /dev/null @@ -1,195 +0,0 @@ -# @package select_db_dbg.py -# @brief Dialog for selecting a database file - -from __future__ import annotations -import os -from typing import Optional -import wx -import wx.lib.scrolledpanel as scrolledpanel - - -# Dialog for selecting an Argos database. -# -# Use ShowModal to display the dialog and then use GetPrefix to see selected -# filename -class SelectDatabaseDlg(wx.Dialog): - - # File name and extension for simulation info files - # - # This can be appended to a prefix to get simulation information - INFO_FILE_EXTENSION = '.info' - - # Initialized the dialog - # @param init_prefix Value of prefix to show in the box by default. - # Must be a str or None - def __init__(self, init_prefix: Optional[str] = None) -> None: - wx.Dialog.__init__(self, - None, - title='Select an Argos transaction database', - size=(800, 380)) - - if not isinstance(init_prefix, str) and init_prefix is not None: - raise TypeError( - f'init_prefix must be a str or None, is a {type(init_prefix)}' - ) - - self.__prefix: Optional[str] = None # Updated in CheckSelectionState - - if init_prefix is not None: - filepath = init_prefix + self.INFO_FILE_EXTENSION - else: - filepath = os.getcwd() - - # Controls - info = wx.StaticText( - self, - label=f'Specify a {self.INFO_FILE_EXTENSION} file from an argos ' - 'transaction database' - ) - info.Wrap(self.GetSize()[0] - 20) - - self.__file_txt = wx.TextCtrl(self, size=(160, -1), value=filepath) - self.__orig_txt_colour = self.__file_txt.GetBackgroundColour() - file_btn = wx.Button(self, id=wx.ID_FIND) - - quit_btn = wx.Button(self, id=wx.ID_EXIT) - self.__ok_btn = wx.Button(self, id=wx.ID_OK) - - file_info_box = wx.StaticBox(self, label='Simulation Info') - self.__scroll_win = scrolledpanel.ScrolledPanel(self) - self.__scroll_win.SetupScrolling() - self.__file_info = wx.StaticText(self.__scroll_win, label='') - - # Bindings - - quit_btn.Bind(wx.EVT_BUTTON, self.__OnClose) - self.__ok_btn.Bind(wx.EVT_BUTTON, self.__OnOk) - file_btn.Bind(wx.EVT_BUTTON, self.__OnFindFile) - self.__file_txt.Bind(wx.EVT_TEXT, self.__OnChangeFilename) - - # Layout - sbs = wx.StaticBoxSizer(file_info_box, wx.HORIZONTAL) - sbs.Add(self.__scroll_win, 1, wx.EXPAND | wx.ALL, 5) - - sws = wx.BoxSizer(wx.VERTICAL) - sws.Add(self.__file_info, 0, wx.EXPAND) - self.__scroll_win.SetSizer(sws) - sws.Fit(self.__scroll_win) - - open_row = wx.BoxSizer(wx.HORIZONTAL) - open_row.Add(self.__file_txt, 1, wx.EXPAND) - open_row.Add((10, 1), 0, wx.EXPAND) - open_row.Add(file_btn, 0, wx.EXPAND) - - buttons_row = wx.BoxSizer(wx.HORIZONTAL) - buttons_row.Add(quit_btn, 0, wx.ALIGN_LEFT | wx.ALIGN_BOTTOM) - buttons_row.Add((1, 1), 1, wx.EXPAND) - buttons_row.Add(self.__ok_btn, 0, wx.ALIGN_BOTTOM) - - sz = wx.BoxSizer(wx.VERTICAL) - sz.Add(info, 0, wx.EXPAND) - sz.Add((1, 15), 0, wx.EXPAND) - sz.Add(open_row, 0, wx.EXPAND) - sz.Add((1, 25), 0, wx.EXPAND) - sz.Add(sbs, 1, wx.EXPAND) - sz.Add((1, 25), 0, wx.EXPAND) - sz.Add(buttons_row, 0, wx.EXPAND) - - border = wx.BoxSizer(wx.VERTICAL) - border.Add(sz, 1, wx.EXPAND | wx.ALL, 10) - self.SetSizer(border) - - self.SetAutoLayout(True) - - self.__CheckSelectionState() - - def Show(self, show: bool = True) -> bool: - raise NotImplementedError( - 'Cannot Show() this dialog. Use ShowModal instead' - ) - - # Gets the prefix selected by the dialog - # @return The prefix selected while the dialog was shown. Is a string if - # found and None if no database was chosen - # - # This should be checked after invoking ShowModal() on this object - def GetPrefix(self) -> Optional[str]: - return self.__prefix - - # Handler for Close button - def __OnClose(self, evt: wx.CommandEvent) -> None: - self.__prefix = None - self.EndModal(wx.CANCEL) - - # Handler for Ok button - def __OnOk(self, evt: wx.CommandEvent) -> None: - # self.__filename already set before this button was enabled - self.EndModal(wx.OK) - - # Handler for Find button - def __OnFindFile(self, evt: wx.CommandEvent) -> None: - ext = self.INFO_FILE_EXTENSION - dlg = wx.FileDialog( - self, - "Select Argos database simulation.info file", - defaultFile=self.__file_txt.GetValue(), - wildcard=f'Argos Simulation info files (*{ext})|*{ext}' - ) - dlg.ShowModal() - - fp = dlg.GetPath() - - if fp is not None and fp != '': - self.__file_txt.SetValue(fp) - - self.__CheckSelectionState() - - # Handler for Changing the filename in file_txt - def __OnChangeFilename(self, evt: wx.CommandEvent) -> None: - self.__CheckSelectionState() - - # Checks on the value in the self.__file_txt box to see if it points to a - # valid simulation - # - # Updates self.__prefix - # Updates or clears self.__file_info and en/disables self.__ok_btn - # depending on whether selection points to a valid file. Also changes - # colors of box - def __CheckSelectionState(self) -> None: - filepath = self.__file_txt.GetValue() - - simulation_file_full_extention = 'simulation'+self.INFO_FILE_EXTENSION - suffix_pos = filepath.find(simulation_file_full_extention) - if suffix_pos != len(filepath) - len(simulation_file_full_extention): - valid = False - elif not os.path.exists(filepath): - valid = False - else: - try: - summary = '' - with open(filepath, 'r') as f: - while True: - ln = f.readline() - if ln == '': - break - summary += ln - - self.__file_info.SetLabel(summary) - - except IOError: - valid = False - else: - valid = True - - self.__ok_btn.Enable(valid) - - if valid: - self.__prefix = filepath[:suffix_pos] - self.__file_txt.SetBackgroundColour(wx.Colour(235, 255, 235)) - else: - self.__prefix = None - self.__file_info.SetLabel('') - self.__file_txt.SetBackgroundColour(wx.Colour(255, 220, 220)) - - self.__scroll_win.FitInside() - self.__scroll_win.Layout() diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/select_layout_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/select_layout_dlg.py deleted file mode 100644 index a3ba0f4186..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/select_layout_dlg.py +++ /dev/null @@ -1,209 +0,0 @@ -# @package select_layout_dbg.py -# @brief Dialog for selecting a layout file - -from __future__ import annotations -import os -import wx -from typing import Optional - -from ...model.layout import Layout - - -# Dialog for selecting an Argos layout file. -# -# Use ShowModal to display the dialog and then use GetPrefix to see selected -# filename -class SelectLayoutDlg(wx.Dialog): - - # Initialized the dialog - # @param init_prefix Value of the filename to show in the alf file textbox - # by default - # Must be a str or None. - def __init__(self, init_prefix: Optional[str] = None) -> None: - wx.Dialog.__init__(self, - None, - title='Select an Argos layout file', - size=(800, -1)) - - if not isinstance(init_prefix, str) and init_prefix is not None: - raise TypeError( - f'init_prefix must be a str or None, is a {type(init_prefix)}' - ) - - self.__filename: Optional[str] = None # Updated in CheckSelectionState - - if init_prefix is not None: - filepath = init_prefix - else: - filepath = os.getcwd() - - # Controls - info = wx.StaticText( - self, - label='The layout controls how information in the transaction ' - 'database is displayed on the screen. Once loaded, layouts ' - 'can be modified. A "shared" layout can be can be modified ' - 'by several view frames simultaneously\n' - ) - info.Wrap(self.GetSize()[0]-5) - - self.__rad_blank = wx.RadioButton(self, -1, 'Create a blank Layout') - self.__rad_load = wx.RadioButton( - self, - -1, - f'Specify an Argos layout file ({Layout.LAYOUT_FILE_EXTENSION})' - ) - - self.__panel_sel_file = wx.Panel(self) # To be enabled/disabled - self.__file_txt = wx.TextCtrl(self.__panel_sel_file, - size=(160, -1), - value=filepath) - self.__orig_txt_colour = self.__file_txt.GetBackgroundColour() - file_btn = wx.Button(self.__panel_sel_file, id=wx.ID_FIND) - - self.__chk_shared = wx.CheckBox( - self, - label='Shared Layout (NOT YET IMPLEMENTED)' - ) - self.__chk_shared.Disable() # Until implemented - - quit_btn = wx.Button(self, id=wx.ID_EXIT) - self.__ok_btn = wx.Button(self, id=wx.ID_OK) - - # Bindings - - quit_btn.Bind(wx.EVT_BUTTON, self.__OnClose) - self.__ok_btn.Bind(wx.EVT_BUTTON, self.__OnOk) - file_btn.Bind(wx.EVT_BUTTON, self.__OnFindFile) - self.__file_txt.Bind(wx.EVT_TEXT, self.__OnChangeFilename) - # Update if user changes radio selection - self.Bind(wx.EVT_RADIOBUTTON, self.__OnChangeFilename) - - # Layout - - INDENT = 35 - open_row = wx.BoxSizer(wx.HORIZONTAL) - open_row.Add((INDENT, 1), 0) - open_row.Add(self.__file_txt, 1, wx.EXPAND) - open_row.Add((10, 1), 0, wx.EXPAND) - open_row.Add(file_btn, 0, wx.EXPAND) - self.__panel_sel_file.SetSizer(open_row) - - sbs_src_border = wx.BoxSizer(wx.VERTICAL) - sbs_src_border.Add(self.__rad_blank, 0, wx.EXPAND) - sbs_src_border.Add((1, 25), 0, wx.EXPAND) - sbs_src_border.Add(self.__rad_load, 0, wx.EXPAND) - sbs_src_border.Add((1, 15), 0, wx.EXPAND) - sbs_src_border.Add(self.__panel_sel_file, 0, wx.EXPAND) - - sbs_src = wx.StaticBoxSizer(wx.VERTICAL, self, 'Layout File') - sbs_src.Add(sbs_src_border, 1, wx.EXPAND | wx.ALL, 10) - - sbs_opts_border = wx.BoxSizer(wx.VERTICAL) - sbs_opts_border.Add(self.__chk_shared, 0, wx.EXPAND) - - sbs_opts = wx.StaticBoxSizer(wx.VERTICAL, self, 'Layout Options') - sbs_opts.Add(sbs_opts_border, 1, wx.EXPAND | wx.ALL, 10) - - buttons_row = wx.BoxSizer(wx.HORIZONTAL) - buttons_row.Add(quit_btn, 0, wx.ALIGN_LEFT | wx.ALIGN_BOTTOM) - buttons_row.Add((1, 1), 1, wx.EXPAND) - buttons_row.Add(self.__ok_btn, 0, wx.ALIGN_BOTTOM) - - sz = wx.BoxSizer(wx.VERTICAL) - sz.Add(info, 0, wx.EXPAND) - sz.Add(sbs_src, 1, wx.EXPAND) - sz.Add((1, 15), 0, wx.EXPAND) - sz.Add(sbs_opts, 1, wx.EXPAND) - sz.Add((1, 15), 0, wx.EXPAND) - sz.Add(buttons_row, 0, wx.EXPAND) - - border = wx.BoxSizer(wx.VERTICAL) - border.Add(sz, 1, wx.EXPAND | wx.ALL, 10) - self.SetSizer(border) - - self.Fit() - self.SetAutoLayout(True) - - self.__CheckSelectionState() - - def Show(self, show: bool = True) -> bool: - raise NotImplementedError( - 'Cannot Show() this dialog. Use ShowModal instead' - ) - - # Gets the Argos layout filename selected by the dialog - # @return The filename selected while the dialog was shown. Is a string if - # found and None if no database was chosen - # - # This should be checked after invoking ShowModal() on this object - def GetFilename(self) -> Optional[str]: - return self.__filename - - # Handler for Close button - def __OnClose(self, evt: wx.CommandEvent) -> None: - self.__filename = None - self.EndModal(wx.CANCEL) - - # Handler for Ok button - def __OnOk(self, evt: wx.CommandEvent) -> None: - # self.__filename already set before this button was enabled - self.EndModal(wx.OK) - - # Handler for Find button - def __OnFindFile(self, evt: wx.CommandEvent) -> None: - ext = Layout.LAYOUT_FILE_EXTENSION - dlg = wx.FileDialog( - self, - "Select Argos layout file", - defaultFile=self.__file_txt.GetValue(), - wildcard=f'Argos layout files (*{ext})|*{ext}' - ) - dlg.ShowModal() - - fp = dlg.GetPath() - if fp is not None and fp != '': - self.__file_txt.SetValue(fp) - - self.__CheckSelectionState() - - # Handler for Changing the filename in file_txt - def __OnChangeFilename(self, evt: wx.CommandEvent) -> None: - self.__CheckSelectionState() - - # Checks on the value in the self.__file_txt box to see if it points to a - # valid simulation - # - # Updates self.__filename - # Updates or clears self.__file_info and en/disables self.__ok_btn - # depending on whether selection points to a valid file. Also changes - # colors of box - def __CheckSelectionState(self) -> None: - if self.__rad_blank.GetValue(): - self.__panel_sel_file.Disable() - filepath = None - valid = True - self.__file_txt.SetBackgroundColour(self.__orig_txt_colour) - else: - self.__panel_sel_file.Enable() - filepath = self.__file_txt.GetValue() - suffix_pos = filepath.find(Layout.LAYOUT_FILE_EXTENSION) - if suffix_pos != len(filepath) - len(Layout.LAYOUT_FILE_EXTENSION): - valid = False - elif not os.path.exists(filepath): - valid = False - else: - # Assume the file is valid - valid = True - - if valid: - self.__file_txt.SetBackgroundColour(wx.Colour(235, 255, 235)) - else: - self.__file_txt.SetBackgroundColour(wx.Colour(255, 220, 220)) - - self.__ok_btn.Enable(valid) - - if valid: - self.__filename = filepath - else: - self.__filename = None diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/shortcut_help.py b/helios/pipeViewer/pipe_view/gui/dialogs/shortcut_help.py deleted file mode 100644 index d6fecf691a..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/shortcut_help.py +++ /dev/null @@ -1,144 +0,0 @@ -from typing import Dict, Tuple, TypedDict, Union -import wx -import wx.html - - -class ShortcutHelpEntry(TypedDict, total=False): - mod: str - keys: Union[Tuple[str, ...], str] - desc: str - - -ShortcutHelpDict = Dict[str, Tuple[ShortcutHelpEntry, ...]] - - -def _is_mac_os() -> bool: - os = wx.GetOsVersion()[0] - return os in [wx.OS_MAC_OS, wx.OS_MAC_OSX_DARWIN, wx.OS_MAC] - - -def _gen_message(shortcut_items: ShortcutHelpDict) -> str: - msg = '\n\n

Argos Keyboard Shortcuts

\n' - - for k, v in shortcut_items.items(): - msg += f'

{k}

\n
    \n' - for item in v: - keys = item['keys'] - desc = item['desc'] - mod = item.get('mod') - - if mod is not None: - if isinstance(keys, tuple): - keys = tuple(f'{mod}+{k}' for k in keys) - else: - keys = f'{mod}+{keys}' - - if isinstance(keys, tuple): - keys = ' or '.join(keys) - - msg += f'
  • {keys}: {desc}
    • \n' - msg += '
\n' - - msg += '' - return msg - - -class ShortcutHelp(wx.Frame): - __SHIFT_KEY = 'Shift' - __CTRL_KEY = 'Command' if _is_mac_os() else 'CTRL' - - __SHORTCUT_ITEMS: ShortcutHelpDict = { - 'Global': ( - {'mod': __CTRL_KEY, - 'keys': ('-', 'Mousewheel Down'), - 'desc': 'Zoom out'}, - {'mod': __CTRL_KEY, - 'keys': ('=', 'Mousewheel Up'), - 'desc': 'Zoom in'}, - {'mod': __CTRL_KEY, - 'keys': '0', - 'desc': 'Reset zoom'}, - {'keys': 'Space', - 'desc': 'Step forward by 1 tick'}, - {'keys': 'G', - 'desc': 'Set focus to Jump box'}, - {'mod': __CTRL_KEY, - 'keys': 'G', - 'desc': 'Toggle background grid'}, - {'keys': 'N', - 'desc': 'Jump to next value change for current element'}, - {'mod': __SHIFT_KEY, - 'keys': 'N', - 'desc': 'Jump to previous value change for current element'} - ), - 'View Mode': ( - {'keys': ('Up Arrow', 'Right Arrow'), - 'desc': 'Step forward by 1 cycle'}, - {'keys': ('Down Arrow', 'Left Arrow'), - 'desc': 'Step backward by 1 cycle'}, - {'keys': 'Page Up', - 'desc': 'Step forward by 10 cycles'}, - {'keys': 'Page Down', - 'desc': 'Step backward by 10 cycles'}, - ), - 'Edit Mode': ( - {'keys': 'Up Arrow', - 'desc': 'Move element up'}, - {'keys': 'Down Arrow', - 'desc': 'Move element down'}, - {'keys': 'Left Arrow', - 'desc': 'Move element left'}, - {'keys': 'Right Arrow', - 'desc': 'Move element right'}, - {'mod': __SHIFT_KEY, - 'keys': 'Arrow Key', - 'desc': 'Move element slower'}, - {'mod': __CTRL_KEY, - 'keys': 'Arrow Key', - 'desc': 'Move element faster'}, - {'mod': __SHIFT_KEY, - 'keys': 'G', - 'desc': 'Snap current element to the grid'} - ) - } - - __MESSAGE = _gen_message(__SHORTCUT_ITEMS) - - def __init__(self, parent: wx.Window, id: int) -> None: - wx.Frame.__init__( - self, - parent, - id, - "Argos Shortcut Information", - style=(wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.CLIP_CHILDREN) - ) - - self.__message_ctrl = wx.html.HtmlWindow( - self, - style=wx.html.HW_SCROLLBAR_AUTO, - size=(400, 525) - ) - self.__message_ctrl.SetPage(ShortcutHelp.__MESSAGE) - self.__close_btn = wx.Button(self, label="Close") - self.__sizer = wx.BoxSizer(wx.VERTICAL) - - self.Bind(wx.EVT_BUTTON, self.OnCloseButton, self.__close_btn) - - self.__sizer.Add(self.__message_ctrl, - proportion=1, - flag=wx.ALL | wx.EXPAND, - border=5) - self.__sizer.Add(self.__close_btn, - proportion=0, - flag=wx.ALL | wx.ALIGN_CENTER, - border=5) - self.SetSizer(self.__sizer) - - self.Fit() - self.Show(True) - - def OnCloseButton(self, evt: wx.CommandEvent) -> None: - self.Destroy() diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/translate_elements_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/translate_elements_dlg.py deleted file mode 100644 index f635d9880f..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/translate_elements_dlg.py +++ /dev/null @@ -1,296 +0,0 @@ -from __future__ import annotations -import wx -from ..font_utils import GetMonospaceFont -from typing import Optional, Tuple, Union, TYPE_CHECKING - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -# TranslateElementsDlg allows relative or absolute translation of a group of -# elements based on user-specified coordinates. This is exepected to be used -# modally -class TranslateElementsDlg(wx.Dialog): - - def __init__(self, parent: Layout_Frame) -> None: - self.__layout_frame = parent - self.__context = parent.GetContext() - self.__sel_mgr = parent.GetCanvas().GetSelectionManager() - - # initialize graphical part - wx.Dialog.__init__( - self, - parent, - -1, - f'Translate Elements - {self.__layout_frame.ComputeTitle()}', - size=(-1, -1), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU) - ) - - self.__fnt_numbers = GetMonospaceFont(12) - - panel = wx.Panel(self) - - lbl_info = wx.StaticText(panel, - -1, - '(Press ENTER to appy changes, ESC to exit)') - lbl_info.SetFont(self.__fnt_numbers) - lbl_info.SetForegroundColour((120, 120, 120)) - - lbl_heading = wx.StaticText( - panel, - -1, - f'{len(self.__sel_mgr.GetSelection())} selected elements' - ) - - box_dims = (60, 40) - panel_box = wx.Panel(panel, size=box_dims, style=wx.BORDER_SIMPLE) - panel_box.SetMinSize(box_dims) - self.__lbl_sel_lt = wx.StaticText(panel, -1, '') - self.__lbl_sel_lt.SetFont(self.__fnt_numbers) - self.__lbl_sel_wh = wx.StaticText(panel_box, -1, '') # Inside box - self.__lbl_sel_wh.SetFont(self.__fnt_numbers) - self.__lbl_sel_rb = wx.StaticText(panel, -1, '') - self.__lbl_sel_rb.SetFont(self.__fnt_numbers) - - self.__UpdateDesc() - - lbl_x = wx.StaticText(panel, -1, "X: ", size=(20, 20)) - self.__txt_x = wx.TextCtrl(panel, -1, "0", size=(70, 20)) - self.__chk_rel_x = wx.CheckBox(panel, -1, label='relative') - self.__chk_rel_x.SetValue(True) - - lbl_y = wx.StaticText(panel, -1, "Y: ", size=(20, 20)) - self.__txt_y = wx.TextCtrl(panel, -1, "0", size=(70, 20)) - self.__chk_rel_y = wx.CheckBox(panel, -1, label='relative') - self.__chk_rel_y.SetValue(True) - - self.__btn_exit = wx.Button(panel, -1, 'Close') - self.__btn_apply = wx.Button(panel, -1, 'Apply') - - panel_box_sizer = wx.BoxSizer(wx.VERTICAL) - panel_box_sizer.Add((1, 1), 1, wx.EXPAND) - panel_box_sizer.Add( - self.__lbl_sel_wh, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.ALIGN_CENTER_HORIZONTAL - ) - panel_box_sizer.Add((1, 1), 1, wx.EXPAND) - panel_box.SetSizer(panel_box_sizer) - - position_sizer = wx.BoxSizer(wx.HORIZONTAL) - position_sizer.Add(self.__lbl_sel_lt, - 0, - wx.ALIGN_TOP | wx.ALIGN_RIGHT | wx.RIGHT, - 4) - position_sizer.Add(panel_box, 1, wx.EXPAND | wx.TOP | wx.BOTTOM, 10) - position_sizer.Add(self.__lbl_sel_rb, - 0, - wx.ALIGN_BOTTOM | wx.ALIGN_RIGHT | wx.LEFT, - 4) - - sizer_x = wx.BoxSizer(wx.HORIZONTAL) - sizer_x.Add(lbl_x, 0, wx.ALL, 4) - sizer_x.Add(self.__txt_x, 0, wx.ALL, 4) - sizer_x.Add(self.__chk_rel_x, 0, wx.ALL, 4) - - sizer_y = wx.BoxSizer(wx.HORIZONTAL) - sizer_y.Add(lbl_y, 0, wx.ALL, 4) - sizer_y.Add(self.__txt_y, 0, wx.ALL, 4) - sizer_y.Add(self.__chk_rel_y, 0, wx.ALL, 4) - - self.__sizer_buttons = wx.BoxSizer(wx.HORIZONTAL) - self.__sizer_buttons.Add(self.__btn_exit, 0, 0, 4) - self.__sizer_buttons.Add((1, 1), 1, wx.EXPAND) - self.__sizer_buttons.Add(self.__btn_apply, 0, 0, 4) - - main_sizer = wx.BoxSizer(wx.VERTICAL) - main_sizer.Add(lbl_heading, 0, wx.ALIGN_CENTER_HORIZONTAL) - # main_sizer.Add((1,6), 0) - main_sizer.Add(position_sizer, 0, wx.EXPAND | wx.ALL, 10) # Indent - main_sizer.Add(wx.StaticLine(panel), 0, wx.EXPAND) - main_sizer.Add((1, 4), 0) - main_sizer.Add(sizer_x, 0, wx.ALIGN_LEFT) - main_sizer.Add(sizer_y, 0, wx.ALIGN_LEFT) - main_sizer.Add((1, 4), 0, wx.EXPAND) - main_sizer.Add(wx.StaticLine(panel), 0, wx.EXPAND) - main_sizer.Add((1, 4), 0) - main_sizer.Add(lbl_info, 0, wx.ALIGN_CENTER) - main_sizer.Add((1, 18), 0) - main_sizer.Add(self.__sizer_buttons, 0, wx.EXPAND) - - panel.SetSizer(main_sizer) - - padding_sizer = wx.BoxSizer(wx.VERTICAL) - padding_sizer.Add(panel, 1, wx.EXPAND | wx.ALL, 10) - self.SetSizer(padding_sizer) - - self.Fit() - - self.__txt_x.SetFocus() - - # bind to events - self.__btn_apply.Bind(wx.EVT_BUTTON, self.OnApply) - self.__btn_exit.Bind(wx.EVT_BUTTON, self.OnExit) - panel.Bind(wx.EVT_KEY_UP, self.OnKeyPress) - self.__txt_x.Bind(wx.EVT_TEXT, self.OnCoordChange) - self.__txt_y.Bind(wx.EVT_TEXT, self.OnCoordChange) - self.__chk_rel_x.Bind(wx.EVT_CHECKBOX, self.OnCoordChange) - self.__chk_rel_y.Bind(wx.EVT_CHECKBOX, self.OnCoordChange) - - def Show(self, show: bool = True) -> bool: - raise Exception('This dialog should only be shown modally') - - # defines how the dialog should pop up - def ShowModal(self) -> int: - res = wx.Dialog.ShowModal(self) - self.Raise() - return res - - # callback that listens for enter being pressed to initiate search - def OnKeyPress(self, evt: wx.KeyEvent) -> None: - ctrl = evt.ControlDown() - shift = evt.ShiftDown() - alt = evt.AltDown() - - if evt.GetKeyCode() == wx.WXK_ESCAPE: - self.EndModal(0) - self.__layout_frame.SetFocus() - elif evt.GetKeyCode() == wx.WXK_RETURN: - self.OnApply() - elif (evt.GetKeyCode() == ord('Z')) and ctrl and not shift and not alt: - # Catch undo hotkeys for this control - self.__sel_mgr.Undo() - elif (evt.GetKeyCode() == ord('Y')) and ctrl and not shift and not alt: - # Catch redo hotkeys for this control - self.__sel_mgr.Redo() - else: - evt.Skip() - - # In case undo/redo made changes: - self.__UpdateDesc() - - # Handle x/y textbox/checkbox change - def OnCoordChange( - self, - evt: Optional[Union[wx.KeyEvent, wx.CommandEvent]] = None - ) -> None: - dx, dy = self.__ComputeMoveDeltas() - - WHITE = (255, 255, 255) - PINK = (255, 100, 100) - self.__txt_x.SetBackgroundColour(PINK if dx is None else WHITE) - self.__txt_y.SetBackgroundColour(PINK if dy is None else WHITE) - - valid_deltas = dx is not None and dy is not None - self.__btn_apply.Enable(valid_deltas) - - is_enter = (evt and - isinstance(evt, wx.KeyEvent) and - evt.GetKeyCode() == wx.WXK_ENTER) - # Handle enter key for applying - if is_enter and valid_deltas: - self.OnApply() - - if evt is not None: - evt.Skip() - - def OnExit(self, evt: wx.CommandEvent) -> None: - self.EndModal(0) - - # Move elements as described - def OnApply(self, evt: Optional[wx.CommandEvent] = None) -> None: - # Move elements to absolute or relative position. Always use the delta - # argument to Selection_Mgr.Move() because it may compute its absolute - # position differently than this dialog (i.e. center of selection - # rather than top-left corner) - - dx, dy = self.__ComputeMoveDeltas() - - # Apply should be ignored for invalid textbox content. Apply button - # will be disabled and textboxes will be red - if dx is None or dy is None: - return - - def rel_str(val: int) -> str: - return 'rel' if self.__chk_rel_x.GetValue() else 'abs' - - self.__sel_mgr.BeginCheckpoint( - f'Translate {dx}({rel_str(dx)}), {dy}({rel_str(dy)})' - ) - - try: - self.__sel_mgr.Move(delta=(dx, dy), force_no_snap=True) - finally: - self.__sel_mgr.CommitCheckpoint() - - self.__layout_frame.Refresh() - - self.__UpdateDesc() # Update label with new coordinates - - # Compute Move deltas based on current textbox/checkbox values - # @return (dx,dy) indicating arguments to Selection_Mgr.Move. Either value - # in this tuple may be None if value data cannot be extracted from the - # dialog widgets - def __ComputeMoveDeltas(self) -> Union[Tuple[int, int], Tuple[None, None]]: - try: - xstr = self.__txt_x.GetValue() - # For relative moves, value can be empty to mean no move - if self.__chk_rel_x.GetValue() and xstr == '': - x = 0 - else: - x = int(xstr) # For non-relative motion - except Exception: - x = None - - try: - # For relative moves, value can be empty to mean no move - ystr = self.__txt_y.GetValue() - if self.__chk_rel_y.GetValue() and ystr == '': - y = 0 - else: - y = int(ystr) - except Exception: - y = None - - if x is None or y is None: - return (None, None) - - bbox = self.__sel_mgr.GetBoundingBox() # l,t,r,b - if bbox is None: - return (None, None) - - l, t, r, b = bbox - - # Compute dx based on relative or absolute movement - if self.__chk_rel_x.GetValue(): - dx = x - else: - dx = x - l - - if self.__chk_rel_y.GetValue(): - dy = y - else: - dy = y - t - - return (dx, dy) - - def __UpdateDesc(self) -> None: - bbox = self.__sel_mgr.GetBoundingBox() # l,t,r,b - if bbox is not None: - l, t, r, b = bbox - bbox_lt = f'x:{l:>6}\ny:{t:>6}' - bbox_wh = f'w:{r - l:>6}\nh:{b - t:>6}' - bbox_rb = f'r:{r:>6}\nb:{b:>6}' - else: - bbox_lt = 'N/A' - bbox_wh = 'N/A' - bbox_rb = 'N/A' - - self.__lbl_sel_lt.SetLabel(bbox_lt) - self.__lbl_sel_wh.SetLabel(bbox_wh) - self.__lbl_sel_rb.SetLabel(bbox_rb) diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/view_settings_dlg.py b/helios/pipeViewer/pipe_view/gui/dialogs/view_settings_dlg.py deleted file mode 100644 index cef805020e..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/view_settings_dlg.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations -from collections import OrderedDict -from typing import Dict, Optional, Set, TypedDict, TYPE_CHECKING -import wx - -from ...model.settings import ArgosSettings -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - - -class ViewSettingsDialog(wx.Dialog): - class ViewSettingType(TypedDict): - label: str - type: str - ctrl: Optional[wx.SpinCtrl] - - __SETTINGS: OrderedDict[str, ViewSettingType] = OrderedDict(( - ( - 'layout_font_size', - { - 'label': 'Layout Font Size', - 'type': 'spin', - 'ctrl': None - } - ), - ( - 'hover_font_size', - { - 'label': 'Hover Font Size', - 'type': 'spin', - 'ctrl': None - } - ), - ( - 'playback_font_size', - { - 'label': 'Playback Bar Font Size', - 'type': 'spin', - 'ctrl': None - } - ) - )) - - def __init__(self, parent: Layout_Frame, settings: ArgosSettings): - wx.Dialog.__init__(self, parent, wx.NewId(), 'View Settings') - - self.__settings_controls = ViewSettingsDialog.__SETTINGS.copy() - - self.__changed_settings: Set[str] = set() - self.__ctrl_map = {} - sizer = wx.BoxSizer(wx.VERTICAL) - setting_sizer = wx.FlexGridSizer(cols=2, gap=(10, 0)) - for k, v in self.__settings_controls.items(): - setting_sizer.Add(wx.StaticText(self, - wx.NewId(), - v['label']), - 0, - wx.ALIGN_CENTER_VERTICAL) - new_ctrl = None - if v['type'] == 'spin': - new_ctrl = wx.SpinCtrl(self, - id=wx.NewId(), - value=str(settings[k])) - new_ctrl.SetRange(1, 999) - new_ctrl.SetMinSize( - new_ctrl.GetSizeFromTextSize(new_ctrl.GetTextExtent('000')) - ) - assert new_ctrl is not None - self.__ctrl_map[new_ctrl.GetId()] = k - setting_sizer.Add(new_ctrl, 0, wx.ALIGN_CENTER_VERTICAL) - self.Bind(wx.EVT_SPINCTRL, self.OnSpinCtrl, new_ctrl) - v['ctrl'] = new_ctrl - - sizer.Add(setting_sizer, 0, wx.ALIGN_CENTER_HORIZONTAL | wx.ALL, 10) - sizer.Add(self.CreateButtonSizer(wx.OK | wx.CANCEL), - 0, - wx.ALIGN_CENTER_HORIZONTAL | wx.ALL, - 10) - sizer.SetSizeHints(self) - self.SetSizer(sizer) - - def OnSpinCtrl(self, evt: wx.SpinEvent) -> None: - self.__changed_settings.add(self.__ctrl_map[evt.GetId()]) - - def GetSettings(self) -> Dict[str, int]: - def get_value(ctrl: Optional[wx.SpinCtrl]) -> int: - assert ctrl is not None - return ctrl.GetValue() - return { - k: get_value(self.__settings_controls[k]['ctrl']) - for k in self.__changed_settings - } diff --git a/helios/pipeViewer/pipe_view/gui/dialogs/watchlist_dialog.py b/helios/pipeViewer/pipe_view/gui/dialogs/watchlist_dialog.py deleted file mode 100644 index 6bb483b068..0000000000 --- a/helios/pipeViewer/pipe_view/gui/dialogs/watchlist_dialog.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations -import wx -from ..widgets.transaction_list import (TransactionList, - TransactionListBaseEntry) -from functools import partial -from typing import List, Tuple, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - -ID_WATCH_DELETE = wx.NewId() - - -# This class persistently displays a list of transactions chosen by the user -class WatchListDlg(wx.Frame): - __BASE_PROPERTIES = ('start', 'loc', 'annotation', 't_offset', 'period') - __NON_QUERIED_PROPERTIES = ('t_offset', 'period') - - def __init__(self, parent: Layout_Frame) -> None: - self.__layout_frame = parent - self.__context = parent.GetContext() - # holds a list of list indices that need updating every cycle - self.__relative_indices: List[int] = [] - - # create GUI - wx.Frame.__init__( - self, - parent, - -1, - 'Watch List', - size=(500, 800), - style=(wx.MAXIMIZE_BOX | - wx.RESIZE_BORDER | - wx.CAPTION | - wx.CLOSE_BOX | - wx.SYSTEM_MENU) - ) - self.__list = TransactionList(self, - parent.GetCanvas(), - name='listbox') - self.__list.SetProperties(self.__BASE_PROPERTIES) - - self.__list.Bind(wx.EVT_LIST_ITEM_RIGHT_CLICK, self.__OnItemMenu) - - # Hide instead of closing - self.Bind(wx.EVT_CLOSE, lambda evt: self.Hide()) - - def __OnItemMenu(self, evt: wx.ListEvent) -> None: - menu = wx.Menu() - menu.Append(ID_WATCH_DELETE, "Delete") - # calculate current mouse position in relative coordinates - mouse_pos = self.ScreenToClient(wx.GetMousePosition()) - - self.Bind(wx.EVT_MENU, - partial(self.__OnDeleteItem, index=evt.GetIndex())) - self.PopupMenu(menu, mouse_pos) - - def __OnDeleteItem(self, evt: wx.ListEvent, index: int) -> None: - self.__list.Remove(index) - index_of_removal = len(self.__relative_indices) - if index in self.__relative_indices: - index_of_removal = self.__relative_indices.index(index) - self.__relative_indices.remove(index) - - for i in range(len(self.__relative_indices)): - if i >= index_of_removal: # shift items after index - self.__relative_indices[i] -= 1 - - def SetFields(self, fields: Tuple[str, ...]) -> None: - fields_new = list(fields) - fields_new.extend(self.__BASE_PROPERTIES) - self.__list.SetProperties(tuple(fields_new)) - - def Add(self, loc: str, t_offset: int, relative: bool = False) -> None: - q_fields = list(self.__list.GetProperties()) - period = self.__context.GetLocationPeriod(loc) - start = t_offset*period + self.__context.hc - for p in self.__NON_QUERIED_PROPERTIES: - q_fields.remove(p) # remove since this is not a valid db query - info = self.__context.GetTransactionFields(start, - loc, - q_fields) - info['t_offset'] = t_offset - info['period'] = period - idx = self.__list.Add(info) - if relative: - self.__relative_indices.append(idx) - - def Show(self, show: bool = True) -> bool: - res = wx.Frame.Show(self, show) - self.Raise() - return res - - # called every hc change. Updates relative entries - def TickUpdate(self, hc: int) -> None: - for relative_index in self.__relative_indices: - transaction = cast(TransactionListBaseEntry, - self.__list.GetTransaction(relative_index)) - q_fields = list(self.__list.GetProperties()) - for p in self.__NON_QUERIED_PROPERTIES: - q_fields.remove(p) - t_offset = transaction['t_offset'] - period = transaction['period'] - updated = self.__context.GetTransactionFields(hc + t_offset*period, - transaction['loc'], - q_fields) - # copy over - for key in updated.keys(): - transaction[key] = updated[key] - - self.__list.RefreshTransaction(relative_index) diff --git a/helios/pipeViewer/pipe_view/gui/font_utils.py b/helios/pipeViewer/pipe_view/gui/font_utils.py deleted file mode 100644 index 07d67966cf..0000000000 --- a/helios/pipeViewer/pipe_view/gui/font_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations -import os -from typing import Optional -import wx - -__DPI: Optional[float] = None - - -def _GetDPI() -> float: - global __DPI - if __DPI is None: - # Use Tk to get display DPI on Wayland platforms since a disconnected - # primary display may still show up as present to wx - if os.environ.get('WAYLAND_DISPLAY') is not None: - from tkinter import Tk - root = Tk() - root.withdraw() - __DPI = root.winfo_fpixels('1i') - else: - __DPI = wx.GetDisplayPPI()[1] - return __DPI - - -def ScaleFont(font_size: int) -> int: - DEFAULT_DPI = 72 - dpi = _GetDPI() - assert dpi > 0 - # Rounding up seems to give better results - return int(round((font_size * DEFAULT_DPI) / dpi)) - - -def GetMonospaceFont(size: int) -> wx.Font: - face_name = 'Monospace' - if not wx.FontEnumerator.IsValidFacename(face_name): - # Pick a fallback generic modern font (not by name) - face_name = '' - return wx.Font(ScaleFont(size), - wx.FONTFAMILY_MODERN, - wx.NORMAL, - wx.NORMAL, - faceName=face_name) - - -def GetDefaultFontSize() -> int: - return 11 - - -def GetDefaultControlFontSize() -> int: - return 12 - - -_DEFAULT_FONT = None - - -def GetDefaultFont() -> wx.Font: - global _DEFAULT_FONT - if _DEFAULT_FONT is None: - _DEFAULT_FONT = GetMonospaceFont(GetDefaultFontSize()) - return _DEFAULT_FONT diff --git a/helios/pipeViewer/pipe_view/gui/hover_preview.py b/helios/pipeViewer/pipe_view/gui/hover_preview.py deleted file mode 100644 index 9244e58a1d..0000000000 --- a/helios/pipeViewer/pipe_view/gui/hover_preview.py +++ /dev/null @@ -1,654 +0,0 @@ -from __future__ import annotations -from logging import warn, error -import textwrap - -import wx -import wx.lib.newevent - -from .dialogs.watchlist_dialog import WatchListDlg -from .font_utils import GetMonospaceFont - -from typing import Any, List, Optional, Tuple, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from ..model.element import Element - from ..model.element_value import Element_Value - from ..model.layout_context import Layout_Context - from .argos_menu import Argos_Menu - from .layout_canvas import Layout_Canvas - -# @brief This new event triggers the canvas to just redraw the mouse-over text. -# No update of the underlying view is executed on this event. -HoverRedrawEvent, EVT_HOVER_REDRAW = wx.lib.newevent.NewEvent() - -ID_HOVER_POPUP_COPY = wx.NewId() -ID_HOVER_POPUP_WATCH_RELATIVE = wx.NewId() -ID_HOVER_POPUP_WATCH_ABSOLUTE = wx.NewId() -ID_HOVER_POPUP_SEARCH = wx.NewId() -ID_PREV_ANNO = wx.NewId() -ID_NEXT_ANNO = wx.NewId() -ID_HIGHLIGHT_UOP_TOGGLE = wx.NewId() - - -class HoverPreview: - ''' - @brief Stores information needed for mouse-over preview - ''' - - LINE_LENGTH = 50 - - # The hover will be rendered in a separate window so we don't have to deal - # with manually repainting the layout canvas - class HoverPreviewWindow(wx.PopupWindow): - def __init__(self, - canvas: Layout_Canvas, - handler: HoverPreview) -> None: - super().__init__(canvas.GetParent()) - self.Show(False) - self.__canvas = canvas - self.__handler = handler - self.Bind(wx.EVT_ENTER_WINDOW, self.OnMouse) - sizer = wx.BoxSizer(wx.HORIZONTAL) - panel_sizer = wx.BoxSizer(wx.HORIZONTAL) - self.__panel = wx.Panel(self) - self.__panel.SetFont( - GetMonospaceFont(self.__canvas.GetSettings().hover_font_size) - ) - self.__panel.SetForegroundColour(wx.BLACK) - self.__text_ctrl = wx.StaticText(self.__panel) - panel_sizer.Add(self.__text_ctrl) - self.__panel.SetSizer(panel_sizer) - sizer.Add( - self.__panel, - flag=wx.TOP | wx.BOTTOM | wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER, - border=1 - ) - self.SetSizer(sizer) - - def UpdateInfo(self, - element: Element, - annotation: str, - text: str, - position: wx.Point) -> None: - BORDER_LIGHTNESS = 70 - - _, brush = self.__canvas.UpdateTransactionColor(element, - annotation) - color = brush.GetColour() - self.__panel.SetBackgroundColour(color) - self.SetBackgroundColour(color.ChangeLightness(BORDER_LIGHTNESS)) - self.__text_ctrl.SetLabel(text) - width, height = self.GetBestSize() - visible_area = self.__canvas.GetVisibleArea() - x, y = position - box_right = x + width - box_bottom = y + height - if box_right > visible_area[2]: # off the right edge - # shift left - x -= (box_right - visible_area[2]) - if box_bottom > visible_area[3]: - y -= (box_bottom - visible_area[3]) - - if x < 0 or y < 0: - return # screen too small - - x, y = self.__canvas.ClientToScreen((x, y)) - old_rect = self.GetRect() - self.SetRect((x, y, width, height)) - self.__UpdateCanvas(old_rect) - - def __UpdateCanvas(self, old_rect: wx.Rect) -> None: - old_rect.SetPosition( - self.__canvas.CalcUnscrolledPosition(old_rect.GetTopLeft()) - ) - old_rect.Inflate(10, 10) - self.__canvas.RefreshRect(old_rect) - - def AcceptsFocus(self) -> bool: - return False - - # Handle the corner case where the user manages to mouse over the - # window - def OnMouse(self, event: wx.MouseEvent) -> None: - self.__handler.DestroyWindow() - - def Destroy(self) -> None: - old_rect = self.GetRect() - self.__UpdateCanvas(old_rect) - self.Disable() - self.Show(False) - self.DestroyLater() - - def __init__(self, canvas: Layout_Canvas, context: Layout_Context) -> None: - self.__canvas = canvas - self.__context = context - self.__value = "" - self.annotation = "" - self.element: Optional[Element] = None - self.__enabled = False - self.show = False - self.position = wx.Point(0, 0) - self.__last_move_tick: Optional[int] = None - self.__window: Optional[HoverPreview.HoverPreviewWindow] = None - - self.__fields = ['annotation'] - - def Enable(self, is_enable: bool) -> None: - ''' - Setter for enable bool - ''' - self.__enabled = is_enable - - # Getter for enable bool - def IsEnabled(self) -> bool: - return self.__enabled - - def SetFields(self, fields: List[str]) -> None: - self.__fields = fields - - def GetFields(self) -> List[str]: - return self.__fields - - def GetElement(self) -> Optional[Element]: - ''' - Gets the element currently referenced by this hover preview - @return None if there is none - ''' - return self.element - - def IsDifferent(self, element: Optional[Element]) -> bool: - ''' - Determine if a given element is different enough from the current - element (self.element) to warrant updating the tooltip. - ''' - - if self.element is None or element is None: - return True - - def prop_num(prop_name: str) -> int: - assert self.element is not None - assert element is not None - return int(self.element.HasProperty(prop_name)) + \ - int(element.HasProperty(prop_name)) - - def prop_diff(prop_name: str) -> bool: - assert self.element is not None - assert element is not None - return self.element.GetProperty(prop_name) != \ - element.GetProperty(prop_name) - - if prop_diff('t_offset'): - return True - else: - num_have_locstr = prop_num('LocationString') - if num_have_locstr == 1 or \ - (num_have_locstr == 2 and prop_diff('LocationString')): - return True - - else: - num_have_toolip = prop_num('tooltip') - if num_have_toolip == 1 or \ - (num_have_toolip == 2 and prop_diff('tooltip')): - return True - else: - return False - - def SetValue(self, string: str) -> None: - ''' - Set what is supposed to be drawn. - ''' - self.__value = textwrap.fill(string, - self.LINE_LENGTH, - replace_whitespace=False) - - def GetText(self) -> str: - ''' - Get what is suppsed to be drawn. - ''' - return self.__value - - def GetWindow(self) -> HoverPreview.HoverPreviewWindow: - if self.__window is None: - self.__window = HoverPreview.HoverPreviewWindow(self.__canvas, - self) - return self.__window - - def HandleMenuClick(self, position: wx.Point) -> None: - ''' - if applicable, pops up menu to copy and paste from hover - ''' - if self.show: - menu = wx.Menu() - copy = menu.Append(ID_HOVER_POPUP_COPY, "Copy Text") - watch_rel = menu.Append(ID_HOVER_POPUP_WATCH_RELATIVE, - "Add Relative Position to Watch List") - watch_abs = menu.Append(ID_HOVER_POPUP_WATCH_ABSOLUTE, - "Add Absolute Position to Watch List") - menu.AppendSeparator() - search_loc = menu.Append(ID_HOVER_POPUP_SEARCH, - "Search from this Location") - menu.AppendSeparator() - next_anno = menu.Append(ID_NEXT_ANNO, - "Next change in annotation\tN") - prev_anno = menu.Append(ID_PREV_ANNO, - "Previous change in annotation\tShift+N") - if self.__context.IsUopHighlighted(self.__value): - self.__is_highlighted = True - highlight_uop_toggle = menu.Append(ID_HIGHLIGHT_UOP_TOGGLE, - "Unhighlight Uop") - else: - self.__is_highlighted = False - highlight_uop_toggle = menu.Append(ID_HIGHLIGHT_UOP_TOGGLE, - "Highlight Uop") - self.__canvas.Bind(wx.EVT_MENU, self.__OnCopyText, copy) - self.__canvas.Bind(wx.EVT_MENU, - self.__OnAddWatchRelative, - watch_rel) - self.__canvas.Bind(wx.EVT_MENU, - self.__OnAddWatchAbsolute, - watch_abs) - if self.element is not None and \ - self.element.HasProperty('LocationString') and \ - cast(str, self.element.GetProperty('LocationString')) != '': - self.__canvas.Bind(wx.EVT_MENU, - self.__OnSearchLocation, - search_loc) - self.__canvas.Bind(wx.EVT_MENU, self.__OnNextAnno, next_anno) - self.__canvas.Bind(wx.EVT_MENU, self.__OnPrevAnno, prev_anno) - self.__canvas.Bind(wx.EVT_MENU, - self.__HighlightUopToggle, - highlight_uop_toggle) - else: - search_loc.Enable(False) - highlight_uop_toggle.Enable(False) - self.__canvas.PopupMenu(menu, position) - - def GotoNextChange(self) -> None: - ''' - Goto the next annotation change - ''' - pair = self.__canvas.GetSelectionManager().GetPlaybackSelected() - if pair: - - self.__GotoNextAnno(pair.GetElement()) - - def GotoPrevChange(self) -> None: - ''' - Goto the previous annotation change - ''' - pair = self.__canvas.GetSelectionManager().GetPlaybackSelected() - if pair is None: - return - el = pair.GetElement() - - self.__GotoPrevAnno(el) - - # @profile - def __HighlightUopToggle(self, event: wx.MenuEvent) -> None: - ''' - Handle click on "Highlight Uop" - ''' - if self.__is_highlighted: - self.__context.UnhighlightUop(self.__value) - self.__is_highlighted = False - else: - self.__context.HighlightUop(self.__value) - self.__canvas.UpdateTransactionHighlighting() - self.__context.RedrawHighlightedElements() - self.__canvas.FullUpdate() - - def __OnCopyText(self, event: wx.MenuEvent) -> None: - ''' - Called when copy text entry in popup menu is pressed. - ''' - clipboard_data = wx.TextDataObject() - clipboard_data.SetText(self.__value) - if not wx.TheClipboard.IsOpened(): - wx.TheClipboard.Open() - wx.TheClipboard.SetData(clipboard_data) - wx.TheClipboard.Close() - else: - warn("Cannot copy. Clipboard already open.") - - def __OnAddWatchRelative(self, evt: wx.MenuEvent) -> None: - # frame could be None. Probably not in this case though. - # going to let it throw an exception if it is None - self.__AddWatch(relative=True) - - def __OnAddWatchAbsolute(self, evt: wx.MenuEvent) -> None: - self.__AddWatch(relative=False) - - # Handle click on "Search From this Location" - def __OnSearchLocation(self, evt: wx.MenuEvent) -> None: - assert self.element is not None - self.__canvas.GetFrame().ShowSearch( - location=self.element.GetProperty('LocationString') - ) - - # Handle click on the "Next Change in Annotation" - def __OnNextAnno(self, evt: Optional[wx.MenuEvent] = None) -> None: - if self.element is None: - return - - self.__GotoNextAnno(self.element) - - def __GotoNextAnno(self, el: Element) -> None: - if not el.HasProperty('LocationString'): - return - - location_str = cast(str, el.GetProperty('LocationString')) - loc_mgr = self.__context.dbhandle.database.location_manager - loc_results = loc_mgr.getLocationInfoNoVars(location_str) - if loc_results == loc_mgr.LOC_NOT_FOUND: - # @todo Prevent this command from being called without a valid - # location - return - location_id, _, clock = loc_results - - # @todo Current tick should be based on this element's t_offset. - cur_tick = self.__context.GetHC() - - fields = self.__context.GetTransactionFields(cur_tick, - location_str, - ['annotation']) - cur_annotation = fields.get('annotation') - if cur_annotation is None: - cur_annotation = '' - - def progress_cb(percent: int, - num_results: int, - info: str) -> Tuple[bool, bool]: - cont = True - skip = False - if num_results > 0: - cont = False # Found some results. No need to keep searching - return (cont, skip) - - wx.BeginBusyCursor() - try: - # @todo Include location ID in searches - results = self.__context.searchhandle.Search( - 'string', - cur_annotation, - cur_tick + 1, # start tick - -1, # end tick - [location_id], - progress_cb, - invert=True - ) - except Exception as ex: - error('Error searching: %s', ex) - return # Failed to search - finally: - wx.EndBusyCursor() - - closest = None - for start, end, loc, annotation in results: - if loc == location_id: - if annotation != cur_annotation: - if closest is None or \ - (start > cur_tick and start < closest): - closest = start - - if closest is not None: - self.__context.GoToHC(closest) - - if wx.IsBusy(): - wx.EndBusyCursor() - - # Handle click on the "Previous Change in Annotation" - def __OnPrevAnno(self, evt: Optional[wx.MenuEvent] = None) -> None: - if self.element is None: - return - - self.__GotoPrevAnno(self.element) - - def __GotoPrevAnno(self, el: Element) -> None: - if not el.HasProperty('LocationString'): - return - - location_str = cast(str, el.GetProperty('LocationString')) - loc_mgr = self.__context.dbhandle.database.location_manager - loc_results = loc_mgr.getLocationInfoNoVars(location_str) - if loc_results == loc_mgr.LOC_NOT_FOUND: - # \todo Prevent this command from being called without a valid - # location - return - location_id, _, clock = loc_results - - # @todo Current tick should be based on this element's t_offset. - cur_tick = self.__context.GetHC() - - fields = self.__context.GetTransactionFields(cur_tick, - location_str, - ['annotation']) - cur_annotation = fields.get('annotation') - if cur_annotation is None: - cur_annotation = '' - - def progress_cb(percent: int, *args: Any) -> Tuple[bool, bool]: - cont = True - skip = False - return (cont, skip) - - wx.BeginBusyCursor() - try: - results = self.__context.searchhandle.Search( - 'string', - cur_annotation, - 0, # start tick - cur_tick - 1, # end tick - [location_id], - progress_cb, - invert=True - ) - # TODO Support some kind of reverse search that doesn't require - # starting from the beginning - except Exception as ex: - error('Error searching: %s', ex) - return # Failed to search - finally: - wx.EndBusyCursor() - - closest = None - for start, end, loc, annotation in results: - if loc == location_id: - if annotation != cur_annotation: - # <= cur_tick because end is exclusive - if closest is None or (end <= cur_tick and end > closest): - closest = end - - if closest is not None: - self.__context.GoToHC(max(0, closest - 1)) - - if wx.IsBusy(): - wx.EndBusyCursor() - - def __AddWatch(self, relative: bool) -> None: - frame = self.__context.GetFrame() - assert frame is not None - watch = cast(WatchListDlg, - frame.ShowDialog('watchlist', WatchListDlg)) - assert self.element is not None - t_offset = cast(int, self.element.GetProperty('t_offset')) - loc = cast(str, self.element.GetProperty('LocationString')) - watch.Add(loc, t_offset, relative=relative) - - def HandleMouseMove(self, - position: wx.Point, - canvas: Layout_Canvas, - redraw: bool = True) -> None: - ''' - Called when mouse move event happens in correct circumstances. - hover needs to be enabled and mode needs to not be edit - ''' - # On a different tick for the same element, force an upate - force_update = self.__context.GetHC() != self.__last_move_tick - self.__last_move_tick = self.__context.GetHC() - (x, y) = canvas.CalcUnscrolledPosition(position) - hits = self.__context.DetectCollision((x, y), include_subelements=True) - old_show_state = self.show - self.show = False - for hit in hits: - e = hit.GetElement() - if e is not None: - self.show = True - # As long as we're showing the preview eventually, - # capture the position. - # offset our box slightly so pointer doesn't obscure the text - self.position = wx.Point(position[0] + 10, position[1] + 10) - if force_update or self.IsDifferent(e): - self.__SetElement(hit) - - is_dirty = old_show_state or self.show - if is_dirty: - assert self.element is not None - self.GetWindow().UpdateInfo(self.element, - self.annotation, - self.__value, - self.position) - if self.__value: - self.GetWindow().Show(True) - else: - self.DestroyWindow() - elif not self.show: - self.DestroyWindow() - - def __SetElement(self, pair: Element_Value) -> None: - ''' - Sets current element - @param pair Element_Value pair - Internal method to implement HandleMouseMove and SetElement - ''' - annotation = pair.GetVal() - e = pair.GetElement() - self.element = e - has_loc_str = e.HasProperty('LocationString') - if has_loc_str: - loc_str = cast(str, e.GetProperty('LocationString')) - has_loc_str = has_loc_str and loc_str != '' - else: - loc_str = None - - # If element currently has an annotation to display - if annotation and has_loc_str: - t_offset = cast(int, e.GetProperty('t_offset')) - self.annotation = cast(str, annotation) - tick_time = \ - int(t_offset * pair.GetClockPeriod() + self.__context.GetHC()) - if tick_time >= 0: - # make query - - results = self.__context.GetTransactionFields( - tick_time, - cast(str, self.element.GetProperty('LocationString')), - self.__fields - ) - intermediate = '' - for field in list(results.keys()): - intermediate += '%s: %s\n' % (field, results[field]) - self.annotation = str(results.get('annotation')) - self.SetValue(intermediate[:-1]) # remove extra newline - else: - self.SetValue(self.annotation) - else: - if e.HasProperty('tooltip') and e.GetProperty('tooltip'): - self.annotation = cast(str, e.GetProperty('tooltip')) - self.SetValue(self.annotation) - elif has_loc_str: - self.annotation = f'<{loc_str}>' - self.SetValue(self.annotation) - elif e.HasProperty('data'): # For node_element - self.annotation = f'<{e.GetProperty("data")}>' - self.SetValue(self.annotation) - elif e.HasProperty('annotation_basis'): # For rpc_element - self.annotation = cast(str, annotation) - self.SetValue(self.annotation) - else: - # No idea what to print for whatever type of element this is - self.annotation = repr(e) - self.SetValue(self.annotation) - - def DestroyWindow(self) -> None: - if self.__window: - self.__window.Destroy() - self.__window = None - - -class HoverPreviewOptionsDialog(wx.Dialog): - ''' - Hover Options dialog display and collection code. No really good place to - put this. - ''' - - def __init__(self, - parent: Argos_Menu, - hover_preview: HoverPreview) -> None: - wx.Dialog.__init__(self, - parent, - wx.NewId(), - 'Hover Preview Options') - - self.hover_preview = hover_preview - self.checkOptions = { - 'start': wx.CheckBox(self, wx.NewId(), 'Start'), - 'end': wx.CheckBox(self, wx.NewId(), 'End'), - 'start_cycle': wx.CheckBox(self, wx.NewId(), 'Start Cycle'), - 'end_cycle': wx.CheckBox(self, wx.NewId(), 'End Cycle'), - 'transaction': wx.CheckBox(self, wx.NewId(), 'Transaction ID'), - 'loc': wx.CheckBox(self, wx.NewId(), 'Location'), - 'parent': wx.CheckBox(self, wx.NewId(), 'Parent Transaction'), - 'opcode': wx.CheckBox(self, wx.NewId(), 'OP Code'), - 'vaddr': wx.CheckBox(self, wx.NewId(), 'Virtual Address'), - 'paddr': wx.CheckBox(self, wx.NewId(), 'Physical Address'), - 'clock': wx.CheckBox(self, wx.NewId(), 'Clock'), - 'annotation': wx.CheckBox(self, wx.NewId(), 'Annotation'), - 'time': wx.CheckBox(self, wx.NewId(), 'Time') - } - - sizer = wx.BoxSizer(wx.VERTICAL) - for value in list(self.checkOptions.values()): - sizer.Add(value, proportion=1, flag=wx.LEFT, border=5) - - done = wx.Button(self, wx.NewId(), 'Done') - - sizerbuttons = wx.BoxSizer(wx.HORIZONTAL) - sizerbuttons.Add(done, proportion=2, flag=wx.ALL, border=5) - self.__select = wx.Button(self, wx.NewId(), 'Select All') - sizerbuttons.Add(self.__select, proportion=2, flag=wx.ALL, border=5) - sizer.Add(sizerbuttons) - self.SetSizer(sizer) - self.Bind(wx.EVT_BUTTON, self.OnSelect, self.__select) - self.Bind(wx.EVT_BUTTON, self.OnDone, done) - self.is_all_selected = True - - # set current settings - self.SetOptions() - self.Fit() - - # Goes through check box elements and appends the checked keys to a list. - def GetOptions(self) -> List[str]: - checked_options = [] - for key, val in self.checkOptions.items(): - if val.GetValue(): - checked_options.append(key) - return checked_options - - def SetOptions(self) -> None: - fields = self.hover_preview.GetFields() - for key, val in self.checkOptions.items(): - if key in fields: - val.SetValue(True) - - def OnSelect(self, evt: wx.CommandEvent) -> None: - if self.is_all_selected: - self.__select.SetLabel('Deselect') - else: - self.__select.SetLabel('Select All') - for val in self.checkOptions.values(): - val.SetValue(self.is_all_selected) - self.is_all_selected = not self.is_all_selected - - def OnDone(self, evt: wx.CommandEvent) -> None: - self.EndModal(wx.ID_OK) - self.hover_preview.SetFields(self.GetOptions()) diff --git a/helios/pipeViewer/pipe_view/gui/input_decoder.py b/helios/pipeViewer/pipe_view/gui/input_decoder.py deleted file mode 100644 index 3203da36c7..0000000000 --- a/helios/pipeViewer/pipe_view/gui/input_decoder.py +++ /dev/null @@ -1,614 +0,0 @@ -from __future__ import annotations -import wx -from . import key_definitions - -from typing import Tuple, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from wx.lib.dragscroller import DragScroller - from .dialogs.element_propsdlg import Element_PropsDlg - from .hover_preview import HoverPreview - from .layout_canvas import Layout_Canvas - from .selection_manager import Selection_Mgr - from ..model.layout_context import Layout_Context - - -# This class provides a central hub of intelligence for responding to -# user-events, namely Mouse or Keyboard, & issuing commands to the likes of -# Canvases, Selection Mgrs, Property Dialogs, etc. -# Should be a singleton class on the view-side -class Input_Decoder: - - __MOVE_SPEED = 5 - __ACCEL_FACTOR = 2.0 - - __ZOOM_MULT = 0.75 - - __SHIFT_STEP_DISTANCE = 10 - __CTRL_STEP_DISTANCE = 100 - - # TODO: improve later - def __init__(self, parent: Layout_Canvas) -> None: - self.__edit_mode = False - self.__parent = parent - self.__context = parent.context - self.__move_speed = self.__MOVE_SPEED - self.__accel_factor = self.__ACCEL_FACTOR - # used to prevent insane quantities of rapid fire copied elementts - # being added to the layout - self.__copy_completed = False - self.__is_traveling = False - - # Handle scenarios for selecting/deslecting Elements based off of Left - # Mouse Down events - def LeftDown(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - dialog: Element_PropsDlg) -> None: - # Quickfix for re-enabling canvas key events after using frame - # playback controls - canvas.SetFocus() - - canvas.CaptureMouse() - - event_pos = canvas.ScreenToClient( - cast(wx.Window, - event.GetEventObject()).ClientToScreen(event.GetPosition()) - ) - - pos = canvas.CalcUnscrolledPosition(event_pos) - - # small routine for playback mode - if not self.__edit_mode: - hits = canvas.context.DetectCollision(pos, - include_subelements=True) - for hit in reversed(hits): # Walk top to bottom layered - if hit.GetElement().IsSelectable(): - selection.SetPlaybackSelected(hit) - break - return - - # Edit-mode selection - hits = canvas.context.DetectCollision(pos, - include_subelements=False, - include_nondrawables=True) - - # Check to see if user clicked a selection handle to initiate a resize - # event - # NOTE: initiating a resize event will short-circuit all other - # LeftDown logic. Fun! - resize_evt = selection.HitHandle(pos) - if resize_evt: - selection.BeginCheckpoint('resize elements') - selection.SetHistory(selection.resize) - return - # Sorta like shift-selecting multiple elements in a spreadsheet or grid - # NOTE: pressing shift will short-circuit all other LeftDown logic. - if event.ShiftDown() and not event.ControlDown() and len(hits) > 0: - selection.SetHistory(selection.collision) - top_hit = hits[0].GetElement() - selection.SetDominant(top_hit) - top_hit_pos = cast(Tuple[int, int], - top_hit.GetProperty('position')) - top_hit_dim = cast(Tuple[int, int], - top_hit.GetProperty('dimensions')) - exes = [top_hit_pos[0]] - whys = [top_hit_pos[1]] - exes.append(top_hit_pos[0]+top_hit_dim[0]) - whys.append(top_hit_pos[1]+top_hit_dim[1]) - hits = canvas.DetectCollision(selection.GetPos()) - for val in hits: - e = val.GetElement() - if selection.IsSelected(e): - (ex, ey) = cast(Tuple[int, int], e.GetProperty('position')) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - exes.append(ex) - whys.append(ey) - exes.append(ex + w) - whys.append(ey + h) - # tl = top left; br = bottom right - tl_pos = (min(exes), min(whys)) - br_pos = (max(exes), max(whys)) - selection.SetPos(tl_pos) - add_these = selection.CalcAddable(br_pos) - for el in add_these: - selection.Add(el) - # necessary to make the selection permanent, and also prevent it - # from interfering with future rubber-band operations - selection.FlushTempRubber() - # corner case(s) - # If you don't like the way corner cases are handled, try toggling - # the variable corner_case below. Hint: shift selecting across - # layered Elements is weird, period. - corner_case = 0 - if corner_case == 0: - # Trust me, it is indeed supposed to be Addable(), not - # Removable() - remove_these = selection.CalcAddable(pos) - for el in remove_these: - selection.Remove(el) - selection.FlushTempRubber() - selection.SetPos(pos) - elif corner_case == 1: - selection.Add(top_hit) - # end corner cases - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - # Keep the return statement! Shift-selecting should short circuit - # the rest of LeftDown() processing - return - - # Multiple Elements under mouse click, fun times ensue - if len(hits) > 1: - selected_hits = [] - not_selected_hits = [] - # Sort the elements beneath the Mouse click into selected or not - for pair in hits: - e = pair.GetElement() - if selection.IsSelected(e): - selected_hits.append(e) - else: - not_selected_hits.append(e) - # Process what to do - # Add the next Element - if event.ControlDown() and not event.ShiftDown(): - selection.SetPos(pos) - if not_selected_hits: - selection.Add(not_selected_hits[-1]) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - selection.SetHistory(selection.collision) - selection.SetDominant(not_selected_hits[-1]) - # Remove the top element - elif event.ControlDown(): - selection.SetHistory(selection.whitespace) - if selected_hits: - selection.Remove(selected_hits[-1]) - - else: - # Maybe we are about to clk-n-drag? - if selected_hits: - selection.SetPos(pos) - selection.SetHistory(selection.prep_remove) - selection.SetDominant(selected_hits[-1]) - # Else Select just one guy - else: - selection.Clear() - selection.SetPos(pos) - selection.SetHistory(selection.collision) - ele = hits[-1].GetElement() - selection.SetDominant(ele) - selection.Add(ele) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - # Single Element under mouse click - elif len(hits) == 1: - e = hits[0].GetElement() - if selection.IsSelected(e): - selection.SetPos(pos) - if event.ControlDown() and not event.ShiftDown(): - selection.SetHistory(selection.collision) - selection.SetDominant(e) - elif event.ControlDown(): - selection.Remove(e) - selection.SetHistory(selection.whitespace) - # Show the user immediately that it was removed - canvas.FullUpdate() - else: - selection.SetHistory(selection.prep_remove) - selection.SetDominant(e) - else: - if event.ControlDown() and not event.ShiftDown(): - selection.SetPos(pos) - selection.Add(e) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - selection.SetHistory(selection.collision) - selection.SetDominant(e) - elif event.ControlDown(): - selection.SetHistory(selection.whitespace) - else: - selection.Clear() - selection.SetPos(pos) - selection.Add(e) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - selection.SetHistory(selection.collision) - selection.SetDominant(e) - - # Selection Manager is responsible for updating the properties - # dialog - - # No Elements under mouse click - elif len(hits) == 0: - selection.SetHistory(selection.rubber_band) - selection.SetPos(pos) - if not (event.ControlDown() and event.ShiftDown()): - selection.ProcessRubber(pos, selection.add) - else: - selection.ProcessRubber(pos, selection.remove) - - if not event.ControlDown(): - selection.Clear() - - # Handle scenarios for selecting/deselecting Elements - def LeftUp(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - dialog: Element_PropsDlg) -> None: - - # Allow Mouse UP action in non-edit mode - - hist = selection.GetHistory() - # A history of prep_remove could mean a few things, depending on what - # the user did between the mouse down which set the history, and the - # mouse up - if hist == selection.prep_remove: - pos = canvas.CalcUnscrolledPosition(event.GetPosition()) - hits = canvas.DetectCollision(pos) - # A Selection Mgr's history should only be set to prep-remove - # during a mouse down over one or more elements, and should be - # changed to something else by movement before a MouseUp - if len(hits) > 0: - selection.SetHistory(selection.collision) - if len(hits) == 1 or \ - selection.IsSelected(hits[0].GetElement()): - selection.Clear() - selection.Add(hits[-1].GetElement()) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - else: - i = 0 - while not selection.IsSelected(hits[i+1].GetElement()): - i = i+1 - selection.Clear() - selection.Add(hits[i].GetElement()) - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - else: - print("this is some weird case of the clicks that isn't supposed to be possible...") # noqa: E501 - print("you should probably make note of what you just did, and file a bug") # noqa: E501 - # Rubber band op finished, clean up the temp memory - elif hist == selection.rubber_band: - selection.FlushTempRubber() - selection.SetHistory(selection.collision) - canvas.FullUpdate() - # Resize op finished, clean up the temp memory - elif hist == selection.resize: - selection.FlushQueue() - selection.CommitCheckpoint() - elif hist == selection.dragged: - selection.CommitCheckpoint() - # Allow the user to click elsewhere, ... - if not event.AltDown(): - while canvas.HasCapture(): - canvas.ReleaseMouse() - canvas.FullUpdate() - - # Set & Show the Element Props Dlg, if double clicked on an Element - def LeftDouble(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - dialog: Element_PropsDlg) -> None: - - # Perform no action unless in edit mode - if not self.__edit_mode: - return - - if not event.AltDown(): - pos = canvas.CalcUnscrolledPosition(event.GetPosition()) - hits = canvas.DetectCollision(pos) - if len(hits) == 1: - # Selection Manager is responsible for updating the properties - # dialog - dialog.Show() - dialog.SetFocus() - dialog.Raise() - - # When the Mouse Right button is pressed, - def RightDown(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - drgscrl: DragScroller) -> None: - - # Perform no action unless in edit mode - if not self.__edit_mode: - return - - drgscrl.Start(canvas.CalcUnscrolledPosition(event.GetPosition())) - - # When the Mouse Right button is released, - def RightUp(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - drgscrl: DragScroller) -> None: - if self.__edit_mode: - drgscrl.Stop() - canvas.GetHoverPreview().HandleMenuClick(event.GetPosition()) - - # Watch the mouse move, re-draw & update Element positions if the user - # is click-and-dragging Elements, or rubber-band-boxing stuff - def MouseMove(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - context: Layout_Context, - mouse_over_preview: HoverPreview) -> None: - if self.__edit_mode: - pos = canvas.CalcUnscrolledPosition(event.GetPosition()) - - # Update the mouse location in the edit toolbar if applicable - if self.GetEditMode(): - canvas.GetFrame().UpdateMouseLocation(pos) - if event.LeftIsDown(): - hist = selection.GetHistory() - if hist == selection.rubber_band: - canvas.FullUpdate() - if not (event.AltDown()): - selection.ProcessRubber(pos) - else: - selection.ProcessRubber(pos) - elif hist == selection.whitespace: - return # Do not drag the selection - elif hist in (selection.collision, - selection.dragged, - selection.prep_remove): - if selection.HasOpenCheckpoint() is False: - # Should be transitioning to dragged for the first - # time, but maybe not - selection.BeginCheckpoint('drag elements') - selection.Move(pos=pos) - selection.SetHistory(selection.dragged) - canvas.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - canvas.Refresh() - elif hist == selection.resize: - if event.AltDown(): - selection.Resize(pos, mode='two') - else: - selection.Resize(pos) - # No click-n-dragging, just waving the mouse around - else: - if selection.DetectCollision(pos): - selection.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - if selection.HitHandle(pos): - selection.SetCursor() - elif mouse_over_preview.IsEnabled(): - mouse_over_preview.HandleMouseMove(event.GetPosition(), canvas) - - def MouseWheel(self, event: wx.MouseEvent, canvas: Layout_Canvas) -> None: - # canvas scale - if event.GetModifiers() == wx.MOD_CONTROL: - rotation = event.GetWheelRotation()/event.GetWheelDelta() - if rotation > 0: - factor = 1.05 - else: - factor = 0.95 - canvas.SetScale(canvas.GetScale() * factor) - else: - event.Skip() - - # Initiate a scrolling operation on the canvas - def MiddleDown(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr) -> None: - # Perform no action unless in edit mode - if not self.__edit_mode: - return - - return - - # End the scrolling operation - def MiddleUp(self, - event: wx.MouseEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr) -> None: - return - - # Handle keystrokes as necessary - def KeyDown(self, - event: wx.KeyEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - dialog: Element_PropsDlg, - context: Layout_Context) -> None: - # Note: KeyCodes are all equivalent to the ASCII values for the - # corresponding capital character - key = event.GetKeyCode() - # print key - key_handled = False - # Mode-dependent key mappings - if not self.__edit_mode: - # Playback Mode - playback_panel = self.__parent.GetFrame().GetPlaybackPanel() - if key in key_definitions.STEP_FORWARD: - if not self.__is_traveling: - self.__is_traveling = True - if event.ShiftDown(): - # move multiple spaces - playback_panel.StepForward(self.__SHIFT_STEP_DISTANCE) - else: - playback_panel.StepForward() # move one space - playback_panel.StartPlaying(9, delay=1) - key_handled = True - - elif key in key_definitions.STEP_BACKWARD: - if not self.__is_traveling: - self.__is_traveling = True - if event.ShiftDown(): - # move multiple spaces - playback_panel.StepBackward(self.__SHIFT_STEP_DISTANCE) - else: - playback_panel.StepBackward() # move one space - playback_panel.StartPlaying(-9, delay=1) - key_handled = True - - elif key in key_definitions.STEP_FORWARD_10: - if not self.__is_traveling: - self.__is_traveling = True - if event.ControlDown(): - # move (more) multiple spaces - playback_panel.StepForward(self.__CTRL_STEP_DISTANCE) - else: - # move multiple spaces - playback_panel.StepForward(self.__SHIFT_STEP_DISTANCE) - playback_panel.StartPlaying(9, delay=1) - key_handled = True - - elif key in key_definitions.STEP_BACKWARD_10: - if not self.__is_traveling: - self.__is_traveling = True - if event.ControlDown(): - # move (more) multiple spaces - playback_panel.StepBackward(self.__CTRL_STEP_DISTANCE) - else: - # move multiple spaces - playback_panel.StepBackward(self.__SHIFT_STEP_DISTANCE) - playback_panel.StartPlaying(-9, delay=1) - key_handled = True - - else: # if not self.__edit_mode: - - # Edit mode - if key_definitions.isNegativeSelection(key, event): - canvas.CaptureMouse() - key_handled = True - - # up-arrow Move Selection - elif key == key_definitions.MOVE_EL_UP: - mod = 1.0 - if key_definitions.isSlowMove(event): - mod = self.__move_speed - elif key_definitions.isFastMove(event): - mod = 1/self.__accel_factor - selection.BeginCheckpoint('move element up') - try: - selection.Move(delta=(0, int(-self.__move_speed/mod))) - finally: - selection.CommitCheckpoint() - key_handled = True - - # down-arrow Move Selection - elif key == key_definitions.MOVE_EL_DOWN: - mod = 1.0 - if key_definitions.isSlowMove(event): - mod = self.__move_speed - elif key_definitions.isFastMove(event): - mod = 1/self.__accel_factor - selection.BeginCheckpoint('move element down') - try: - selection.Move(delta=(0, int(self.__move_speed/mod))) - finally: - selection.CommitCheckpoint() - key_handled = True - - # Left-arrow Move Selection - elif key == key_definitions.MOVE_EL_LEFT: - mod = 1.0 - if key_definitions.isSlowMove(event): - mod = self.__move_speed - elif key_definitions.isFastMove(event): - mod = 1/self.__accel_factor - selection.BeginCheckpoint('move element left') - try: - selection.Move(delta=(int(-self.__move_speed/mod), 0)) - finally: - selection.CommitCheckpoint() - key_handled = True - - # right-arrow Move Selection - elif key == key_definitions.MOVE_EL_RIGHT: - mod = 1.0 - if key_definitions.isSlowMove(event): - mod = self.__move_speed - elif key_definitions.isFastMove(event): - mod = 1/self.__accel_factor - selection.BeginCheckpoint('move element right') - try: - selection.Move(delta=(int(self.__move_speed/mod), 0)) - finally: - selection.CommitCheckpoint() - key_handled = True - - # Snap all edges to the grid if applicable - elif key_definitions.isSnapToGrid(key, event): - selection.SnapCorner() - key_handled = True - - if key_handled: - canvas.SetFocus() - - # Mode-independent key mappings - - # Advance Hypercycle - if key == key_definitions.ADVANCE_HYPERCYCLE: - context.GoToHC(context.GetHC() + 1) - - # Turn background grid on/off - elif key_definitions.isToggleBackgroundGrid(key, event): - canvas.ToggleGrid() - - # jump - elif key == key_definitions.JUMP_KEY: - self.__parent.GetFrame().FocusJumpBox() - - # zoom in - elif key_definitions.isZoomInKey(key, event): - canvas.SetScale(canvas.GetScale()/self.__ZOOM_MULT) - - # zoom out - elif key_definitions.isZoomOutKey(key, event): - canvas.SetScale(canvas.GetScale()*self.__ZOOM_MULT) - - # zoom reset - elif key_definitions.isZoomResetKey(key, event): - canvas.SetScale(1.0) - - # next change - elif key_definitions.isNextChange(key, event): - canvas.GetHoverPreview().GotoNextChange() - - # previous change - elif key_definitions.isPrevChange(key, event): - canvas.GetHoverPreview().GotoPrevChange() - - # otherwise, this event should be handled by someone else - elif not key_handled: - event.Skip() - - # Handle key releases as necessary - def KeyUp(self, - event: wx.KeyEvent, - canvas: Layout_Canvas, - selection: Selection_Mgr, - dialog: Element_PropsDlg, - context: Layout_Context) -> None: - # Note: KeyCodes are all equivalent to the ASCII values for the - # corresponding capital character - key = event.GetKeyCode() - # Allow another copy event on another key down - if key == ord('D'): - selection.PrepNextCopy() - elif (key in key_definitions.STEP_FORWARD) or \ - (key in key_definitions.STEP_BACKWARD) or \ - (key in key_definitions.STEP_FORWARD_10) or \ - (key in key_definitions.STEP_BACKWARD_10): - if self.__is_traveling: - layout_frame = self.__parent.GetFrame() - layout_frame.GetPlaybackPanel().PausePlaying() - self.__is_traveling = False - - def Undo(self) -> None: - self.__parent.GetSelectionManager().Undo() - - def Redo(self) -> None: - self.__parent.GetSelectionManager().Redo() - - # Used for specifying edit mode - # @param menuEditBool Edit Mode on or off - # @param selection SelectionManager instance - def SetEditMode(self, - menuEditBool: bool, - selection: Selection_Mgr) -> None: - self.__edit_mode = menuEditBool - selection.SetEditMode(menuEditBool) - - # Returns whether this decoder is in edit mode - def GetEditMode(self) -> bool: - return self.__edit_mode diff --git a/helios/pipeViewer/pipe_view/gui/key_definitions.py b/helios/pipeViewer/pipe_view/gui/key_definitions.py deleted file mode 100644 index 7a43577b94..0000000000 --- a/helios/pipeViewer/pipe_view/gui/key_definitions.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations -import wx - -# This module provides a central place to define how the keys -# are mapped to the interface. - -# playback mode -STEP_FORWARD = [wx.WXK_UP, wx.WXK_RIGHT] -STEP_BACKWARD = [wx.WXK_DOWN, wx.WXK_LEFT] -STEP_FORWARD_10 = [wx.WXK_PAGEUP] -STEP_BACKWARD_10 = [wx.WXK_PAGEDOWN] - -# edit mode -# element moving keys -MOVE_EL_UP = wx.WXK_UP -MOVE_EL_DOWN = wx.WXK_DOWN -MOVE_EL_LEFT = wx.WXK_LEFT -MOVE_EL_RIGHT = wx.WXK_RIGHT - -# mode independent mappings -ADVANCE_HYPERCYCLE = ord(' ') -JUMP_KEY = ord('G') -FLIP_SELECTION_HORIZ = [wx.WXK_HOME, wx.WXK_END] -FLIP_SELECTION_VERT = [wx.WXK_PAGEDOWN, wx.WXK_PAGEUP] - - -# checks if motion of element should be fast -def isFastMove(event: wx.KeyEvent) -> bool: - return event.ShiftDown() and not event.ControlDown() - - -def isSlowMove(event: wx.KeyEvent) -> bool: - return event.ControlDown() and not event.ShiftDown() - - -# when selecting objects, negative selection behavior -def isNegativeSelection(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - if (key == wx.WXK_CONTROL) and event.ShiftDown(): - return True - elif (key == wx.WXK_SHIFT) and event.ControlDown(): - return True - return False - - -def isToggleBackgroundGrid(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('G') and event.ControlDown() - - -def isSnapToGrid(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('G') and event.ShiftDown() - - -def isZoomInKey(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('=') and event.ControlDown() - - -def isZoomOutKey(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('-') and event.ControlDown() - - -def isZoomResetKey(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('0') and event.ControlDown() - - -# Goto Next annotation change -def isNextChange(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('N') and not event.ShiftDown() - - -# Goto Prev annotation change -def isPrevChange(key: wx.KeyCode, event: wx.KeyEvent) -> bool: - return key == ord('N') and event.ShiftDown() diff --git a/helios/pipeViewer/pipe_view/gui/layout_canvas.py b/helios/pipeViewer/pipe_view/gui/layout_canvas.py deleted file mode 100644 index 3d0be4f623..0000000000 --- a/helios/pipeViewer/pipe_view/gui/layout_canvas.py +++ /dev/null @@ -1,663 +0,0 @@ -from __future__ import annotations -import wx -import wx.lib.dragscroller as drgscrlr -import math -import sys -import time -import logging -from typing import Any, Dict, List, Optional, Tuple, Union, cast, TYPE_CHECKING - -from .selection_manager import Selection_Mgr -from .input_decoder import Input_Decoder -from .hover_preview import HoverPreview -from functools import partial -from . import autocoloring -from ..model import highlighting_utils -from .font_utils import GetMonospaceFont, GetDefaultFont - -if TYPE_CHECKING: - from .dialogs.element_propsdlg import Element_PropsDlg - from .layout_frame import Layout_Frame - from ..model.element import Element - from ..model.element_value import Element_Value - from ..model.layout_context import Layout_Context - from ..model.settings import ArgosSettings - -try: - from .. import core -except ImportError as e: - print('Argos failed to import module: "core". Argos requires Make', - file=sys.stderr) - print(f'Exception: {e}', file=sys.stderr) - sys.exit(1) - - -ColoredTransactionTuple = \ - Tuple[str, wx.Brush, bool, bool, Optional[int], Optional[int]] - - -class Layout_Canvas(wx.ScrolledWindow): - - __AUTO_CANVAS_SIZE_MARGIN = 40 # Pixels - - MIN_WIDTH = 500 - MIN_HEIGHT = 500 - MIN_ZOOM = 0.1 - MAX_ZOOM = 2 - - # Construct a Layout_Canvas - # @param parent Parent Layout_Frame - # @param context Associated Layout_Context - # @param dialog Element Properties Dialog for this canvas/frame - # @param ID Window ID - def __init__(self, - parent: Layout_Frame, - context: Layout_Context, - dialog: Element_PropsDlg, - ID: int = -1, - pos: wx.Point = wx.DefaultPosition, - size: wx.Size = wx.DefaultSize, - style: int = wx.NO_FULL_REPAINT_ON_RESIZE): - - wx.ScrolledWindow.__init__(self, parent) - self.SetBackgroundStyle(wx.BG_STYLE_PAINT) - self.SetBackgroundColour(wx.WHITE) - self.__renderer = core.Renderer() - assert autocoloring.REASON_BRUSHES and autocoloring.BACKGROUND_BRUSHES - self.SetRendererBrushes() - self.__renderer.setExtensions(context.GetExtensionManager()) - self.__dragscrl = drgscrlr.DragScroller(self, - rate=300, - sensitivity=.01) - self.__WIDTH = 2000 - self.__HEIGHT = 2000 - - # full canvas scale - self.__canvas_scale = 1.0 - self.__font_scale = (1.0, 1.0) - - self.__SCROLL_RATE = 20 - self.__scroll_ratios = (0.0, 0.0) - self.__UpdateScrollbars() - self.__parent = parent - self.__context = context - self.__layout = context.GetLayout() - self.__dlg = dialog - self.__draw_grid = False - # TODO, make neat #################################### - self.__gridsize = 14 - self.__gridlines: List[Tuple[int, int, int, int]] = [] - self.__UpdateGrid(self.__gridsize) - # End TODO ########################################### - self.__selection = Selection_Mgr(self, dialog) - self.__user_handler = Input_Decoder(self) - - self.__set_renderer_font = False - self.__schedule_line_draw_style = 4 # classic - - # used for color highlighting of transactions - self.__colored_transactions: Dict[str, ColoredTransactionTuple] = {} - - self.__hover_preview = HoverPreview(self, context) - # Load images - - self.__fnt_layout = GetMonospaceFont( - self.GetSettings().layout_font_size - ) - self.__UpdateFontScaling() - - # Disable background erasing - def disable_event(*pargs: Any, **kwargs: Any) -> None: - pass - - self.Bind(wx.EVT_ERASE_BACKGROUND, disable_event) - self.Bind(wx.EVT_PAINT, self.OnPaint) - self.Bind(wx.EVT_SIZE, self.OnSize) - - # Bindings for mouse events - self.Bind(wx.EVT_LEFT_DOWN, - partial(self.__user_handler.LeftDown, - canvas=self, - selection=self.__selection, - dialog=self.__dlg)) - self.Bind(wx.EVT_MOTION, - partial(self.__user_handler.MouseMove, - canvas=self, - selection=self.__selection, - context=self.__context, - mouse_over_preview=self.__hover_preview)) - self.Bind(wx.EVT_LEFT_DCLICK, - partial(self.__user_handler.LeftDouble, - canvas=self, - selection=self.__selection, - dialog=self.__dlg)) - self.Bind(wx.EVT_LEFT_UP, - partial(self.__user_handler.LeftUp, - canvas=self, - selection=self.__selection, - dialog=self.__dlg)) - self.Bind(wx.EVT_RIGHT_DOWN, - partial(self.__user_handler.RightDown, - canvas=self, - drgscrl=self.__dragscrl)) - self.Bind(wx.EVT_RIGHT_UP, - partial(self.__user_handler.RightUp, - canvas=self, - drgscrl=self.__dragscrl)) - # Bindings for key events - self.Bind(wx.EVT_KEY_DOWN, - partial(self.__user_handler.KeyDown, - canvas=self, - selection=self.__selection, - dialog=self.__dlg, - context=self.__context)) - self.Bind(wx.EVT_KEY_UP, - partial(self.__user_handler.KeyUp, - canvas=self, - selection=self.__selection, - dialog=self.__dlg, - context=self.__context)) - - self.Bind(wx.EVT_MOUSEWHEEL, - partial(self.__user_handler.MouseWheel, canvas=self)) - - # required to actually receive key events - self.SetFocus() - - self.Bind(wx.EVT_SCROLLWIN, self.ScrollWin) - - # Size of the canvas grid - @property - def gridsize(self) -> int: - return self.__gridsize - - # Snap sensitivity range (+/- each gridline) - @property - def range(self) -> int: - return self.__snap_capture_delta - - @property - def scrollrate(self) -> int: - return self.__SCROLL_RATE - - @property - def context(self) -> Layout_Context: - return self.__context - - # Returns the parent frame which owns this Canvas - def GetFrame(self) -> Layout_Frame: - return self.__parent - - # Returns a tuple showing the area of the canvas visible to the screen - # @return (x,y,w,h) - def GetVisibleArea(self) -> Tuple[int, int, int, int]: - x, y = self.GetViewStart() - w, h = self.GetClientSize() - return (x, y, w, h) - - # Updates color for a transaction - def UpdateTransactionColor(self, - el: Element, - annotation: str) -> Tuple[str, wx.Brush]: - if el.HasProperty('auto_color_basis') \ - and el.HasProperty('color_basis_type'): - auto_color_basis = cast(str, el.GetProperty('auto_color_basis')) - color_basis_type = cast(str, el.GetProperty('color_basis_type')) - content_type = cast(str, el.GetProperty('Content')) - record = self.GetTransactionColor(annotation, - content_type, - color_basis_type, - auto_color_basis) - if record: - string_to_display, brush, _, _, _, _ = record - else: - string_to_display, brush, _, _ = self.AddColoredTransaction( - annotation, - content_type, - color_basis_type, - auto_color_basis, - 0, - el - ) - else: - string_to_display = self.__hover_preview.annotation - brush = wx.Brush((200, 200, 200)) - - return string_to_display, brush - - # Here the canvas does all its drawing - def DoDraw(self, dc: wx.DC) -> None: - logging.debug('xxx: Started C drawing') - - # Draw grid - # This has effectively 0 cost - if self.__draw_grid: - brush = dc.GetBackground() - brush.SetColour('black') - dc.SetBackground(brush) - dc.Clear() - dc.SetPen(wx.Pen((220, 220, 220), 1)) - # dc.SetLogicalFunction(wx.INVERT) - xoff, yoff = self.GetRenderOffsets() - for x, y, x1, y1 in self.__gridlines: - dc.DrawLine(int(x - xoff), - int(y - yoff), - int(x1 - xoff), - int(y1 - yoff)) - - t_start = time.monotonic() - self.__renderer.drawElements(dc, self, self.__context.GetHC()) - - logging.debug('%ss: C drawing', time.monotonic() - t_start) - - def ScrollWin(self, evt: wx.ScrollWinEvent) -> None: - super().Refresh() - - bounds = self.__GetScrollBounds() - x_bound = bounds[0] - self.scrollrate - y_bound = bounds[1] - self.scrollrate - - w_pix, h_pix = self.GetClientSize() - percent_bar_x = w_pix / self.__WIDTH - percent_bar_y = h_pix / self.__HEIGHT - - self.__scroll_ratios = ( - self.GetScrollPos(wx.HORIZONTAL) / x_bound + percent_bar_x / 4.0, - self.GetScrollPos(wx.VERTICAL) / y_bound + percent_bar_y / 4.0 - ) - - # If we're in edit mode, update the cursor location in the toolbar - if self.__user_handler.GetEditMode(): - (x, y) = self.GetMousePosition() - offset = self.CalcScrollInc(evt) - scroll_units = self.GetScrollPixelsPerUnit() - - if evt.GetOrientation() == wx.HORIZONTAL: - x += offset * scroll_units[0] - elif evt.GetOrientation() == wx.VERTICAL: - y += offset * scroll_units[1] - - self.__parent.UpdateMouseLocation((x, y)) - - mousePos = wx.GetMousePosition() - self.__hover_preview.HandleMouseMove(mousePos, self) - evt.Skip() - - # Returns the position of the mouse within the canvas - def GetMousePosition(self) -> Tuple[int, int]: - return self.CalcUnscrolledPosition( - self.ScreenToClient(wx.GetMousePosition()) - ) - - # Execute all drawing logic - def FullUpdate(self) -> None: - - # update quad tree (if needed) - self.__context.MicroUpdate() - self.Refresh() - # Tell the Element Prop's Dlg window to update itself - self.__dlg.Refresh() - - # Execute all drawing logic - def OnPaint(self, event: wx.PaintEvent) -> None: - paint_dc = wx.AutoBufferedPaintDC(self) - paint_dc.Clear() - context = wx.GraphicsContext.Create(paint_dc) - dc = wx.GCDC(context) - dc.SetLogicalScale(self.__canvas_scale, self.__canvas_scale) - dc.SetFont(self.__fnt_layout) - if not self.__set_renderer_font: - self.__renderer.setFontFromDC(dc) - self.__set_renderer_font = True - self.DoDraw(dc) - self.__selection.Draw(dc) - - # Returns Hover Preview - def GetHoverPreview(self) -> HoverPreview: - return self.__hover_preview - - # Turns on or off the drawing of the grid - def ToggleGrid(self) -> None: - self.__draw_grid = not self.__draw_grid - self.Refresh() - - # Refresh. Recalc size and forward to superclass - def Refresh( - self, - eraseBackground: bool = True, - rect: Optional[Union[Tuple[int, int, int, int], wx.Rect]] = None - ) -> None: - # Recalculate canvas size - if self.__CalcCanvasSize(): - self.__UpdateScrollbars() - self.__UpdateGrid(self.__gridsize) - - super().Refresh() - - # Gets called when the window is resized and when the canvas is initialized - def OnSize(self, event: wx.SizeEvent) -> None: - # update the screen - self.FullUpdate() - - # Gets the selection manager for this canvas - def GetSelectionManager(self) -> Selection_Mgr: - return self.__selection - - # Gets the input decoder for this canvas - def GetInputDecoder(self) -> Input_Decoder: - return self.__user_handler - - # Forward on the draw-orderd list of all Elements in this - # frame/canvas/layout... - def GetElements(self) -> List[Element]: - return self.__context.GetElements() - - # Return Element/Value pairs - def GetElementPairs(self) -> List[Element_Value]: - return self.__context.GetElementPairs() - - # Compute and returns display bounds - def GetBounds(self) -> Tuple[int, int, int, int]: - posx, posy, width, height = self.GetVisibleArea() - mins = self.CalcUnscrolledPosition((0, 0)) - bounds = (mins[0], - mins[1], - int(mins[0] + width / self.__canvas_scale), - int(mins[1] + height / self.__canvas_scale)) - return bounds - - # Gets the update region relative to the current visible area - # Coordinates are scaled to account for current zoom factor - # Used by schedule elements to determine where to blit - def GetScaledUpdateRegion(self) -> wx.Rect: - box = self.GetUpdateRegion().GetBox() - box = wx.Rect(int(math.floor(box[0] / self.__canvas_scale)), - int(math.floor(box[1] / self.__canvas_scale)), - int(math.ceil(box[2] / self.__canvas_scale)), - int(math.ceil(box[3] / self.__canvas_scale))) - - box.Inflate(10, 10) # Fudge factor to ensure no dirt is left behind - return box - - # Gets the update region relative to the origin of the scrolled area - # Region is scaled to account for current zoom factor - # Used to determine which elements need to be updated - def GetScrolledUpdateRegion(self) -> wx.Rect: - box = self.GetUpdateRegion().GetBox() - box.SetPosition(self.CalcUnscrolledPosition(box.GetTopLeft())) - box.SetHeight(int(math.ceil(box.GetHeight() / self.__canvas_scale))) - box.SetWidth(int(math.ceil(box.GetWidth() / self.__canvas_scale))) - - box.Inflate(10, 10) # Fudge factor to ensure no dirt is left behind - return box - - # Returns element pairs suitable for drawing - def GetDrawPairs(self) -> List[Element_Value]: - box = self.GetScrolledUpdateRegion() - top_left = box.GetTopLeft() - bottom_right = box.GetBottomRight() - bounds = (top_left[0], top_left[1], bottom_right[0], bottom_right[1]) - return self.__context.GetDrawPairs(bounds) - - def GetVisibilityTick(self) -> int: - return self.__context.GetVisibilityTick() - - # Set the brushes in the renderer to the current brush set - def SetRendererBrushes(self) -> None: - self.__renderer.setBrushes(autocoloring.REASON_BRUSHES.as_dict(), - autocoloring.BACKGROUND_BRUSHES.as_dict()) - - # Resets the renderer brushes and purges the brush cache - # Needed whenever the palette or color shuffle mode changes - def RefreshBrushes(self) -> None: - self.SetRendererBrushes() - self.PurgeBrushCache() - - # Returns brushes needed to draw elements - def GetBrushes(self) -> Tuple[autocoloring.BrushRepository, - autocoloring.BrushRepository]: - return autocoloring.BACKGROUND_BRUSHES, autocoloring.REASON_BRUSHES - - # Returns offsets needed to render - def GetRenderOffsets(self) -> Tuple[int, int]: - return self.CalcUnscrolledPosition((0, 0)) - - # Returns the Cython renderer - def GetRenderer(self) -> core.Renderer: - return self.__renderer - - # Returns element settings dialog - def GetDialog(self) -> Element_PropsDlg: - return self.__dlg - - # Returns global setting for schedule line drawing - def GetScheduleLineStyle(self) -> int: - return self.__schedule_line_draw_style - - def GetScale(self) -> float: - return self.__canvas_scale - - # Updates the highlighting state of all cached transactions - def UpdateTransactionHighlighting(self) -> None: - for key, val in self.__colored_transactions.items(): - self.__colored_transactions[key] = \ - val[:2] + \ - (self.__context.IsUopUidHighlighted(val[4]),) + \ - (self.__context.IsSearchResult(val[5]),) + \ - val[4:] - - @staticmethod - def __GetTransactionKey( - annotation: str, - content_type: str, - color_basis_type: str, - auto_color_basis: str - ) -> str: - return f'{annotation}:{color_basis_type}:{auto_color_basis}:{hash(content_type)}' # noqa: E501 - - def GetTransactionColor( - self, - annotation: str, - content_type: str, - color_basis_type: str, - auto_color_basis: str - ) -> Optional[ColoredTransactionTuple]: - return self.__colored_transactions.get( - self.__GetTransactionKey(annotation, - content_type, - color_basis_type, - auto_color_basis) - ) - - # Track the tagging of one transaction - # @note OThis can be called mutiple times to override existing colors - def AddColoredTransaction( - self, - annotation: str, - content_type: str, - color_basis_type: str, - auto_color_basis: str, - start_tick: int, - element: Element - ) -> Tuple[str, wx.Brush, bool, bool]: - if len(self.__colored_transactions) > 10000: - self.__colored_transactions.clear() - - string_to_display, brush = self.__renderer.parseAnnotationAndGetColor( - annotation, - content_type, - color_basis_type, - auto_color_basis - ) - - key = self.__GetTransactionKey(annotation, - content_type, - color_basis_type, - auto_color_basis) - - uop_uid = highlighting_utils.GetUopUid(annotation) - highlighted = self.__context.IsUopUidHighlighted(uop_uid) - - if element.HasProperty('LocationString'): - search_result_hash = self.__context.SearchResultHash( - start_tick, - self.__context.GetLocationId(element) - ) - else: - search_result_hash = None - - search_result = self.__context.IsSearchResult(search_result_hash) - self.__colored_transactions[key] = (string_to_display, - brush, - highlighted, - search_result, - uop_uid, - search_result_hash) - return string_to_display, brush, highlighted, search_result - - # Gets old-style coloring (no basis configured) - def GetAutocolorColor(self, annotation: str) -> wx.Colour: - string_to_display, brush = self.__renderer.parseAnnotationAndGetColor( - annotation, - 'auto_color_annotation' - ) - return brush.GetColour() - - # remove all entries from the dictionary of colored transactions - # will auto regenerate next render frame - def PurgeBrushCache(self) -> None: - self.__colored_transactions.clear() - self.__context.FullRedraw() - self.Refresh() - - # Set the global style for schedule lines - def SetScheduleLineStyle(self, style: int) -> None: - if style != self.__schedule_line_draw_style: - self.__schedule_line_draw_style = style - self.__context.FullUpdate() - self.Refresh() - - def SetScale(self, scale: float) -> None: - # cap at 2x zoom in - self.__canvas_scale = min(max(self.MIN_ZOOM, scale), self.MAX_ZOOM) - - self.__CalcCanvasSize() - self.__UpdateScrollbars() - self.__UpdateGrid(self.__gridsize) - - super().Refresh() - - # Returns a list of all Elements beneath the given point - def DetectCollision( - self, - pt: Union[Tuple[int, int], wx.Point] - ) -> List[Element_Value]: - return self.__context.DetectCollision(pt) - - # override to handle full-canvas scale - def CalcUnscrolledPosition( - self, - position: Union[wx.Point, Tuple[int, int]] - ) -> Tuple[int, int]: - view_start = self.GetViewStart() - x0 = view_start[0] * self.scrollrate - y0 = view_start[1] * self.scrollrate - return (int((x0 + position[0]) / self.__canvas_scale), - int((y0 + position[1]) / self.__canvas_scale)) - - # Calculate the canvas size based on all elements - # @return True if size changed, False if not - def __CalcCanvasSize(self) -> bool: - - l, t, r, b = self.__context.GetElementExtents() - r = int(r * self.__canvas_scale) - b = int(b * self.__canvas_scale) - - width = max(self.MIN_WIDTH, r + self.__AUTO_CANVAS_SIZE_MARGIN) - height = max(self.MIN_HEIGHT, b + self.__AUTO_CANVAS_SIZE_MARGIN) - - if self.__WIDTH == width and self.__HEIGHT == height: - return False # No change - - self.__WIDTH = width - self.__HEIGHT = height - return True - - def __GetScrollBounds(self) -> Tuple[float, float]: - sr = float(self.scrollrate) - return self.__WIDTH / sr, self.__HEIGHT / sr - - # Update the scrollbars based on a new canvas size - # Restores prior scroll offsets - # Uses instance attributes __SCROLL_RATE, __WIDTH, __HEIGHT - def __UpdateScrollbars(self, - force_x: Optional[int] = None, - force_y: Optional[int] = None) -> None: - sr = self.scrollrate - - w_pix, h_pix = self.GetClientSize() - x_bound, y_bound = self.__GetScrollBounds() - x, y = self.__scroll_ratios - percent_bar_x = w_pix / self.__WIDTH - percent_bar_y = h_pix / self.__HEIGHT - if percent_bar_x > 1: - percent_bar_x = 1 - if percent_bar_y > 1: - percent_bar_y = 1 - - if force_x is not None: - x_scroll = force_x - else: - x_scroll = int(x * x_bound * (1 - percent_bar_x)) - - if force_y is not None: - y_scroll = force_y - else: - y_scroll = int(y * y_bound * (1 - percent_bar_y)) - - self.SetScrollbars(sr, - sr, - int(x_bound), - int(y_bound), - x_scroll, - y_scroll, - True) - - # Regenerates the gridlines based on __WIDTH and __HEIGHT - # @param gridsize Space between each gridline - def __UpdateGrid(self, gridsize: int) -> None: - assert gridsize % 2 == 0 - self.__snap_capture_delta = 7 - assert self.__snap_capture_delta <= gridsize / 2 - self.__gridlines = [] - for x in range(round(self.__WIDTH / gridsize)): - grid_x = x * gridsize - self.__gridlines.append((grid_x, 0, grid_x, self.__HEIGHT)) - for y in range(round(self.__HEIGHT / gridsize)): - grid_y = y * gridsize - self.__gridlines.append((0, grid_y, self.__WIDTH, grid_y)) - - def GetSettings(self) -> ArgosSettings: - return self.__parent.GetSettings() - - def __UpdateFontScaling(self) -> None: - default_font_w, default_font_h = GetDefaultFont().GetPixelSize() - cur_font_w, cur_font_h = self.__fnt_layout.GetPixelSize() - self.__font_scale = (cur_font_w / default_font_w, - cur_font_h / default_font_h) - for e in self.__context.GetElementPairs(): - e.GetElement().SetProperty('scale_factor', self.__font_scale) - self.__context.UpdateElementExtents() - - def UpdateFontSize(self) -> None: - old_font = self.__fnt_layout - self.__fnt_layout = GetMonospaceFont( - self.GetSettings().layout_font_size - ) - if old_font.GetPointSize() != self.__fnt_layout.GetPointSize(): - self.__set_renderer_font = False - self.__UpdateFontScaling() - self.__CalcCanvasSize() - self.__UpdateScrollbars(force_x=0, force_y=0) - self.__context.FullRedraw() - self.FullUpdate() diff --git a/helios/pipeViewer/pipe_view/gui/layout_frame.py b/helios/pipeViewer/pipe_view/gui/layout_frame.py deleted file mode 100644 index 085b8f6598..0000000000 --- a/helios/pipeViewer/pipe_view/gui/layout_frame.py +++ /dev/null @@ -1,529 +0,0 @@ -from __future__ import annotations -from contextlib import contextmanager, nullcontext -import logging -import os -import wx -from typing import (Any, - Dict, - Iterator, - List, - Optional, - Tuple, - Type, - Union, - cast, - TYPE_CHECKING) - -from .layout_canvas import Layout_Canvas -from ..model.layout import Layout -from .argos_menu import Argos_Menu -from .dialogs.search_dlg import SearchDialog -from .dialogs.find_element_dlg import FindElementDialog -from .dialogs.element_propsdlg import Element_PropsDlg -from .dialogs.location_window import LocationWindow -from .dialogs.layout_exit_dialog import LayoutExitDialog -from .widgets.frame_playback_bar import FramePlaybackBar - -if TYPE_CHECKING: - from .dialogs.watchlist_dialog import WatchListDlg - from ..model.layout_context import Layout_Context - from ..model.workspace import Workspace - from ..model.settings import ArgosSettings - - -DialogUnion = Union[wx.Frame, wx.Dialog] - - -# The GUI-side top-level 'window' which will house a Layout Canvas, MenuBar -# and Playback Controls. One-to-one mapping with a layout display, and has -# minimal functionality itself. Primarily provides the hosting frame/window -# for the Layout Canvas -class Layout_Frame(wx.Frame): - _DB_UPDATE_DELAY_MS = 10000 - - # Gets the wx frame created, and ties together a Layout Canvas and a - # Element Properties Dialog - def __init__(self, - ws: Workspace, - context: Layout_Context, - update_enabled: bool, - title_prefix: str, - title_override: str, - title_suffix: str) -> None: - self.__workspace = ws - self.__context = context - self.__layout = context.GetLayout() - assert self.__layout is not None - # stores currently active dialogs, keyed by name - self.__dialogs: Dict[str, List[DialogUnion]] = {} - - size = (800, 600) - if ws is not None: - pos = ws.GetNextNewFramePosition(size) - else: - pos = wx.DefaultPosition - - self.__title_prefix = title_prefix - self.__title_override = title_override - self.__title_suffix = title_suffix - - if title_prefix is not None: - title = title_prefix - else: - title = '' - - if title_override is not None: - title += title_override - else: - title += self.ComputeTitle() - - if title_suffix is not None: - title += title_suffix - - wx.Frame.__init__(self, None, -1, title, pos=pos, size=size) - lg = logging.getLogger('Layout_Frame') - lg.debug('Creating Layout_Frame %s at pos %s, size %s', - self, - pos, - size) - - self.__title = title - self.__playback_panel = FramePlaybackBar(self) - - # Element properties dialog - # It is important never to destroy this until the Frame dies - self.__dlg: Optional[Element_PropsDlg] = Element_PropsDlg(self, - -1, - title) - self.__canvas = Layout_Canvas(self, context, self.__dlg) - self.__context.Update() - - self.__dlg.SetElements([], self.__canvas.GetSelectionManager()) - if self.__layout.GetElements(): - self.__dlg.Show(False) - - self.__menu = Argos_Menu(self, self.__layout, update_enabled) - self.SetMenuBar(self.__menu) - self.SetToolBar(self.__menu.GetEditToolbar()) - - # Enter edit mode if this is a new layout (no associated file on disk) - if self.__layout.GetFilename() is None: - self.SetEditMode(True) - - # Layout - - vert_sz = wx.BoxSizer(wx.VERTICAL) - vert_sz.Add(self.__canvas, 1, wx.EXPAND) - vert_sz.Add(self.__playback_panel, 0, wx.EXPAND) - self.SetSizer(vert_sz) - - # Binding - - self.Bind(wx.EVT_CLOSE, self.Close) - self.Bind(wx.EVT_SIZE, self.__OnResize) - - # Register self with Workspace - - self.__workspace.AddFrame(self) - - # Associated context with this frame AND a group - - self.__context.SetFrame(self) - self.__context.SetGroup(self.__workspace.GetDefaultGroup()) - - # DB update timer - self.__update_timer = wx.Timer(self, wx.NewId()) - self.Bind(wx.EVT_TIMER, self.CheckDBUpdate) - - if update_enabled: - self.__EnableDBUpdates() - - def __del__(self) -> None: - for dlgs in self.__dialogs.values(): - for dlg in dlgs: - dlg.Close() - - def CheckDBUpdate(self, event: wx.TimerEvent) -> None: - if self.__context.dbhandle.api.isUpdateReady(): - self.OnDBUpdate() - self.__context.dbhandle.api.ackUpdate() - - def OnDBUpdate(self, show_wait_cursor: bool = True) -> None: - @contextmanager - def wait_cursor() -> Iterator[Union[wx.BusyCursor, nullcontext]]: - yield wx.BusyCursor() if show_wait_cursor else nullcontext() - - with wait_cursor(): - self.__playback_panel.Refresh() - self.__context.DBUpdate() - self.__canvas.FullUpdate() - # update dialog - watchdlgs = self.__dialogs.get('watchlist', []) - for wdlg in watchdlgs: - wdlg = cast('WatchListDlg', wdlg) - if wdlg.IsShown(): - wdlg.TickUpdate(self.__context.hc) - - wx.Frame.Refresh(self) - - def __EnableDBUpdates(self) -> None: - self.__update_timer.Start(self._DB_UPDATE_DELAY_MS) - - def SetPollMode(self, mode: bool) -> None: - if mode: - if not self.__update_timer.IsRunning(): - self.Bind(wx.EVT_TIMER, self.CheckDBUpdate) - self.__EnableDBUpdates() - self.__context.dbhandle.api.enableUpdate() - else: - if self.__update_timer.IsRunning(): - self.__update_timer.Stop() - self.__context.dbhandle.api.disableUpdate() - - def ForceDBUpdate(self) -> None: - with wx.BusyCursor(): - self.__context.dbhandle.api.forceUpdate() - self.OnDBUpdate(False) - - # Performs a full redraw - def Refresh( - self, - eraseBackground: bool = True, - rect: Optional[Union[Tuple[int, int, int, int], wx.Rect]] = None - ) -> None: - self.__canvas.FullUpdate() - self.__playback_panel.Refresh() - - # update dialog - watchdlgs = self.__dialogs.get('watchlist', []) - for wdlg in watchdlgs: - wdlg = cast('WatchListDlg', wdlg) - if wdlg.IsShown(): - wdlg.TickUpdate(self.__context.hc) - - wx.Frame.Refresh(self) - - # Returns the workspace owned by this frame - def GetWorkspace(self) -> Workspace: - return self.__workspace - - # Returns the Canvas owned by this Layout Frame - def GetCanvas(self) -> Layout_Canvas: - return self.__canvas - - def GetTitlePrefix(self) -> str: - return self.__title_prefix - - def GetTitleOverride(self) -> str: - return self.__title_override - - def GetTitleSuffix(self) -> str: - return self.__title_suffix - - # Returns the context contained by this Layout Frame - def GetContext(self) -> Layout_Context: - return self.__context - - # Returns the window title - def GetTitle(self) -> str: - return self.__title - - # Returns the Playback Panel owned by this Layout Frame - def GetPlaybackPanel(self) -> FramePlaybackBar: - return self.__playback_panel - - # Show a dialog. Shows existing dialog unless create_new=True. - # Forwards **kwargs to dialog_class - def ShowDialog(self, - name: str, - dialog_class: Type[DialogUnion], - create_new: bool = False, - **kwargs: Any) -> DialogUnion: - windows = self.__dialogs.setdefault(name, []) - if len(windows) == 0 or create_new is True: - dlg = dialog_class(self, **kwargs) - self.__dialogs[name].append(dlg) - else: - dlg = windows[-1] - dlg.Show() - dlg.SetFocus() - return dlg - - # Shows the location list dialog - # @note Location list does not disappear when close. It is just hidden - def ShowLocationsList(self) -> None: - self.ShowDialog('locations', - LocationWindow, - False, - elpropsdlg=self.__dlg) - - # Shows the search dialog - # @note Search does not disappear when close. It is just hidden - # @param kwargs Interperts and does not foward certain kwargs: - # \li location="starting search location" - def ShowSearch(self, *args: Any, **kwargs: Any) -> None: - if 'location' in kwargs: - loc = kwargs['location'] - del kwargs['location'] - else: - loc = None - dlg = cast(SearchDialog, self.ShowDialog('search', - SearchDialog, - True, - *args, - **kwargs)) - if loc is not None: - dlg.SetSearchLocation(loc) - - # Shows the find-element dialog - # @note Search does not disappear when close. It is just hidden - def ShowFindElement(self, *args: Any, **kwargs: Any) -> None: - self.ShowDialog('find element', - FindElementDialog, - False, - *args, - **kwargs) - - # Show or hide the navigation controls - def ShowNavigationControls(self, show: bool = True) -> None: - self.__playback_panel.Show(show) - self.Layout() - - # Attempt to select the given clock by name - def SetDisplayClock(self, - clock_name: str, - error_if_not_found: bool = True) -> bool: - return self.__playback_panel.SetDisplayClock(clock_name, - error_if_not_found) - - # To to a specific cyle on the currently displayed clock for this frame - def GoToCycle(self, cycle: int) -> None: - assert cycle is not None - self.__playback_panel.GoToCycle(cycle) - - # Handles shutting down both this window and the Element Properties Dialog - def Close(self, force: bool = False) -> bool: - self._HandleClose() - return True - - # 'Saving' means saving the Layout to file, nothing else is currently - # preserved about a session (no user preferences, current selection, HC) - # @return True if saved, False if cancelled (because it deferred to SaveAs) - def Save(self) -> bool: - logging.info('Saving') - assert self.__layout is not None - filename = self.__layout.GetFilename() - if not filename: - return self.SaveAs() - - if self.__layout.CanSaveToFile(): - self._SaveToFileWithErrorDlg() - return True - - message = \ - f'The file "{filename}" has been modified by another process (or '\ - 'a different layout instance) since being last written by this '\ - 'Layout. Do you want to overwrite these changes?' - dlg = wx.MessageDialog(self, - message, - "Save Layout - Overwrite Changed File?", - wx.YES_NO | wx.YES_DEFAULT | wx.ICON_QUESTION) - dlg.ShowModal() - ret = dlg.GetReturnCode() - dlg.Destroy() - if ret == wx.ID_NO: - return self.SaveAs() - - os.remove(filename) - self._SaveToFileWithErrorDlg() - - return True - - # Handle saving the file to a selected path - # @return True if saved, False if cancelled - def SaveAs(self) -> bool: - assert self.__layout is not None - fp = self.__layout.GetFilename() - logging.info('Saving As. Current="%s"', fp) - if fp is not None: - # directory - initial_file = os.path.dirname(os.path.abspath(fp)) + '/' - else: - initial_file = os.getcwd() + '/' - - # Loop until user saves or cancels - dlg = wx.FileDialog(self, - "Save As Argos layout file as", - defaultFile=initial_file, - wildcard=Layout.LAYOUT_FILE_WILDCARD, - style=(wx.FD_SAVE | - wx.FD_OVERWRITE_PROMPT | - wx.FD_CHANGE_DIR)) - dlg.ShowModal() - ret = dlg.GetReturnCode() - fp = dlg.GetPath() - dlg.Destroy() - - if ret == wx.ID_CANCEL: - return False # No save - - # NOTE: File guaranteed not to exisxt by FileDialog - self._SaveToFileWithErrorDlg(fp) - - self.UpdateTitle() # Title is based on layout filename - - return True - - def _SaveToFileWithErrorDlg(self, filename: Optional[str] = None) -> None: - assert self.__layout is not None - try: - self.__layout.SaveToFile(filename) - except Exception as ex: - if filename is None: - filename = self.__layout.GetFilename() - assert filename is not None - - msg = 'Failed to save layout to\n' \ - f'"{os.path.abspath(filename)}"\n\n' \ - f'{ex}' - - print(msg) - - dlg = wx.MessageDialog(self, - msg, - 'Save To File', - wx.OK) - dlg.ShowModal() - dlg.Destroy() - - # Determine if the user frame can be closed and gives the user a chance to - # save of discard or cancel if the layout was modified. This does not - # close the frame, but must be followed by a _HandleClose(force=True) call - # @note This exists so that all layouts can be tested for changes before - # closing any of them - # @return True if closing is allowed, False if not - def _PromptBeforeClose(self) -> bool: - assert self.__layout is not None - if not self.__layout.HasChanged(): - return True # Closing is OK - - while self.__canvas.HasCapture(): - self.__canvas.ReleaseMouse() - dlg = LayoutExitDialog(self) - ret = dlg.ShowModal() - dlg.Destroy() - if ret == wx.ID_CANCEL: - return False # Do not exit - if ret == wx.ID_SAVE: - # Attempt to save. Do not quit if user failed to save - return self.Save() - elif ret == wx.ID_DELETE: - return True # discard changes - else: - raise Exception(f'Unkonwn result from LayoutExitDialog: {ret}') - - # Handles a closing request from this frame or the menu bar. - # @param force (kwargs only, default False) Force closed without prompting. - # This is dangerous and should only be done when proceeded by a call to - # _PromptBeforeClose(). - # - # Prompts the user about actually quitting if the layout has changed. - # If successful, destroys - def _HandleClose(self, **kwargs: Any) -> None: - force = False - for k, v in list(kwargs.items()): - if k == 'force': - force = v - else: - raise KeyError(f'Unknown kwargs {k}') - - # Only prompt about closing if force is False - if force is False: - if not self._PromptBeforeClose(): - return - - # Stop emitting messages which delay the wx.CallAfter - self.__playback_panel.PausePlaying() - while self.__dialogs: - name, dlgs = self.__dialogs.popitem() - for dlg in dlgs: - dlg.Destroy() - - if self.__dlg is not None: - self.__dlg.Destroy() - self.__dlg = None - - self.__update_timer.Stop() - self.__context.LeaveGroup() - # Delay destruction to ensure that this handler does not refer to - # this window - wx.CallAfter(self.Destroy) - self.__workspace.RemoveFrame(self) - - # Handle resize events on this frame - def __OnResize(self, evt: wx.SizeEvent) -> None: - # Resize pauses playing because when timer events are too close - # together there is no chance to refresh the whole frame as is needed. - self.__playback_panel.PausePlaying() - - evt.Skip() - - # Computes a good window title containing the database and layout file - # information - def ComputeTitle(self) -> str: - title = os.path.split(self.__context.dbhandle.database.filename)[1] - title += ':' - layout = self.__context.GetLayout() - assert layout is not None - lf = layout.GetFilename() - if lf is not None: - title += os.path.split(lf)[1] - else: - title += "" - return title - - # Updates the current title based on ComputeTitle - def UpdateTitle(self) -> None: - self.SetTitle(self.ComputeTitle()) - - # Used for specifying edit mode - def SetEditMode(self, menuEditBool: bool) -> None: - self.GetCanvas().GetInputDecoder().SetEditMode( - menuEditBool, - self.__canvas.GetSelectionManager() - ) - self.__menu.SetEditModeSettings(menuEditBool) - self.__menu.ShowEditToolbar(menuEditBool) - # Need to send a resize event to draw the toolbar correctly in v2.x - self.SendSizeEvent() - # Update the mouse location in the edit bar - if menuEditBool and self.IsShown(): - self.UpdateMouseLocation(self.__canvas.GetMousePosition()) - - def SetHoverPreview(self, isHoverPreview: bool) -> None: - self.GetCanvas().GetHoverPreview().Enable(isHoverPreview) - - def SetHoverPreviewFields(self, fields: List[str]) -> None: - self.GetCanvas().GetHoverPreview().SetFields(fields) - - # Sets cursor to busy if True - def SetBusy(self, busy: bool) -> None: - if busy: - wx.BeginBusyCursor() - else: - wx.EndBusyCursor() - - # focuses the jump-to-time box in the playback panel - def FocusJumpBox(self) -> None: - self.__playback_panel.FocusJumpBox() - - # Updates the mouse location in the edit toolbar - def UpdateMouseLocation(self, - pos: Union[Tuple[int, int], wx.Point]) -> None: - self.__menu.UpdateMouseLocation(pos) - - def GetSettings(self) -> ArgosSettings: - return self.__workspace.GetSettings() - - def UpdateSettings(self, new_settings: Dict[str, Any]) -> None: - self.__workspace.UpdateSettings(new_settings) diff --git a/helios/pipeViewer/pipe_view/gui/selection_manager.py b/helios/pipeViewer/pipe_view/gui/selection_manager.py deleted file mode 100644 index 39ebe6bac9..0000000000 --- a/helios/pipeViewer/pipe_view/gui/selection_manager.py +++ /dev/null @@ -1,2054 +0,0 @@ -# Selection Mgrs are in charge of drawing to the canvas a demonstration of -# the current selection, hence importing wx -from __future__ import annotations -import os -import traceback -from typing import (Any, - Callable, - Dict, - List, - Optional, - Set, - Sequence, - Tuple, - Union, - cast, - TYPE_CHECKING) - -import wx - -from ..model.element import Element -from ..model.layout_delta import Checkpoint - -if TYPE_CHECKING: - from ..model.element_value import Element_Value - from ..model.layout import Layout - from .dialogs.element_propsdlg import Element_PropsDlg - from .layout_canvas import Layout_Canvas - - -# This class is responsible for keeping track of which Elements have been -# 'selected' by the user, within a given Layout, as viewed through a Layout -# Context. -class Selection_Mgr: - - # width of the selection handle boxes - __c_box_wid = 5 - - # Used for determining how far the mouse moved during a resize op. - __prev_x: Optional[int] = None - __prev_y: Optional[int] = None - - # For storing the proposed changes to the properties of elements in the - # selection. 'Queue' as in, 'this information is queued up, and it will - # all be accounted for at once' - __queue: Dict[Element, Tuple[int, int, int, int]] = {} - - # Max depth of the undo/redo stack - # This is really intended to prevent runaway allocations and should not be - # a burden during normal use. - MAX_UNDO_STACK_SIZE = 200 - - SNAP_ON_RESIZE = True - - # Possible sides of a selection/element. Used primarily for alignment - # states and tracking resize operations. Also used to describe the - # direction in which to operate, for actions like Indent() - RIGHT = 4 - BOTTOM = 3 - LEFT = 2 - TOP = 1 - NONE = 0 - - # Calculates the actual resize operation, relative to the data in the - # queue, and saves results to the queue - # @param pt The current mouse location - # @param base The element who owns the selection handle being dragged - def __TopResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - _, by, _, bh = self.__queue[base] - assert self.__prev_y is not None - delta = pt[1] - self.__prev_y - # Check that the mouse is moving in a direction at a location worth - # resizing, since due to snapping, the element may already be where - # the mouse is headed - if (delta < 0 and by > pt[1]) or (delta > 0 and by < pt[1]): - mh = self.__queue[self.__min_ht][3] - mh = mh - delta - if mh >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - h = h - delta - self.__queue[e] = (x, y + delta, w, h) - self.__prev_y = pt[1] - - # Calculates the actual resize operation, relative to the data in the - # queue, and saves results to the queue - def __LeftResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - bx, _, bw, bh = self.__queue[base] - assert self.__prev_x is not None - delta = pt[0] - self.__prev_x - if (delta < 0 and bx > pt[0]) or (delta > 0 and bx < pt[0]): - mw = self.__queue[self.__min_wid][2] - mw = mw - delta - if mw >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - w = w - delta - self.__queue[e] = (x + delta, y, w, h) - self.__prev_x = pt[0] - - # Calculates the actual resize operation, relative to the data in the - # queue, and saves results to the queue - def __RightResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - bx, _, bw, bh = self.__queue[base] - assert self.__prev_x is not None - delta = pt[0] - self.__prev_x - if (delta < 0 and (bx + bw) > pt[0]) or \ - (delta > 0 and (bx + bw) < pt[0]): - mw = self.__queue[self.__min_wid][2] - mw = mw + delta - if mw >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - w = w + delta - self.__queue[e] = (x, y, w, h) - self.__prev_x = pt[0] - - def __BottomResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - ''' - Calculates the actual resize operation, relative to the data in the - queue, and saves results to the queue - ''' - _, by, bw, bh = self.__queue[base] - assert self.__prev_y is not None - delta = pt[1] - self.__prev_y - if (delta < 0 and (by + bh) > pt[1]) or \ - (delta > 0 and (by + bh) < pt[1]): - mh = self.__queue[self.__min_ht][3] - mh = mh + delta - if mh >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - h = h + delta - self.__queue[e] = (x, y, w, h) - self.__prev_y = pt[1] - - # Forwarder method - def __BottomMidResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResize(pt, base) - self.SetProperties([self.BOTTOM]) - - # Forwarder method - def __TopMidResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResize(pt, base) - self.SetProperties([self.TOP]) - - # Forwarder method - def __LeftMidResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__LeftResize(pt, base) - self.SetProperties([self.LEFT]) - - # Forwarder method - def __RightMidResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__RightResize(pt, base) - self.SetProperties([self.RIGHT]) - - # Forwarder method - def __TopLeftResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResize(pt, base) - self.__LeftResize(pt, base) - self.SetProperties([self.TOP, self.LEFT]) - - # Forwarder method - def __TopRightResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResize(pt, base) - self.__RightResize(pt, base) - self.SetProperties([self.TOP, self.RIGHT]) - - # Forwarder method - def __BottomLeftResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResize(pt, base) - self.__LeftResize(pt, base) - self.SetProperties([self.BOTTOM, self.LEFT]) - - # Forwarder method - def __BottomRightResize(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResize(pt, base) - self.__RightResize(pt, base) - self.SetProperties([self.BOTTOM, self.RIGHT]) - - # Look at all the proposed data for Element properties stored in the - # __queue, and actually write them to their respective Elements, - # modifying them for snapping if applicable. This method is called at - # the end of all resize operations - # @param sides List of sides which are allowed to snap. - # Order doesn't matter - def SetProperties(self, sides: List[int]) -> None: - grid = self.__canvas.gridsize - canvasRange = self.__canvas.range - for e in self.__selected: - # This was determined to be the currently best place of rounding - # and typcasting, since Element properties can only be ints - (x, y, w, h) = (int(round(elem)) for elem in self.__queue[e]) - (nx, ny, nw, nh) = (x, y, w, h) - for side in sides: - if side == self.TOP: - if self.SNAP_ON_RESIZE: - dy = y % grid - # Check if the edge is within snapping canvasRange on - # either side of a gridline - if dy <= canvasRange: - ny = y - dy - nh = h + dy - elif dy >= grid - canvasRange: - ny = y - dy + grid - nh = h + dy - grid - # Verify that the element hasn't shrunk beyond - # allowable limits - if h == self.__c_box_wid * 2 or \ - nh <= self.__c_box_wid * 2: - nh = self.__c_box_wid * 2 - ny = y + h - nh - - elif side == self.LEFT: - if self.SNAP_ON_RESIZE: - dx = x % grid - if dx <= canvasRange: - nx = x - dx - nw = w + dx - elif dx >= grid - canvasRange: - nx = x - dx + grid - nw = w + dx - grid - if w == self.__c_box_wid * 2 or \ - nw <= self.__c_box_wid * 2: - nw = self.__c_box_wid * 2 - nx = x + w - nw - - elif side == self.RIGHT: - if self.SNAP_ON_RESIZE: - dw = (x + w) % grid - if dw <= canvasRange: - nw = w - dw - elif dw >= grid - canvasRange: - nw = w - dw + grid - if w == self.__c_box_wid * 2 or \ - nw <= self.__c_box_wid * 2: - nw = self.__c_box_wid * 2 - - elif side == self.BOTTOM: - if self.SNAP_ON_RESIZE: - dh = (y + h) % grid - if dh <= canvasRange: - nh = h - dh - elif dh >= grid - canvasRange: - nh = h - dh + grid - if h == self.__c_box_wid * 2 or \ - nh <= self.__c_box_wid * 2: - nh = self.__c_box_wid * 2 - # In case anything changed on this iteration, those changes - # are available to the next iteration - (x, y, w, h) = (nx, ny, nw, nh) - # All done verifying / snapping, go ahead and commit the new - # properties - e.SetProperty('position', (x, y)) - e.SetProperty('dimensions', (w, h)) - - # Delete all the temporary storage of Element properties - def FlushQueue(self) -> None: - self.__queue = {} - - # Original resize callbacks, these operations allow overlap - __RESIZE_OPTIONS_ONE = { - 0: __TopLeftResize, - 1: __BottomLeftResize, - 2: __TopRightResize, - 3: __BottomRightResize, - 4: __TopMidResize, - 5: __BottomMidResize, - 6: __LeftMidResize, - 7: __RightMidResize - } - - def __TopResizeScaling(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - ''' - Calculates the actual resize operation, relative to the queue, and - saves results to the queue. This is 'Scale' mode - @param pt The current mouse location - @param base The element who owns the selection handle being dragged - ''' - bx, by, bw, bh = self.__queue[base] - assert self.__prev_y is not None - delta = self.__prev_y - pt[1] - current = self.__bottom - self.__prev_y - # Check that the mouse is moving in a direction at a location worth - # resizing, since due to snapping, the element may already be where - # the mouse is headed - if ((-delta < 0 and by > pt[1]) or (-delta > 0 and by < pt[1])) and \ - current > 0: - factor = -(delta / current + 1.0) - mh = self.__queue[self.__min_ht][3] - mh = int(-mh * factor) - if mh >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - y = int((self.__bottom - y) * factor + self.__bottom) - h = int(-h * factor) - h = max(h, self.__c_box_wid * 2) - self.__queue[e] = (x, y, w, h) - self.__prev_y = pt[1] - - # Calculates the actual resize operation, relative to the queue, and - # saves results to the queue. This is 'Scale' mode - def __LeftResizeScaling(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - bx, by, bw, bh = self.__queue[base] - assert self.__prev_x is not None - delta = self.__prev_x - pt[0] - current = self.__right - self.__prev_x - if ((-delta < 0 and bx > pt[0]) or (-delta > 0 and bx < pt[0])) and \ - current > 0: - factor = -(delta / current + 1.0) - mw = self.__queue[self.__min_wid][2] - mw = int(-mw * factor) - if mw >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - x = int((self.__right - x) * factor + self.__right) - w = int(-w * factor) - w = max(w, self.__c_box_wid * 2) - self.__queue[e] = (x, y, w, h) - self.__prev_x = pt[0] - - # Calculates the actual resize operation, relative to the queue, and - # saves results to the queue. This is 'Scale' mode - def __RightResizeScaling(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - bx, by, bw, bh = self.__queue[base] - assert self.__prev_x is not None - delta = pt[0] - self.__prev_x - current = self.__prev_x - self.__left - if ((delta < 0 and (bx + bw) > pt[0]) or - (delta > 0 and (bx + bw) < pt[0])) and \ - current > 0: - factor = 1.0 + delta / current - mw = self.__queue[self.__min_wid][2] - mw = int(mw * factor) - if mw >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - x = int((x - self.__left) * factor + self.__left) - w = int(w * factor) - w = max(w, self.__c_box_wid * 2) - self.__queue[e] = (x, y, w, h) - self.__prev_x = pt[0] - - # Calculates the actual resize operation, relative to the queue, and - # saves results to the queue. This is 'Scale' mode - def __BottomResizeScaling(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - bx, by, bw, bh = self.__queue[base] - assert self.__prev_y is not None - delta = pt[1] - self.__prev_y - current = self.__prev_y - self.__top - if ((delta < 0 and (by + bh) > pt[1]) or - (delta > 0 and (by + bh) < pt[1])) and \ - current != 0: - if current > 0: - factor = 1.0 + delta / current - else: - factor = 1.0 - delta / current - mh = self.__queue[self.__min_ht][3] - mh = int(mh * factor) - if mh >= self.__c_box_wid * 2: - for e in self.__selected: - x, y, w, h = self.__queue[e] - h = int(h * factor) - y = int((y - self.__top) * factor + self.__top) - h = max(h, self.__c_box_wid * 2) - self.__queue[e] = (x, y, w, h) - self.__prev_y = pt[1] - - # Forwarder method - def __BottomMidResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResizeScaling(pt, base) - self.SetProperties([self.BOTTOM, self.TOP]) - - # Forwarder method - def __TopMidResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResizeScaling(pt, base) - self.SetProperties([self.TOP, self.BOTTOM]) - - # Forwarder method - def __LeftMidResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__LeftResizeScaling(pt, base) - self.SetProperties([self.LEFT, self.RIGHT]) - - # Forwarder method - def __RightMidResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__RightResizeScaling(pt, base) - self.SetProperties([self.RIGHT, self.LEFT]) - - # Forwarder method - def __TopLeftResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResizeScaling(pt, base) - self.__LeftResizeScaling(pt, base) - self.SetProperties([self.TOP, self.LEFT, self.RIGHT, self.BOTTOM]) - - # Forwarder method - def __TopRightResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__TopResizeScaling(pt, base) - self.__RightResizeScaling(pt, base) - self.SetProperties([self.TOP, self.RIGHT, self.LEFT, self.BOTTOM]) - - # Forwarder method - def __BottomLeftResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResizeScaling(pt, base) - self.__LeftResizeScaling(pt, base) - self.SetProperties([self.BOTTOM, self.LEFT, self.RIGHT, self.TOP]) - - # Forwarder method - def __BottomRightResize2(self, - pt: Union[wx.Point, Tuple[int, int]], - base: Element) -> None: - self.__BottomResizeScaling(pt, base) - self.__RightResizeScaling(pt, base) - self.SetProperties([self.BOTTOM, self.RIGHT, self.TOP, self.LEFT]) - - # This is 'Scaling'-mode resize op callbacks - __RESIZE_OPTIONS_TWO = { - 0: __TopLeftResize2, - 1: __BottomLeftResize2, - 2: __TopRightResize2, - 3: __BottomRightResize2, - 4: __TopMidResize2, - 5: __BottomMidResize2, - 6: __LeftMidResize2, - 7: __RightMidResize2 - } - - # Prepare for a resize operation on the named side. Get the bounding box - # of the selection, find the smallest Element in the corresponding - # dimension. Called on MouseDown before the operation actually takes place - def __TopMidResizePrep(self, base: Element) -> None: - bh = base.GetYDim() - by = base.GetYPos() - self.__top = by - self.__bottom = by + bh - self.__min_ht = base - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__queue[e] = (x, y, w, h) - if h < bh: - bh = h - self.__min_ht = e - if y < self.__top: - self.__top = y - if y + h > self.__bottom: - self.__bottom = y + h - - # Prepare for a resize operation on the named side. Get the bounding box - # of the selection, find the smallest Element in the corresponding - # dimension. Called on MouseDown before the operation actually takes place - def __LeftMidResizePrep(self, base: Element) -> None: - bw = base.GetXDim() - bx = base.GetXPos() - self.__min_wid = base - self.__left = bx - self.__right = bx + bw - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__queue[e] = (x, y, w, h) - if w < bw: - bw = w - self.__min_wid = e - if x < self.__left: - self.__left = x - if x + w > self.__right: - self.__right = x + w - - # Prepare for a resize operation on the named side. Get the bounding box - # of the selection, find the smallest Element in the corresponding - # dimension. Called on MouseDown before the operation actually takes place - def __RightMidResizePrep(self, base: Element) -> None: - bw = base.GetXDim() - bx = base.GetXPos() - self.__min_wid = base - self.__left = bx - self.__right = bx + bw - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__queue[e] = (x, y, w, h) - if w < bw: - bw = w - self.__min_wid = e - if x < self.__left: - self.__left = x - if x + w > self.__right: - self.__right = x + w - - # Prepare for a resize operation on the named side. Get the bounding box - # of the selection, find the smallest Element in the corresponding - # dimension. Called on MouseDown before the operation actually takes place - def __BottomMidResizePrep(self, base: Element) -> None: - bh = base.GetYDim() - by = base.GetYPos() - self.__top = by - self.__bottom = by + bh - self.__min_ht = base - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__queue[e] = (x, y, w, h) - if h < bh: - bh = h - self.__min_ht = e - if y < self.__top: - self.__top = y - if y + h > self.__bottom: - self.__bottom = y + h - - # Forwarder method - def __TopLeftResizePrep(self, base: Element) -> None: - self.__TopMidResizePrep(base) - self.__LeftMidResizePrep(base) - - # Forwarder method - def __TopRightResizePrep(self, base: Element) -> None: - self.__TopMidResizePrep(base) - self.__RightMidResizePrep(base) - - # Forwarder method - def __BottomRightResizePrep(self, base: Element) -> None: - self.__BottomMidResizePrep(base) - self.__RightMidResizePrep(base) - - # Forwarder method - def __BottomLeftResizePrep(self, base: Element) -> None: - self.__LeftMidResizePrep(base) - self.__BottomMidResizePrep(base) - - # Callbacks for pre-resize op preparations - __RESIZE_PREPARATIONS = { - 0: __TopLeftResizePrep, - 1: __BottomLeftResizePrep, - 2: __TopRightResizePrep, - 3: __BottomRightResizePrep, - 4: __TopMidResizePrep, - 5: __BottomMidResizePrep, - 6: __LeftMidResizePrep, - 7: __RightMidResizePrep - } - - # It is worth noting that as of right now (8/8/2012) wx.CURSOR_SIZENESW & - # wx.CURSOR_SIZENWSE don't work in this environment, so options 0-3 are - # less-than-ideal standins, till someone custom creates bitmaps for cursors - __CURSOR_OPTIONS = { - 0: wx.CURSOR_SIZENWSE, - 1: wx.CURSOR_SIZENESW, - 2: wx.CURSOR_SIZENESW, - 3: wx.CURSOR_SIZENWSE, - 4: wx.CURSOR_SIZENS, - 5: wx.CURSOR_SIZENS, - 6: wx.CURSOR_SIZEWE, - 7: wx.CURSOR_SIZEWE, - } - - # Available modes of rubber-band or lasso selecting - ADD_TO_SELECTION = 1 - REMOVE_FROM_SELECTION = 2 - - # The available history types for tracking. These are all positive numbers - # so that boolean logic returns 'True' for these values. Mostly set, read, - # and used by Input Decoder, for mouse operations - CLICKED_ON_WHITESPACE = 1 - CLICKED_ON_ELEMENT = 2 - RUBBER_BAND_BOX = 3 - PREP_REMOVE = 4 - DRAGGED = 5 - RESIZE = 6 - - # Available modes of snapping a selection during a move operation - DOMINANT_ELEMENT = True - EACH_ELEMENT_SELECTED = False - NO_SNAP_ON_MOVE = False - - # Get prepared to start tracking and adjusting user-selected Elements - # @param canvas Layout_Canvas - # @param elpropsdlg Elements Properties Dialog - def __init__(self, - canvas: Layout_Canvas, - elpropsdlg: Element_PropsDlg) -> None: - self.__canvas = canvas - self.__el_props_dlg = elpropsdlg - # The Elements will be stored as the keys to a dict, where the values - # corresponding to each Element are tuples indicating the offset - # between a MouseEvt & the Element's top left corner - self.__selected: Dict[Element, Tuple[int, int]] = {} - self.__playback_selected: Optional[Element_Value] = None - self.__is_edit_mode = False - self.__dominant: Optional[Element] = None - self.__position = (0, 0) - self.__history = -1 - self.__resize_direction: Optional[Tuple[Element, int]] = None - self.__default_cursor = canvas.GetCursor() - self.__handle_color = (20, 220, 220) - self.__playback_handle_color = (0, 0, 0) - self.__alignment = self.NONE - self.__edge: Optional[int] = None - # This list is used to make sure that only Elements which were not - # previously selected, but have been selected since the user began a - # ctrl-clk&drg rubber-banding operation, are added to the selection - # during/after the rubber-banding operation - self.__temp_rubber: List[Element] = [] - # holds the coords of the corner to draw a rubber-band box (should - # correspond to the mouse position) - self.__rubberendpt: Optional[Tuple[int, int]] = None - self.__which_rubber_band = 0 - self.__rubber_color = (20, 220, 220) - - # used to prevent insane quantities of rapid fire copied elements - # being added to the layout - self.__copy_completed = False - - # undo/redo stack - self.__undo_stack: List[Checkpoint] = [] - # Position of next undo checkpoint to allocate - self.__undo_next_ptr = 0 - - # Current move action checkpoint. To be closed after the move - self.__cur_open_chkpt: Optional[Checkpoint] = None - - # Hooks for notifying clients about undo/redo. - self.__undo_redo_hooks: List[Callable] = [] - - resources_dir = os.path.join( - os.path.dirname( - os.path.dirname(os.path.join(os.getcwd(), __file__)) - ), - 'resources' - ) - nesw_image = wx.Image(os.path.join(resources_dir, "resize_nesw.png"), - wx.BITMAP_TYPE_PNG) - nesw_image.SetOption(wx.IMAGE_OPTION_CUR_HOTSPOT_X, 6) - nesw_image.SetOption(wx.IMAGE_OPTION_CUR_HOTSPOT_Y, 6) - self.__custom_cursor_nesw = wx.Cursor(nesw_image) - nwse_image = wx.Image(os.path.join(resources_dir, "resize_nwse.png"), - wx.BITMAP_TYPE_PNG) - nwse_image.SetOption(wx.IMAGE_OPTION_CUR_HOTSPOT_X, 6) - nwse_image.SetOption(wx.IMAGE_OPTION_CUR_HOTSPOT_Y, 6) - self.__custom_cursor_nwse = wx.Cursor(nwse_image) - - # Open and return a checkpoint. The returned checkpoint must be 'after'ed - # to complete the delta so that it can be used as an undo/redo delta. - # Appends new delta to the undo stack. Removes everything at and after the - # current next undo pointer from the undo stack because it cannot be - # redone following this new change. Maybe an undo/redo tree could be - # built, but that is probably unnecessary - # @param description Describes the checkpoint - # @param force_elements Elements to use as the selection - # (if not self.__selected.keys()) - def __Checkpoint( - self, - description: str, - force_elements: Optional[List[Element]] = None - ) -> Checkpoint: - assert self.__cur_open_chkpt is None, \ - 'Cannot start a undo/redo checkpoint before the current move ' \ - 'checkpoint is closed' - - if force_elements is None: - selection = list(self.__selected.keys()) - else: - selection = force_elements - - layout = self.__canvas.context.GetLayout() - assert layout is not None - chkpt = Checkpoint(layout, selection, description) - - # Drop the first element if stack is at maximum size - if len(self.__undo_stack) == self.MAX_UNDO_STACK_SIZE: - self.__undo_stack = self.__undo_stack[1:] - if self.__undo_next_ptr > 0: - self.__undo_next_ptr -= 1 - - self.__undo_stack = self.__undo_stack[:self.__undo_next_ptr] - self.__undo_stack.append(chkpt) - self.__undo_next_ptr += 1 - assert self.__undo_next_ptr == len(self.__undo_stack), \ - 'Checkpoint undo stack next ' \ - f'({self.__undo_next_ptr}) != size ({len(self.__undo_stack)})' - return chkpt - - # Add an undo/redo hook to be called when an undo/redo has taken place or - # the undo/redo stack has been modified. This can be used to update the - # menu or undo/redo lists - # @param hook nullary function - # @note Hooks should check NumUndos/NumRedos - def AddUndoRedoHook(self, hook: Callable) -> None: - self.__undo_redo_hooks.append(hook) - - # Removes a undo/redo hook installed by AddUndoRedoHook - def RemoveUndoRedoHook(self, hook: Callable) -> None: - self.__undo_redo_hooks.remove(hook) - - # Invoke all undo/redo hooks installed by AddUndoRedoHook - def __CallUndoRedoHooks(self) -> None: - for hook in self.__undo_redo_hooks: - try: - hook() - except Exception as ex: - print('Exception in Undo/Redo hook:', ex) - - # Undo the next delta - # @return True if checkpoint was undone - def Undo(self) -> None: - if self.__undo_next_ptr == 0: - return - - chkpt = self.__undo_stack[self.__undo_next_ptr - 1] - errors, current = chkpt.remove() - if errors > 0: - print('Undo errors: ', errors) - self.__undo_next_ptr -= 1 - self.ClearSelection() - # Adds current elements after removing checkpoint and updates - # selection/elpropsdlg - self.Add(current) - - self.__canvas.context.GoToHC() - self.__canvas.Refresh() - - self.__CallUndoRedoHooks() - - # Redo the next delta - # @return True if checkpoint was undone - def Redo(self) -> None: - if self.__undo_next_ptr >= len(self.__undo_stack): - return - - chkpt = self.__undo_stack[self.__undo_next_ptr] - errors, current = chkpt.apply() - if errors > 0: - print('Redo errors: ', errors) - self.__undo_next_ptr += 1 - self.ClearSelection() - if current is not None: - # Adds current elements after applying checkpoint and updates - # selection/elpropsdlg - self.Add(current) - - self.__canvas.context.GoToHC() - self.__canvas.Refresh() - - self.__CallUndoRedoHooks() - - # Redo all deltas - # @return True if checkpoint was undone - def RedoAll(self) -> None: - current = None - while self.__undo_next_ptr < len(self.__undo_stack): - chkpt = self.__undo_stack[self.__undo_next_ptr] - errors, current = chkpt.apply() - if errors > 0: - print('Redo errors: ', errors) - self.__undo_next_ptr += 1 - - self.ClearSelection() - - # implies something was redone - if current is not None: - # Adds current elements after applying checkpoint and updates - # selection/elpropsdlg - self.Add(current) - - self.__canvas.context.GoToHC() - self.__canvas.Refresh() - - self.__CallUndoRedoHooks() - - # Number of undos remaining before the reaching the bottom of the stack - def NumUndos(self) -> int: - return self.__undo_next_ptr - - # Gets the description of the next undo. "" If NumUndos() is 0 - def GetNextUndoDesc(self) -> str: - if self.NumUndos() == 0: - return '' - return self.__undo_stack[self.__undo_next_ptr - 1].description - - # Number of redos remainign before reaching the top of the stack - def NumRedos(self) -> int: - return len(self.__undo_stack) - self.__undo_next_ptr - - # Gets the description of the next redo. "" If NumRedos() is 0 - def GetNextRedoDesc(self) -> str: - if self.NumRedos() == 0: - return '' - return self.__undo_stack[self.__undo_next_ptr].description - - # Used to prevent insane quantities of rapid fire copied elements - # being added to the layout - def PrepNextCopy(self) -> None: - self.__copy_completed = False - - @property - def each(self) -> bool: - return self.EACH_ELEMENT_SELECTED - - @property - def dominant(self) -> bool: - return self.DOMINANT_ELEMENT - - @property - def add(self) -> int: - return self.ADD_TO_SELECTION - - @property - def remove(self) -> int: - return self.REMOVE_FROM_SELECTION - - @property - def dragged(self) -> int: - return self.DRAGGED - - @property - def prep_remove(self) -> int: - return self.PREP_REMOVE - - @property - def collision(self) -> int: - return self.CLICKED_ON_ELEMENT - - @property - def whitespace(self) -> int: - return self.CLICKED_ON_WHITESPACE - - @property - def rubber_band(self) -> int: - return self.RUBBER_BAND_BOX - - @property - def resize(self) -> int: - return self.RESIZE - - # Clears the current selection - def ClearSelection(self) -> None: - self.SetSelection({}) - - # Used for forcing the current selection, instantly disregarding what - # was stored previously - # @param selected Dictionary of elements {element:[rel_x,rel_y]}. The - # values in this dict are implementation details so are not explained here - # @todo This is really an internal-only method and should be made hidden - def SetSelection(self, selected: Dict[Element, Tuple[int, int]]) -> None: - assert isinstance(selected, dict) - self.__selected = selected - self.__temp_rubber = [] - self.__el_props_dlg.SetElements(list(self.__selected.keys()), self) - self.__canvas.Refresh() - - # Sets selected object to supplied pair - def SetPlaybackSelected(self, pair: Optional[Element_Value]) -> None: - self.__playback_selected = pair - - # Returns selected object pair (None if none selected) - def GetPlaybackSelected(self) -> Optional[Element_Value]: - return self.__playback_selected - - # Return the entire selection as a sequence - def GetSelection(self) -> Dict[Element, Tuple[int, int]]: - return self.__selected - - # Toggle whether or not an Element is selected. Used for inverting - # selection - # @param e Element or sequence of elements to toggle - def Toggle(self, e: Union[Element, Sequence[Element]]) -> None: - if isinstance(e, Element): - els = [e] - else: - els = list(e) - - to_remove = [e for e in els if self.IsSelected(e)] - to_add = [e for e in els if not self.IsSelected(e)] - - self.Remove(to_remove) - self.Add(to_add) - - # Sets whether selection manager should be in edit mode or playback mode - def SetEditMode(self, is_edit_mode: bool) -> None: - self.__is_edit_mode = is_edit_mode - - # Set which Element is the 'dominant' one for the selection. i.e. which - # is directly beneath the mouse, used for a snapping mode during move - # operations - def SetDominant(self, e: Element) -> None: - self.__dominant = e - - # Used for specifiying which mode of snapping during resize operations - def SetSnapMode(self, mode: str) -> None: - # Move operation snap modes - if mode == 'mdominant': - self.DOMINANT_ELEMENT = True - self.EACH_ELEMENT_SELECTED = False - self.NO_SNAP_ON_MOVE = False - elif mode == 'meach': - self.DOMINANT_ELEMENT = False - self.EACH_ELEMENT_SELECTED = True - self.NO_SNAP_ON_MOVE = False - elif mode == 'freemove': - self.DOMINANT_ELEMENT = False - self.EACH_ELEMENT_SELECTED = False - self.NO_SNAP_ON_MOVE = True - # Resize operation snap modes - elif mode == 'reach': - self.SNAP_ON_RESIZE = True - elif mode == 'rdominant': - pass # not implemented - elif mode == 'freesize': - self.SNAP_ON_RESIZE = False - - # Snap each side of each selected Element to the grid, if able - def SnapFill(self) -> None: - self.FlushQueue() - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__queue[e] = (x, y, w, h) - self.SetProperties([self.TOP, self.BOTTOM, self.RIGHT, self.LEFT]) - self.__canvas.FullUpdate() - - # Snap the closest corner of each selected Element to the grid, if able - def SnapCorner(self) -> None: - self.FlushQueue() - grid = self.__canvas.gridsize - - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - # Calculate how far away both edges are from the grid in either - # direction - dx = x % grid - dx1 = grid - dx - dw = (x + w) % grid - dw1 = grid - dw - # each tuple is the absolute displacement from a gridline, and the - # direction to the gridline - horiz = [(dx, -1), (dx1, 1), (dw, -1), (dw1, 1)] - - # used so that the horiz/vert lists can be sorted by the first - # value of each tuple - def extract(tuple: Tuple[int, int]) -> int: - return tuple[0] - - temp = sorted(horiz, key=extract)[0] - # whichever edge is closest gets snapped - x = x + temp[0] * temp[1] - dy = y % grid - dy1 = grid - dy - dh = (y + h) % grid - dh1 = grid - dh - vert = [(dy, -1), (dy1, 1), (dh, -1), (dh1, 1)] - temp = sorted(vert, key=extract)[0] - y = y + temp[0] * temp[1] - e.SetProperty('position', (x, y)) - self.__canvas.FullUpdate() - - # Add an Element (or overwrite the associated tuple) - # @param e Element or sequence of elements to append to selection - def Add(self, element: Union[Element, Sequence[Element]]) -> None: - if isinstance(element, Element): - els = [element] - else: - els = list(element) - for e in els: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__selected[e] = (x - self.__position[0], - y - self.__position[1]) - - # Continue aligning, if the selection used to be aligned, and the - # new guy is on the same edge - if self.__alignment == self.LEFT: - if not (x == self.__edge): - self.__alignment = self.NONE - if self.__alignment == self.TOP: - if not (y == self.__edge): - self.__alignment = self.NONE - if self.__alignment == self.RIGHT: - if not (x + w == self.__edge): - self.__alignment = self.NONE - if self.__alignment == self.BOTTOM: - if not (y + h == self.__edge): - self.__alignment = self.NONE - self.__el_props_dlg.SetElements(list(self.__selected.keys()), self) - self.__canvas.Refresh() - - def IsSelected(self, e: Element) -> bool: - ''' - Checks whether or not an Element is currently in the selection - ''' - if (e is not None) and e in self.__selected: - return True - return False - - def Remove( - self, - e: Union[Element, Tuple[Element, ...], Sequence[Element]] - ) -> None: - ''' - Remove an Element from the selection, if present - @param e Element to remove or sequence of elements - ''' - if isinstance(e, Element): - els = [e] - else: - els = list(e) - - for e in els: - if self.IsSelected(e): - del self.__selected[e] - - self.__el_props_dlg.SetElements(list(self.__selected.keys()), self) - self.__canvas.Refresh() - - def Delete(self, layout: Layout) -> None: - ''' - Permanently remove all Elements in the selection from the specified - layout - ''' - to_delete = self.__selected - if to_delete: - self.ClearSelection() - self.Add(list(to_delete.keys())) - self.BeginCheckpoint('delete element') - for e in self.__selected: - e.EnableDraw(False) - layout.RemoveElement(e) - self.Clear() - self.CommitCheckpoint() # Nothing selected after checkpoint - - # Generate a brand new Element in the layout, select it - # @param add_to_selection Should the new element be added to the - # existent selection. If False, element becomes new selection - # @return Returns the newly-created element - def GenerateElement(self, - layout: Layout, - type_string: str, - add_to_selection: bool = False) -> Element: - self.BeginCheckpoint('create element') - e = layout.CreateAndAddElement(element_type=type_string) - mouse: Union[wx.Point, Tuple[int, int]] = wx.GetMousePosition() - mouse = self.__canvas.CalcUnscrolledPosition(mouse) - screen = self.__canvas.GetScreenPosition() - rel_pt = (mouse[0] - screen[0], mouse[1] - screen[1]) - self.SetPos(rel_pt) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - e.SetProperty('position', (int(rel_pt[0] - w / 2), - int(rel_pt[1] - h / 2))) - - # Add to selection, clearing unless asked not to - if add_to_selection is not True: - self.Clear() - self.Add(e) - self.__canvas.GetFrame().Refresh() - self.CommitCheckpoint() - return e - - # Generate a complete copy of the currently selected Elements in the - # layout, offset by delta. This operation is prevented from happening - # continuously - # @param layout Layout to modify - # @param delta Single integer specifying both the x and y coordinate - # adjustment of the new element - def GenerateDuplicateSelection(self, layout: Layout, delta: int) -> None: - if not self.__copy_completed: - self.BeginCheckpoint('duplicate element') - temp_buffer = [] - original_selection = list(self.__selected.keys()) - for e in original_selection: - ce = layout.CreateAndAddElement( - e, - element_type=cast(str, e.GetProperty('type')) - ) - x, y = cast(Tuple[int, int], ce.GetProperty('position')) - ce.SetProperty('position', (x + delta, y + delta)) - temp_buffer.append(ce) - - self.Clear() - self.Add(temp_buffer) - # after all the Elements are added, get their Element Values pairs - # populated, then refresh the entire frame and prevent an - # immediately consecutive duplication operation - self.__canvas.context.GoToHC() - self.__canvas.GetFrame().Refresh() - # Requires that PrepNextCopy be called before this will work again - self.__copy_completed = True - - # Committed checkpoint must know about all elements that still - # exist from the selection with which the checkpoint was created - self.CommitCheckpoint( - force_elements=temp_buffer + original_selection - ) - - # Toggles the selection state of each element on the canvas - def InvertSelection(self, layout: Layout) -> None: - # Mouse info needed for accurate offsets to store in self.__selected - # when self.Add() is eventually called - mouse: Union[wx.Point, Tuple[int, int]] = wx.GetMousePosition() - mouse = self.__canvas.CalcUnscrolledPosition(mouse) - screen = self.__canvas.GetScreenPosition() - rel_pt = (mouse[0] - screen[0], mouse[1] - screen[1]) - self.SetPos(rel_pt) - self.Toggle(self.__canvas.GetElements()) - self.__canvas.FullUpdate() - - # Empty the current selection. Does not modify the layout. - # @note No current implementation for 'undo' - def Clear(self) -> None: - self.__selected = {} - self.__el_props_dlg.SetElements(list(self.__selected.keys()), self) - self.__canvas.Refresh() - - # Selects every Element on the Canvas/in the Layout. Ctrl-A - def SelectEntireLayout(self) -> None: - mouse: Union[wx.Point, Tuple[int, int]] = wx.GetMousePosition() - mouse = self.__canvas.CalcUnscrolledPosition(mouse) - screen = self.__canvas.GetScreenPosition() - rel_pt = (mouse[0] - screen[0], mouse[1] - screen[1]) - self.SetPos(rel_pt) - self.Add(self.__canvas.GetElements()) - if self.DetectCollision(rel_pt): - self.SetCursor(wx.Cursor(wx.CURSOR_SIZING)) - self.__canvas.FullUpdate() - - # Begin checkpoint before a move - # @note This is separate from Move. It must be explicitly called to ensure - # that there are clear stats and ends to undo/redo checkpoint creation. - # Implicitly calling this may hide bugs. - # @param force_elements (kwarg only) Begins using a specific set of - # elements. These elements will be treated as if they were selected - # @pre self.__selected must contain all elements which will be affected by - # this checkpoint OR force_elements must be used to override. This - # obviously excludes any elements created as a result of the checkpoint - def BeginCheckpoint(self, description: str, **kwargs: Any) -> None: - assert self.__cur_open_chkpt is None, \ - 'Should not enter BeginCheckpoint with an open checkpoint' - self.__cur_open_chkpt = self.__Checkpoint(description, - kwargs.get('force_elements')) - - # Complete checkpoint after a move - # @pre self.__selected must contain all elements which changed as a result - # of this checkpoint OR force_elements must be used to override - # @param force_elements (kwarg only) Begins using a specific set of - # elements. These elements will be treated as if they were selected - def CommitCheckpoint(self, **kwargs: Any) -> None: - if self.__cur_open_chkpt is None: - print('Should not enter CommitCheckpoint without an open checkpoint. This is a bug:') # noqa: E501 - print(traceback.format_stack()) - return - - chkpt = self.__cur_open_chkpt - self.__cur_open_chkpt = None - selected = kwargs.get('force_elements', self.__selected) - chkpt.after(selected) - self.__CallUndoRedoHooks() - - # Cancel the open checkpoint. - # @note does not require an open checkpoint. Has no effect if there is no - # open checkpoint - def CancelCheckpoint(self) -> None: - self.__cur_open_chkpt = None - - # Is there a checkpoint currently open (between BeginCheckpoint and - # CommitCheckpoint calls) - def HasOpenCheckpoint(self) -> bool: - return self.__cur_open_chkpt is not None - - # Translate the entire selection - # @param pos move to given position, usually a Mouse position - # @param delta move from current position by the specified offset - def Move(self, - pos: Optional[Union[wx.Point, Tuple[int, int]]] = None, - delta: Optional[Tuple[int, int]] = None, - **kwargs: Any) -> None: - force_no_snap = False - for k, v in kwargs.items(): - if k == 'force_no_snap': - force_no_snap = v - else: - raise KeyError(f'Invalid argument to Move: {k}={v}') - - grid = self.__canvas.gridsize - range = self.__canvas.range - # handle the move operation different dependent on the snap-mode - if (pos is not None): - if force_no_snap or self.NO_SNAP_ON_MOVE: - for e in self.__selected: - x, y = self.__selected[e] - x1 = x + pos[0] - y1 = y + pos[1] - e.SetProperty('position', (int(x1), int(y1))) - elif self.EACH_ELEMENT_SELECTED: - for e in self.__selected: - x, y = self.__selected[e] - x1 = x + pos[0] - y1 = y + pos[1] - dx = x1 % grid - dy = y1 % grid - if dx <= range: - x1 = x1 - dx - elif dx >= grid - range: - x1 = x1 - dx + grid - if dy <= range: - y1 = y1 - dy - elif dy >= grid - range: - y1 = y1 - dy + grid - e.SetProperty('position', (int(x1), int(y1))) - elif self.DOMINANT_ELEMENT: - assert self.__dominant is not None - x, y = self.__selected[self.__dominant] - x1 = x + pos[0] - y1 = y + pos[1] - dx = x1 % grid - dy = y1 % grid - if dx <= range: - x1 = x1 - dx - elif dx >= grid - range: - x1 = x1 - dx + grid - if dy <= range: - y1 = y1 - dy - elif dy >= grid - range: - y1 = y1 - dy + grid - pos = (x1 - x, y1 - y) - for e in self.__selected: - x, y = self.__selected[e] - x1 = x + pos[0] - y1 = y + pos[1] - e.SetProperty('position', (int(x1), int(y1))) - else: - raise ValueError('Move called with no snap types set and force_no_snap was not given as kwarg') # noqa: E501 - elif delta is not None: - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - e.SetProperty('position', (int(x + delta[0]), - int(y + delta[1]))) - self.SetHistory(self.whitespace) - else: - raise ValueError('Move called with no position and no delta') - - self.__canvas.FullUpdate() - - # Add spacing between the selected Elements - # @param base The element to remain stationary - # @param subtractive When true the operation removes spacing - def Indent(self, base: int, subtractive: bool = False) -> None: - if not self.__selected: - return - - self.BeginCheckpoint('indent elements') - # Are we adding s pace or removing it? - if subtractive: - mod = -1 - else: - mod = 1 - exes, x_sorted, whys, y_sorted, plus_w, w_sorted, plus_h, h_sorted = self.SortByPosition() # noqa: E501 - - if base == self.TOP or base == self.BOTTOM: - # add space in the opposite direction, and work the list backwards - if base == self.BOTTOM: - whys.reverse() - mod = -mod - idx = 0 - # Cannot subtract if they are already aligned - if len(whys) == 1 and not subtractive: - y = whys[0] - for x in exes: - for e in x_sorted[x]: - e.SetY(y + idx) - idx = idx + mod - # The most normal case - else: - for y in whys: - for e in y_sorted[y]: - e.SetY(y + idx) - idx = idx + mod - # Same math and logic as vertical operations - elif base == self.RIGHT or base == self.LEFT: - if base == self.RIGHT: - exes.reverse() - mod = -mod - idx = 0 - if len(exes) == 1 and not subtractive: - x = exes[0] - for y in whys: - for e in y_sorted[y]: - e.SetX(x + idx) - idx = idx + mod - else: - for x in exes: - for e in x_sorted[x]: - e.SetX(x + idx) - idx = idx + mod - self.__canvas.Refresh() - self.CommitCheckpoint() - - # Flip/Reverse the selection in the given direction - # @param self.TOP & self.BOTTOM behave identically, same with RIGHT & LEFT - def Flip(self, direction: int) -> None: - if not self.__selected: - return - - self.BeginCheckpoint('flip elements') - exes, x_sorted, whys, y_sorted, plus_w, w_sorted, plus_h, h_sorted = self.SortByPosition() # noqa: E501 - if direction == self.TOP or direction == self.BOTTOM: - for y in whys: - for e in y_sorted[y]: - e.SetY(plus_h[-1] - y + whys[0] - h_sorted[e]) - elif direction == self.RIGHT or direction == self.LEFT: - for x in exes: - for e in x_sorted[x]: - e.SetX(plus_w[-1] - x + exes[0] - w_sorted[e]) - self.__canvas.Refresh() - self.CommitCheckpoint() - - def MoveToTop(self) -> None: - if self.__selected: - drawables = [e for e in self.__selected if e.IsDrawable()] - - self.BeginCheckpoint('move to top') - try: - self.__canvas.context.MoveElementsAbovePINs(drawables, [None]) - except Exception: - # TODO: other places should follow this protocol - self.CancelCheckpoint() - raise - else: - self.CommitCheckpoint() - self.__canvas.FullUpdate() - - def MoveToBottom(self) -> None: - if not self.__selected: - return - - drawables = [e for e in self.__selected if e.IsDrawable()] - - self.BeginCheckpoint('move to bottom') - try: - self.__canvas.context.MoveElementsBelowPINs(drawables, [-1]) - except Exception: - # TODO: other places should follow this protocol - self.CancelCheckpoint() - raise - else: - self.CommitCheckpoint() - - self.__canvas.FullUpdate() - - def MoveUp(self) -> None: - if not self.__selected: - return - - drawables = [e for e in self.__selected if e.IsDrawable()] - drawable_pins = set([e.GetPIN() for e in drawables]) - - # All pairs - regardless of bounds. - # Note that no vis-tick filtering is needed because bounds=None - draw_pairs = [ - p.GetElement() - for p in self.__canvas.context.GetDrawPairs(bounds=None) - ] - - # Drawables found excluding elements in drawables up to the point where - # all drawables are accounted for (found==num_drawables). - # In other words, this is a list of PINS above which the selected - # elements should be moved as a result of this operation - above_list = self.__FindPINsUntilMatches(draw_pairs, - drawable_pins, - None, - full_set=True) - - self.BeginCheckpoint('move up') - try: - self.__canvas.context.MoveElementsAbovePINs(drawables, above_list) - except Exception: - # TODO: other places should follow this protocol - self.CancelCheckpoint() - raise - else: - self.CommitCheckpoint() - self.__canvas.FullUpdate() - - def MoveDown(self) -> None: - if not self.__selected: - return - - drawables = [e for e in self.__selected if e.IsDrawable()] - drawable_pins = set([e.GetPIN() for e in drawables]) - - # All pairs - regardless of bounds. Note that no vis-tick filtering is - # needed because bounds=None - draw_pairs = [ - p.GetElement() - for p in self.__canvas.context.GetDrawPairs(bounds=None) - ] - - # Drawables found excluding elements in drawables up to the point where - # all drawables are accounted for (found==num_drawables). - # In other words, this is a list of PINS above which the selected - # elements should be moved as a result of this operation - below_list = self.__FindPINsUntilMatches(draw_pairs, - drawable_pins, - -1, - full_set=False) - if len(below_list) > 1: - # Drop last item so that these elements are inserted before it! - below_list = below_list[:-1] - - self.BeginCheckpoint('move down') - try: - self.__canvas.context.MoveElementsBelowPINs(drawables, below_list) - except Exception: - # TODO: other places should follow this protocol - self.CancelCheckpoint() - raise - else: - self.CommitCheckpoint() - self.__canvas.FullUpdate() - - # Iterate through a sequence of elements until every PIN (or the first - # encountered, depending on the value of full_set) in set_to_match - # has been matched with an element pin from the elements list while - # appending to a results list each element's PIN from elements which is - # not contained within set_to_match. If the end of the elements list is - # reached before every element in the elements list is matched to a PIN in - # set_to_match, then the result list will have incomplete_set_append - # appended to its end - def __FindPINsUntilMatches(self, - elements: List[Element], - set_to_match: Set[int], - incomplete_set_append: Optional[int], - full_set: bool = True) -> List[Optional[int]]: - num_drawables = len(set_to_match) - found = 0 # set_to_match items found iterating the list - - results: List[Optional[int]] = [] - - for e in elements: - pin = e.GetPIN() - if pin in set_to_match: - if full_set is False: - if not results: - return [-1] - return results # Found first match. Stop here - - # Found a match in the set - # Assume no duplicates in the element sequence - found += 1 - continue - else: - results.append(pin) - if found == num_drawables: - # Stop iterating now that the element above the last - # drawable was found - break - else: - # Indicates that things must be moved to the end of the list - results.append(incomplete_set_append) - - return results - - # Take the selection and make every element the average dimensions - def Average(self) -> None: - self.BeginCheckpoint('average elements') - tot_w = 0.0 - tot_h = 0.0 - count = len(self.__selected) - for e in self.__selected: - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - tot_w = tot_w + w - tot_h = tot_h + h - avg_w, avg_h = int(round(tot_w / count)), int(round(tot_h / count)) - for e in self.__selected: - e.SetProperty('dimensions', (avg_w, avg_h)) - self.CommitCheckpoint() - self.__canvas.Refresh() - - # Returns a bounding box in the form (l,t,r,b) - # If there is no selection, returns None - def GetBoundingBox(self) -> Optional[Tuple[int, int, int, int]]: - if not self.__selected: - return None - - left, top, right, bottom = (None, None, None, None) - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - if left is None or x < left: - left = x - if right is None or x + w > right: - right = x + w - if top is None or y < top: - top = y - if bottom is None or y + h > bottom: - bottom = y + h - - assert left is not None - assert top is not None - assert right is not None - assert bottom is not None - return (left, top, right, bottom) - - # Set the current reference position and re-compute the offsets to each - # Element in the selection. Usually happens due to a MouseDown evt - def SetPos(self, position: Tuple[int, int]) -> None: - self.__position = position - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - self.__selected[e] = (x - self.__position[0], - y - self.__position[1]) - - # Returns the selection mgr's last recorded position from which to - # compute offsets - def GetPos(self) -> Tuple[int, int]: - return self.__position - - # Accomplishes the calculating of what is selected by the rubber-band - # box and either adds or removes those Elements. Called while processing - # a MouseMove evt while history is set to rubber_band - def ProcessRubber(self, - xxx_todo_changeme: Tuple[int, int], - action: Optional[int] = None) -> None: - (x, y) = xxx_todo_changeme - if action: - self.__which_rubber_band = action - self.__rubberendpt = (x, y) - # Is this operation for adding new Elements? - if self.__which_rubber_band == self.add: - add_these = list(self.CalcAddable((x, y))) - self.Add(add_these) - # If an element was added earlier during this same operation (no - # MouseUp evt yet), but is no longer covered by the rubber band, - # remove it - for e in self.__temp_rubber: - if not (e in add_these): - self.Remove(e) - # Keep a temporary listing of which Elements have been added due - # to this rubber-band operation - self.__temp_rubber = add_these - elif self.__which_rubber_band == self.remove: - remove_these = self.CalcRemovable((x, y)) - self.Remove(remove_these) - self.Add([e for e in self.__temp_rubber if e not in remove_these]) - self.__temp_rubber = list(remove_these) - - # After the Layout Canvas has done its normal rendering, draw a faint - # outline & resize handles on the Elements in a selection - # @param dc The device context upon which to draw - def Draw(self, dc: wx.DC) -> None: - # Put selection handles on everything currently selected - (xoff, yoff) = self.__canvas.GetRenderOffsets() - brush = wx.Brush((255, 255, 255), wx.TRANSPARENT) - if self.__is_edit_mode: - pen = wx.Pen(self.__handle_color, 1) - brush2 = wx.Brush((255, 255, 255), wx.SOLID) - dc.SetPen(pen) - for e in self.__selected: - # Outline each element - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (x, y) = (x - xoff, y - yoff) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - dc.SetBrush(brush) - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - c_box_wid = self.__c_box_wid - corners = [ - (x - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x - c_box_wid / 2.0, y + h - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y + h - c_box_wid / 2.0) - ] - edges = [ - (x + w / 2.0 - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x + w / 2.0 - c_box_wid / 2.0, y + h - c_box_wid / 2.0), - (x - c_box_wid / 2.0, y + h / 2.0 - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y + h / 2.0 - c_box_wid / 2.0) - ] - dc.SetBrush(brush2) - # Put the selection 'hit' handles around the corners/middles of - # edges - for coords in corners: - dc.DrawRectangle(int(coords[0]), - int(coords[1]), - int(c_box_wid), - int(c_box_wid)) - for coords in edges: - dc.DrawRectangle(int(coords[0]), - int(coords[1]), - int(c_box_wid), - int(c_box_wid)) - # Draw the rubber-band box if applicable - if self.__history == self.rubber_band: - assert self.__rubberendpt is not None - (x, y) = self.__rubberendpt - (x, y) = (x - xoff, y - yoff) - pen = wx.Pen(self.__rubber_color, 2, style=wx.LONG_DASH) - dc.SetPen(pen) - brush = wx.Brush((255, 255, 255), wx.TRANSPARENT) - dc.SetBrush(brush) - top_left = (self.__position[0] - xoff, - self.__position[1] - yoff) - dc.DrawRectangle(int(top_left[0]), - int(top_left[1]), - int(x - top_left[0]), - int(y - top_left[1])) - elif self.__playback_selected is not None: - # playback mode has selected object - e = self.__playback_selected.GetElement() - pen = wx.Pen(self.__playback_handle_color, 2) - dc.SetPen(pen) - dc.SetBrush(brush) - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (x, y) = (x - xoff, y - yoff) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - - # For detecting collisions with the resize-handles on elements within - # the selection - def HitHandle(self, pt: Union[wx.Point, Tuple[int, int]]) -> bool: - if not self.__is_edit_mode: - return False - c_box_wid = self.__c_box_wid - # TODO:: Right now, checking every single - # element. Inefficient. revise later - for e in self.__selected: - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - corners = [ - (x - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x - c_box_wid / 2.0, y + h - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y + h - c_box_wid / 2.0) - ] - edges = [ - (x + w / 2.0 - c_box_wid / 2.0, y - c_box_wid / 2.0), - (x + w / 2.0 - c_box_wid / 2.0, y + h - c_box_wid / 2.0), - (x - c_box_wid / 2.0, y + h / 2.0 - c_box_wid / 2.0), - (x + w - c_box_wid / 2.0, y + h / 2.0 - c_box_wid / 2.0) - ] - tx, ty = pt - for i in range(4): - if corners[i][0] < tx < corners[i][0] + c_box_wid and \ - corners[i][1] < ty < corners[i][1] + c_box_wid: - # e will become the base element, and i is used for - # indexing to the proper callback for the appropriate - # resize prep & operations. Reference 'RESIZE_OPTIONS_*' - self.__resize_direction = (e, i) - self.SetCursor() - # deltas for resizing will be determined relative to - # previous mouse position, so the mouse position is stored - # when it hits a handle, before a mouse-move evt actually - # triggers the resize operation. Note, this is separate - # from the Selection Mgr's .__position, which has more - # 'public' uses - self.__prev_x, self.__prev_y = tx, ty - # Found a hit! - return True - if edges[i][0] < tx < edges[i][0] + c_box_wid and \ - edges[i][1] < ty < edges[i][1] + c_box_wid: - self.__resize_direction = (e, i + 4) - self.SetCursor() - self.__prev_x, self.__prev_y = tx, ty - return True - self.SetCursor(self.__default_cursor) - # Nope, mouse not over a selection handle - return False - - # Set's the history to correspond to the most recent type of event. - # @param val should correspond to one of the previously declared options - def SetHistory(self, val: int) -> None: - self.__history = val - if val == self.resize: - assert self.__resize_direction is not None - self.__RESIZE_PREPARATIONS[self.__resize_direction[1]]( - self, self.__resize_direction[0] - ) - self.__canvas.FullUpdate() - - # Returns the most recent class of event. NOTE: this cannot be used for - # redo/undo purposes - def GetHistory(self) -> int: - return self.__history - - # Resize the entire selection to match the mouse position. Forwards the - # call to one or more actually specific methods with the resize logic - # @param pt The current mouse location - # @param mode Which of the available implemented modes of resize to do - def Resize(self, - pt: Union[wx.Point, Tuple[int, int]], - mode: str = 'one') -> None: - if mode == 'one': - assert self.__resize_direction is not None - self.__RESIZE_OPTIONS_ONE[self.__resize_direction[1]]( - self, pt, self.__resize_direction[0] - ) - if mode == 'two': - assert self.__resize_direction is not None - self.__RESIZE_OPTIONS_TWO[self.__resize_direction[1]]( - self, pt, self.__resize_direction[0] - ) - self.__canvas.FullUpdate() - - # Sets the Canvas' cursor according to what sort of resize option the - # mouse is over. - # @note: cursor options/functionality currently limited 8/23/2012 - def SetCursor(self, cursor: Optional[wx.Cursor] = None) -> None: - if cursor is None: - assert self.__resize_direction is not None - selected_cursor = self.__CURSOR_OPTIONS[self.__resize_direction[1]] - if selected_cursor == wx.CURSOR_SIZENESW: - self.__canvas.SetCursor(self.__custom_cursor_nesw) - elif selected_cursor == wx.CURSOR_SIZENWSE: - self.__canvas.SetCursor(self.__custom_cursor_nwse) - else: - self.__canvas.SetCursor(wx.Cursor(selected_cursor)) - else: - self.__canvas.SetCursor(cursor) - - # Figure out what Elements are currently within the rubber-band box the - # user is drawing that are eligible to be added to the selection - # (clk-n-drag) - # @param pt Usually the mouse position. The point to use as the opposite - # corner from self.__position, in determining the bounding box - def CalcAddable(self, - pt: Union[wx.Point, Tuple[int, int]]) -> Sequence[Element]: - res = [] - # convert the coords into top left and bottom right - tlx = min(pt[0], self.__position[0]) - tly = min(pt[1], self.__position[1]) - brx = max(pt[0], self.__position[0]) - bry = max(pt[1], self.__position[1]) - for e in self.__canvas.GetElements(): - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - (x1, y1) = (x + w, y + h) - # Check that the element is completely enclosed by the rubber band - # box @note: change here for intersection opposed to enclosure - if tlx <= x <= brx and tlx <= x1 <= brx and \ - tly <= y <= bry and tly <= y1 <= bry: - # Check that it is either on the temp list for previously - # captured via rubber band, or else it is not yet selected - if (e in self.__temp_rubber) or not self.IsSelected(e): - res.append(e) - return res - - # Figure out what Elements are currently within the rubber-band box the - # user is drawing that are eligible to be removed from selection (alt - # clk-n-drag) - # @param pt Usually the mouse position. The point to use as the opposite - # corner from self.__position, in determining the bounding box - def CalcRemovable( - self, - pt: Union[wx.Point, Tuple[int, int]] - ) -> Sequence[Element]: - res = [] - # convert the coords into top left and bottom right - tlx = min(pt[0], self.__position[0]) - tly = min(pt[1], self.__position[1]) - brx = max(pt[0], self.__position[0]) - bry = max(pt[1], self.__position[1]) - for e in self.__canvas.GetElements(): - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - (x1, y1) = (x + w, y + h) - # Check that the element is completely enclosed by the rubber band - # box @note: change here for intersection opposed to enclosure - if tlx <= x <= brx and tlx <= x1 <= brx and \ - tly <= y <= bry and tly <= y1 <= bry: - # Check that it is either on the temp list for previously - # captured via rubber band, or else it is currently selected - if (e in self.__temp_rubber) or self.IsSelected(e): - res.append(e) - return res - - # Returns lists and dictionaries of positions and elements in the - # selection, organized by their positions (and dimensions). Align, - # Indent, Flip, and Stack make use of this - def SortByPosition(self) -> Tuple[List[int], - Dict[int, List[Element]], - List[int], - Dict[int, List[Element]], - List[int], - Dict[Element, int], - List[int], - Dict[Element, int]]: - assert self.__selected - exes = [] # A list of the unique x-coords for Element positions, incrs - # Key = x-coord; Val = [Element,Element, ...] - x_sorted: Dict[int, List[Element]] = {} - whys = [] # A list of the unique y-coords for Element positions, incrs - # Key = y-coord; Val = [Element,Element, ...] - y_sorted: Dict[int, List[Element]] = {} - plus_w = [] - w_sorted = {} # Key = Element; Val = width (no extra calls necessary) - plus_h = [] - h_sorted = {} # Key = Element; Val = height (no extra calls) - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - if not (x in exes): - exes.append(x) - temp = x_sorted.get(x, []) - temp.append(e) - x_sorted[x] = temp - if not (y in whys): - whys.append(y) - temp = y_sorted.get(y, []) - temp.append(e) - y_sorted[y] = temp - plus_w.append(x + w) - w_sorted[e] = w - plus_h.append(y + h) - h_sorted[e] = h - exes.sort() # Put 'em all in increasing order - whys.sort() - plus_h.sort() - plus_w.sort() - return (exes, - x_sorted, - whys, - y_sorted, - plus_w, - w_sorted, - plus_h, - h_sorted) - - # Align the given edge of every Element in the selection to the - # corresponding edge of the Element currently located farthest in that - # direction - # @param direction The side of each Element to align - def Align(self, direction: str) -> None: - if not self.__selected: - return - - self.BeginCheckpoint('align elements') - exes, x_sorted, whys, y_sorted, plus_w, w_sorted, plus_h, h_sorted = self.SortByPosition() # noqa: E501 - if direction == 'left': - for e in self.__selected: - e.SetProperty('position', (exes[0], e.GetYPos())) - self.__alignment = self.LEFT - self.__edge = exes[0] - elif direction == 'top': - for e in self.__selected: - e.SetProperty('position', (e.GetXPos(), whys[0])) - self.__alignment = self.TOP - self.__edge = whys[0] - elif direction == 'bottom': - for e in self.__selected: - e.SetProperty('position', (e.GetXPos(), - plus_h[-1] - h_sorted[e])) - self.__alignment = self.BOTTOM - self.__edge = plus_h[-1] - elif direction == 'right': - for e in self.__selected: - e.SetProperty('position', (plus_w[-1] - w_sorted[e], - e.GetYPos())) - self.__alignment = self.RIGHT - self.__edge = plus_w[-1] - self.__canvas.FullUpdate() - self.CommitCheckpoint() - - # Line the selection up in a row/column behind the element located - # farthest in the given direction - # @param direction The element located farthest in this direction - # becomes the stationary base of the 'stack' which will progress in the - # opposite direction - def Stack(self, direction: str) -> None: - if len(self.__selected) == 0: - return - - self.BeginCheckpoint('stack elements') - exes, x_sorted, whys, y_sorted, plus_w, w_sorted, plus_h, h_sorted = self.SortByPosition() # noqa: E501 - # Stack from left - if direction == 'left': - bx = exes[0] - base = x_sorted[bx][0] - by = base.GetYPos() - for x in exes: - # Used in the event that the selection was already aligned - # to the left edge - if len(x_sorted[x]) > 1: - temp: Dict[int, List[Element]] = {} - subwhys = [] - for e in x_sorted[x]: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - # Vertical position is the secondary level of - # sorting, when precedence based on horizontal - # position fails - if not (y in subwhys): - subwhys.append(y) - subtemp = temp.get(y, []) - subtemp.append(e) - temp[y] = subtemp - subwhys.sort() - for y in subwhys: - for e in temp[y]: - e.SetProperty('position', (bx, by)) - bx = bx + w_sorted[e] - # 'Original' case. Stack them by preserving their - # horizontal order - else: - e = x_sorted[x][0] - e.SetProperty('position', (bx, by)) - bx = bx + w_sorted[e] - # Make note that the selection is currently aligned - self.__alignment = self.TOP - self.__edge = by - # Stack from right - elif direction == 'right': - bx = plus_w[-1] - exes.reverse() - base = x_sorted[exes[0]][0] - by = base.GetYPos() - for x in exes: - if len(x_sorted[x]) > 1: - temp = {} - subwhys = [] - for e in x_sorted[x]: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - if not (y in subwhys): - subwhys.append(y) - subtemp = temp.get(y, []) - subtemp.append(e) - temp[y] = subtemp - subwhys.sort() - for y in subwhys: - for e in temp[y]: - e.SetProperty('position', (bx - w_sorted[e], by)) - bx = bx - w_sorted[e] - else: - e = x_sorted[x][0] - e.SetProperty('position', (bx - w_sorted[e], by)) - bx = bx - w_sorted[e] - self.__alignment = self.TOP - self.__edge = by - # Stack from top - elif direction == 'top': - by = whys[0] - base = y_sorted[by][0] - bx = base.GetXPos() - for y in whys: - if len(y_sorted[y]) > 1: - temp = {} - subexes = [] - for e in y_sorted[y]: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - if not (x in subexes): - subexes.append(x) - subtemp = temp.get(x, []) - subtemp.append(e) - temp[x] = subtemp - subexes.sort() - for x in subexes: - for e in temp[x]: - e.SetProperty('position', (bx, by)) - by = by + h_sorted[e] - else: - e = y_sorted[y][0] - e.SetProperty('position', (bx, by)) - by = by + h_sorted[e] - self.__alignment = self.LEFT - self.__edge = bx - # Stack from bottom - elif direction == 'bottom': - by = plus_h[-1] - whys.reverse() - base = y_sorted[whys[0]][0] - bx = base.GetXPos() - for y in whys: - if len(y_sorted[y]) > 1: - temp = {} - subexes = [] - for e in y_sorted[y]: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - if not (x in subexes): - subexes.append(x) - subtemp = temp.get(x, []) - subtemp.append(e) - temp[x] = subtemp - subexes.sort() - for x in subexes: - for e in temp[x]: - e.SetProperty('position', (bx, by - h_sorted[e])) - by = by - h_sorted[e] - else: - e = y_sorted[y][0] - e.SetProperty('position', (bx, by - h_sorted[e])) - by = by - h_sorted[e] - self.__alignment = self.LEFT - self.__edge = bx - self.__canvas.FullUpdate() - self.CommitCheckpoint() - - # This should be called on mouse ups in order to make sure that the - # Elements selected by rubber-band operation stay selected - def FlushTempRubber(self) -> None: - self.__temp_rubber = [] - - # Returns true if there is a selected Element beneath the given point - def DetectCollision(self, pt: Union[wx.Point, Tuple[int, int]]) -> bool: - mx, my = pt - for e in self.__selected: - x, y = cast(Tuple[int, int], e.GetProperty('position')) - w, h = cast(Tuple[int, int], e.GetProperty('dimensions')) - if x <= mx <= (x + w) and y <= my <= (y + h): - return True - return False - - # debug purposes only - def __str__(self) -> str: - return str(self.__selected) diff --git a/helios/pipeViewer/pipe_view/gui/widgets/__init__.py b/helios/pipeViewer/pipe_view/gui/widgets/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/gui/widgets/element_list.py b/helios/pipeViewer/pipe_view/gui/widgets/element_list.py deleted file mode 100644 index 3ab6e4c022..0000000000 --- a/helios/pipeViewer/pipe_view/gui/widgets/element_list.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING -import wx -from ..font_utils import GetMonospaceFont - -if TYPE_CHECKING: - from ..layout_canvas import Layout_Canvas - from ...model.element import Element - - -# This class is a GUI list control element that shows elements and allows -# selection of them. -class ElementList(wx.ListCtrl): - def __init__(self, - parent: wx.Frame, - canvas: Layout_Canvas, - name: str = '', - properties: Optional[List[str]] = None) -> None: - if properties is None: - properties = ['element'] - wx.ListCtrl.__init__(self, - parent=parent, - id=wx.NewId(), - name=name, - style=wx.LC_REPORT | wx.SUNKEN_BORDER) - self.SetFont(GetMonospaceFont(canvas.GetSettings().layout_font_size)) - - # used for coloring - self.__canvas = canvas - # list of dictionary of properties - self.__elements: List[Dict[str, Any]] = [] - # list of element pointers - self.__element_ptrs: List[Element] = [] - # properties to show. - self.__properties = properties[:] # must have at least 1 - # insertion point for elements at end - self.__current_new_idx = 0 - self.RefreshAll() - - def GetProperties(self) -> Tuple[str, ...]: - return tuple(self.__properties) - - def SetProperties(self, properties: List[str]) -> None: - self.__properties = properties[:] - self.RefreshAll() - - def Clear(self) -> None: - self.__elements = [] - self.__element_ptrs = [] - - # Destroy all graphical elements and recreate them - def RefreshAll(self) -> None: - self.ClearAll() - # make header - width = int(self.GetClientSize()[0] / len(self.__properties)) - if width < 100: - width = 100 - for col_idx, column in enumerate(self.__properties): - self.InsertColumn(col_idx, column) - self.SetColumnWidth(col_idx, width) - self.__current_new_idx = 0 - for el in self.__elements: - self.__AddGraphicalElement(el) - - def RefreshElement(self, index: int) -> None: - element = self.__elements[index] - for col_idx, prop in enumerate(self.__properties): - self.SetItem(index, col_idx, str(element.get(prop))) - color = (255, 255, 255) - self.SetItemBackgroundColour(index, color) - - def GetElement(self, index: int) -> Element: - return self.__element_ptrs[index] - - def __AddGraphicalElement(self, element: Dict[str, Any]) -> None: - self.InsertItem(self.__current_new_idx, - str(element.get(self.__properties[0]))) - self.RefreshElement(self.__current_new_idx) - self.__current_new_idx += 1 - - # Add a new element to bottom of list - # New item must be dictionary of properties. - def Add(self, - element_ptr: Element, - element_properties: Dict[str, Any]) -> int: - self.__element_ptrs.append(element_ptr) - self.__elements.append(element_properties) - self.__AddGraphicalElement(element_properties) - return self.__current_new_idx-1 - - def Remove(self, index: int) -> None: - del self.__element_ptrs[index] - del self.__elements[index] - self.DeleteItem(index) - self.__current_new_idx -= 1 - - # Attempts to resize the columns based on size - # @pre All content should be added before this is called (once) - def FitColumns(self) -> None: - for idx, _ in enumerate(self.__properties): - self.SetColumnWidth(idx, wx.LIST_AUTOSIZE) - self.Layout() diff --git a/helios/pipeViewer/pipe_view/gui/widgets/element_property_list.py b/helios/pipeViewer/pipe_view/gui/widgets/element_property_list.py deleted file mode 100644 index ec6d90b030..0000000000 --- a/helios/pipeViewer/pipe_view/gui/widgets/element_property_list.py +++ /dev/null @@ -1,825 +0,0 @@ -from __future__ import annotations -import wx -import wx.grid -from wx.lib.colourchooser.pycolourchooser import (ColourChangedEvent, - PyColourChooser, - EVT_COLOUR_CHANGED) -from ...model import content_options as copts -from ...model.schedule_element import ScheduleLineElement -from ...model.element import Element, LocationallyKeyedElement -from ...model.rpc_element import RPCElement -from ast import literal_eval -from functools import cmp_to_key -from typing import (Dict, - List, - Optional, - Tuple, - Type, - Union, - cast, - TYPE_CHECKING) - -if TYPE_CHECKING: - from ..dialogs.element_propsdlg import Element_PropsDlg - from ..layout_frame import Layout_Frame - from ..selection_manager import Selection_Mgr - from ...model.location_manager import LocationTree - -MULTIPLE_VALS_STR = '' -MULTIPLE_VALS_COLOR = wx.Colour(200, 200, 100) - - -# Grid descendent used to display and edit element properties -class ElementPropertyList(wx.grid.Grid): - - def __init__(self, parent: Element_PropsDlg, id: int) -> None: - self.__in_resize = False - - wx.grid.Grid.__init__(self, parent, id) - self.__elements: List[Element] = [] - self.__keys: List[str] = [] - # population function (SetElements) is working - self.__internal_editing = False - self.__parent = parent - self.__sel_mgr: Optional[Selection_Mgr] = None - - # make the graphical stuff - self.CreateGrid(1, 1) - self.SetColLabelSize(0) - self.SetRowLabelSize(150) - - self.Bind(wx.grid.EVT_GRID_CELL_CHANGED, self.OnCellChange) - self.Bind(wx.EVT_SIZE, self.OnResize) - self.__db = cast( - 'Layout_Frame', - self.__parent.GetParent() - ).GetContext().dbhandle.database - - self.__InstallGridHint() - - def OnResize(self, evt: wx.SizeEvent) -> None: - remaining = max(0, self.GetClientSize()[0] - self.GetRowLabelSize()) - - # Update column sizes with recursion prevention - if not self.__in_resize and remaining != 0: - # Do a resize of slots here but ignore the next resize event - # setColSize can cause recusion - self.__in_resize = True - self.SetColSize(0, remaining) - - self.ForceRefresh() # Prevent ghosting of gridlines - - else: - self.__in_resize = False - - evt.Skip() - - # Set the elements. - # @param elements Elements being edited - # @param sem_mgr Selection manager - def SetElements(self, - elements: List[Element], - sel_mgr: Selection_Mgr) -> None: - self.__elements = elements[:] - self.__sel_mgr = sel_mgr - if self.__elements: - # show intersection of all current selected properties - props_set = set(self.__elements[0].GetProperties().keys()) - hidden_props_set = set(self.__elements[0].GetHiddenProperties()) - read_only_props_set = set( - self.__elements[0].GetReadOnlyProperties() - ) - for i in range(1, len(self.__elements)): - el = self.__elements[i] - props_set.union(el.GetElementProperties()) - hidden_props_set.intersection_update(el.GetHiddenProperties()) - read_only_props_set.intersection_update( - el.GetReadOnlyProperties() - ) - - # Remove hidden keys - props_set.difference_update(hidden_props_set) - read_only_props_set.difference_update(hidden_props_set) - props = list(props_set) - read_only_props = list(read_only_props_set) - else: - props = [] - read_only_props = [] - - self.__internal_editing = True - # make our GUI alterations - number_old_rows = self.GetNumberRows() - if number_old_rows: - self.DeleteRows(0, number_old_rows) - self.AppendRows(len(props), True) - self.__keys = list(props) - self.__keys = sorted(self.__keys) - - # move type to top - for i, key in enumerate(self.__keys): - if key == 'type': - self.__keys.insert(0, self.__keys.pop(i)) - break - - # Move read only keys to bottom - read_only_keys = list(filter(lambda k: k in read_only_props, - self.__keys)) - self.__keys = list(filter(lambda k: k not in read_only_props, - self.__keys)) + read_only_keys - - index = 0 - for key in self.__keys: - self.SetRowLabelValue(index, key) - self.__UpdateItem(index) # set value - if key in ('type', - 'children', - 'name', - 'connections_in', - 'connections_out'): - self.SetReadOnly(index, 0, True) - elif key == 'Content': - cont_opts = copts.GetContentOptions() - self.SetCellEditor(index, 0, DropdownCellEditor(cont_opts)) - elif key == 'line_style': - self.SetCellEditor( - index, - 0, - DropdownCellEditor( - list(ScheduleLineElement.DRAW_LOOKUP.keys()) - ) - ) - elif key == 'color_basis_type': - self.SetCellEditor( - index, - 0, - DropdownCellEditor( - LocationallyKeyedElement.COLOR_BASIS_TYPES - ) - ) - elif key == 'anno_basis_type': - self.SetCellEditor( - index, - 0, - DropdownCellEditor(RPCElement.ANNO_BASIS_TYPES) - ) - elif key == 'color': - self.SetCellEditor(index, 0, ColorCellEditor()) - elif key == 'LocationString': - self.SetCellEditor( - index, - 0, - TreeCellEditor(self.__db.location_manager.location_tree) - ) - elif key == 'short_format': - self.SetCellEditor( - index, - 0, - DropdownCellEditor(ScheduleLineElement.SHORT_FORMAT_TYPES) - ) - elif key == 'clock': - self.SetCellEditor( - index, - 0, - DropdownCellEditor( - [clk.name - for clk in self.__db.clock_manager.getClocks()] - ) - ) - if key in read_only_props: - self.SetReadOnly(index, 0, True) - self.SetReadOnly(index, 1, True) - index += 1 - self.currentItem = 0 - self.AutoSizeColumn(0, setAsMin=True) - self.__internal_editing = False - - def OnCellChange(self, evt: wx.grid.GridEvent) -> None: - row = evt.GetRow() - - value = self.GetCellValue(row, 0) - # hopefully whatever the user input is valid... (data will pass - # through the validation steps on the Element side) - - # if the population function isn't calling - if not self.__internal_editing: - k = self.__keys[row] - # Begin checkpoint including all elements selected so that - # selection is correct checkpoint apply/remove - if self.__sel_mgr is not None: - self.__sel_mgr.BeginCheckpoint(f'set property {k}', - force_elements=self.__elements) - try: - els = [e for e in self.__elements if e.HasProperty(k)] - success = False - for e in els: - try: - e.SetProperty(k, value) - # but if it throws one of these errors, we'll catch it - # and display it on the status bar. If other sorts of - # errors/exceptions are being raised, they will need to - # be hardcoded in the same fashion - except ValueError as v: - self.__parent.ShowError(v) - break # Show first error and break - except TypeError as t: - self.__parent.ShowError(t) - break # Show first error and break - else: - # complete - if len(self.__elements) > 1: - # probably need to set multiple-value color to white - self.SetCellBackgroundColour(row, 0, (255, 255, 255)) - self.__parent.GetCanvas().context.GoToHC() - self.__parent.GetCanvas().Refresh() - success = True - - if not success: - # not fully completed due to error - # set back - self.SetCellValue(row, 0, str(e.GetProperty(k))) - except Exception: - raise - finally: - if self.__sel_mgr is not None: - self.__sel_mgr.CommitCheckpoint( - force_elements=self.__elements - ) - - def GetNumberOfElements(self) -> int: - return len(self.__elements) - - # override read-only setter to gray out text - def SetReadOnly(self, - row: int, - col: int, - is_read_only: bool = True) -> None: - wx.grid.Grid.SetReadOnly(self, row, col, is_read_only) - if is_read_only: - color = (128, 128, 128) - else: - color = (0, 0, 0) - self.SetCellTextColour(row, col, color) - - def __UpdateItem(self, index: int) -> None: - # Look through each element. If property value read all match, show - # that value. If they differ, show a string saying that - val = None - prop = self.__keys[index] - for e in self.__elements: - if not e.HasProperty(prop): - continue - el_val = str(e.GetProperty(prop)) - if val is None: # First value encoutered in iteration - val = el_val - elif val != el_val: # Differs from last value - val = MULTIPLE_VALS_STR - self.SetCellBackgroundColour(index, 0, MULTIPLE_VALS_COLOR) - break - - self.SetCellValue(index, 0, val) - - # From http://wiki.wxpython.org/wxGrid%20ToolTips - def __InstallGridHint(self) -> None: - prev_rowcol = [None, None] - - def OnMouseMotion(evt: wx.MouseEvent) -> None: - # evt.GetRow() and evt.GetCol() would be nice to have here, - # but as this is a mouse event, not a grid event, they are not - # available and we need to compute them by hand. - x, y = self.CalcUnscrolledPosition(evt.GetPosition()) - row = self.YToRow(y) - col = self.XToCol(x) - - if (row, col) != prev_rowcol and row >= 0 and col >= 0: - prev_rowcol[:] = [row, col] - hinttext = self.GetCellValue(row, col) - if hinttext is None: - hinttext = '' - if self.IsReadOnly(row, col): - hinttext = "(read-only attribute)" + hinttext - self.GetGridWindow().SetToolTip(hinttext) - evt.Skip() - - self.GetGridWindow().Bind(wx.EVT_MOTION, OnMouseMotion, id=wx.ID_NONE) - - -# Class that displays drop-down-list formatted options when the user edits a -# cell -class DropdownCellEditor(wx.grid.GridCellEditor): - # options is a list of options. - def __init__(self, options: List[str]) -> None: - wx.grid.GridCellEditor.__init__(self) - self.__options = options - self.__chooser: Optional[wx.Choice] = None - - # Event handler for selecting an item - def ChoiceEvent(self, evt: wx.CommandEvent) -> None: - if self.__chooser is not None: - if self.__event_handler: - self.__chooser.PushEventHandler(self.__event_handler) - new_idx = self.__chooser.GetCurrentSelection() - new_val = self.__chooser.GetString(new_idx) - self.__chooser.SetSelection(new_idx) - self.__grid.SetCellValue(self.__row, self.__column, new_val) - grid_evt = wx.grid.GridEvent( - id=wx.NewId(), - type=wx.grid.EVT_GRID_CELL_CHANGED.typeId, - obj=self.__grid, - row=self.__row, - col=self.__column - ) - wx.PostEvent(self.__chooser.GetParent(), grid_evt) - - # make the selection dialog - def Create(self, - parent: ElementPropertyList, - id: int, event_handler: wx.EvtHandler) -> None: - self.__chooser = wx.Choice(parent, id, choices=self.__options) - self.__chooser.Bind(wx.EVT_CHOICE, self.ChoiceEvent) - self.SetControl(self.__chooser) - - self.__event_handler = event_handler - if event_handler: - self.__chooser.PushEventHandler(event_handler) - - def SetSize(self, rect: wx.Rect) -> None: - if self.__chooser is not None: - offset = 4 - self.__chooser.SetSize(rect.x, rect.y, - rect.width + offset, rect.height + offset, - wx.SIZE_ALLOW_MINUS_ONE) - - def BeginEdit(self, row: int, column: int, grid: wx.grid.Grid) -> None: - assert self.__chooser is not None - if self.__event_handler: - self.__chooser.PopEventHandler() - self.__grid = grid - self.__row = row - self.__column = column - self.__init_val: str = cast(str, grid.GetTable().GetValue(row, column)) - self.__chooser.SetStringSelection(self.__init_val) - self.__chooser.SetFocus() - - def ApplyEdit(self, row: int, column: int, grid: wx.grid.Grid) -> None: - grid.SetCellValue(row, column, self.__new_val) - - def EndEdit(self, - row: int, - column: int, - grid: wx.grid.Grid, - old_val: Optional[str] = None, - new_val: Optional[str] = None) -> Optional[str]: - assert self.__chooser is not None - if self.__event_handler: - self.__chooser.PushEventHandler(self.__event_handler) - new_val = self.__chooser.GetStringSelection() - if new_val != self.__init_val: - self.__new_val = new_val - return str(new_val) - return None - - def Reset(self) -> None: - assert self.__chooser is not None - self.__chooser.SetStringSelection(self.__init_val) - - def Clone(self) -> DropdownCellEditor: - return DropdownCellEditor(self.__options) - - -# Class which allows user to edit a cell with text entry or a popup window -class PopupCellEditor(wx.grid.GridCellEditor): - __chooser = None - __popup = None - __grid = None - __row = None - __column = None - __init_val = '' - __event_handler = None - __in_update_handler = False - - # The class to use for the popup window is specified in popup_type - def __init__(self, - popup_type: Optional[Type[wx.ComboPopup]] = None) -> None: - wx.grid.GridCellEditor.__init__(self) - self.__chooser = None - self.__popup_type = popup_type - - # make the selection dialog - def Create(self, - parent: ElementPropertyList, - id: int, - event_handler: wx.EvtHandler) -> None: - # A ComboCtrl handles the text entry and popup window - self.__chooser = wx.ComboCtrl(parent, - wx.NewId(), - "", - style=wx.TE_PROCESS_ENTER) - if self.__popup_type is not None: - self.SetPopup(self.__popup_type()) - self.SetControl(self.__chooser) - self.__event_handler = event_handler - if event_handler is not None: - self.__chooser.PushEventHandler(event_handler) - self.__chooser.Bind(wx.EVT_KILL_FOCUS, self.FocusLost) - - def SetPopup(self, popup: wx.ComboPopup) -> None: - self.__popup = popup - assert self.__chooser is not None - self.__chooser.SetPopupControl(self.__popup) - - def SetSize(self, rect: wx.Rect) -> None: - if self.__chooser is not None: - offset = 4 - assert self.__popup is not None - (popup_width, popup_height) = \ - self.__popup.GetControl().GetBestSize() - self.__chooser.SetSize(rect.x, rect.y, - rect.width + offset, rect.height + offset, - wx.SIZE_ALLOW_MINUS_ONE) - self.__chooser.SetPopupMinWidth(popup_width) - - def BeginEdit(self, row: int, column: int, grid: wx.grid.Grid) -> None: - assert self.__chooser is not None - if self.__event_handler is not None: - self.__chooser.PopEventHandler() - self.__init_val = cast(str, grid.GetTable().GetValue(row, column)) - self.__grid = grid - self.__row = row - self.__column = column - self.__chooser.SetValue(self.__init_val) - # wx v2.8 has issues with this - self.__chooser.SetFocus() - - def GetChooser(self) -> Optional[wx.ComboCtrl]: - return self.__chooser - - def GetPopup(self) -> Optional[wx.ComboPopup]: - return self.__popup - - def GetGrid(self) -> Optional[wx.grid.Grid]: - return self.__grid - - def GetRow(self) -> Optional[int]: - return self.__row - - def GetColumn(self) -> Optional[int]: - return self.__column - - def FocusLost(self, evt: wx.FocusEvent) -> None: - if evt.GetWindow() != self.__chooser: - if not self.__in_update_handler: - self.__in_update_handler = True - assert self.__chooser is not None - if self.__event_handler: - self.__chooser.PushEventHandler(self.__event_handler) - self.UpdateGrid() - grid_evt = wx.grid.GridEvent( - id=wx.NewId(), - type=wx.grid.EVT_GRID_CELL_CHANGED.typeId, - obj=self.__chooser.GetParent(), - row=self.GetRow(), - col=self.GetColumn() - ) - wx.PostEvent(self.__chooser.GetParent(), grid_evt) - else: - self.__in_update_handler = False - - def UpdateGrid(self) -> None: - assert self.__chooser is not None - new_val = self.__chooser.GetValue() - assert self.__grid is not None - self.__grid.SetCellValue(self.GetRow(), self.GetColumn(), new_val) - - def ApplyEdit(self, row: int, column: int, grid: wx.grid.Grid) -> None: - grid.SetCellValue(row, column, self.__new_val) - - def EndEdit(self, - row: int, - column: int, - grid: wx.grid.Grid, - old_val: Optional[str] = None, - new_val: Optional[str] = None) -> Optional[str]: - assert self.__chooser is not None - if self.__event_handler: - self.__chooser.PushEventHandler(self.__event_handler) - new_val = self.__chooser.GetValue() - if new_val != self.__init_val: - self.__new_val = new_val - return str(new_val) - return None - - def Reset(self) -> None: - assert self.__chooser is not None - self.__chooser.SetValue(self.__init_val) - - def Clone(self) -> PopupCellEditor: - return PopupCellEditor(self.__popup_type) - - -# Subclass of ComboPopup that allows us to set a flag for whether it should -# call back to the parent ComboCtrl and update its text -class TextUpdatePopup(wx.ComboPopup): - - def __init__(self) -> None: - wx.ComboPopup.__init__(self) - self.__should_update_text = True - - def SetUpdateText(self, update_text: bool) -> None: - self.__should_update_text = update_text - - def ShouldUpdateText(self) -> bool: - return self.__should_update_text - - -# Color chooser popup class -class ColorPopup(TextUpdatePopup): - - def __init__(self) -> None: - TextUpdatePopup.__init__(self) - self.__colour_chooser: Optional[PyColourChooser] = None - self.__should_update_text = True - - def Create(self, parent: ElementPropertyList) -> bool: - self.__colour_chooser = PyColourChooser(parent, wx.NewId()) - self.__colour_chooser.Bind(EVT_COLOUR_CHANGED, self.OnColorChanged) - return True - - def OnColorChanged(self, evt: ColourChangedEvent) -> None: - if self.ShouldUpdateText(): - assert self.__colour_chooser is not None - self.GetComboCtrl().SetText( - str(self.__colour_chooser.GetValue().Get(includeAlpha=False)) - ) - - def SetValue(self, color: wx.Colour) -> None: - assert self.__colour_chooser is not None - self.__colour_chooser.SetValue(color) - - def GetValue(self) -> wx.Colour: - assert self.__colour_chooser is not None - return self.__colour_chooser.GetValue() - - def GetControl(self) -> wx.Window: - assert self.__colour_chooser is not None - return self.__colour_chooser - - -# Color cell editor class -class ColorCellEditor(PopupCellEditor): - - def __init__(self) -> None: - PopupCellEditor.__init__(self, ColorPopup) - - # Event handler for selecting a color - updates the GridEditor - def OnColorPickerChanged(self, evt: wx.CommandEvent) -> None: - chooser = self.GetChooser() - assert chooser is not None - popup = self.GetPopup() - assert popup is not None - popup = cast(ColorPopup, popup) - chooser.SetText(str(popup.GetValue().Get())) - self.UpdateGrid() - - # make the selection dialog - def Create(self, - parent: ElementPropertyList, - id: int, - event_handler: wx.EvtHandler) -> None: - super().Create(parent, id, event_handler) - chooser = self.GetChooser() - assert chooser is not None - # This event doesn't exist in wx v2.8 - chooser.Bind(wx.EVT_TEXT_ENTER, self.OnColorPickerChanged) - chooser.Bind(wx.EVT_TEXT, self.OnTextChanged) - - def OnTextChanged(self, evt: wx.CommandEvent) -> None: - chooser = self.GetChooser() - if evt.GetEventObject() == chooser: - # If a valid color tuple has been entered, update the color - # chooser's value - try: - assert chooser is not None - color_tuple = literal_eval(chooser.GetTextCtrl().GetValue()) - color = wx.Colour() - color.Set(color_tuple[0], color_tuple[1], color_tuple[2], 255) - popup = self.GetPopup() - assert popup is not None - popup = cast(ColorPopup, popup) - popup.SetUpdateText(False) - popup.SetValue(color) - popup.SetUpdateText(True) - # Otherwise, ignore the entered value - except Exception: - pass - else: - evt.Skip() - - def Clone(self) -> ColorCellEditor: - return ColorCellEditor() - - -# Tree view popup class -class TreePopup(TextUpdatePopup): - - def __init__(self) -> None: - TextUpdatePopup.__init__(self) - self.__tree: Optional[wx.TreeCtrl] = None - # Maps full path strings (top.path.to.object) to their respective nodes - # in the TreeCtrl - self.__tree_dict: Dict[str, wx.TreeItemId] = {} - self.__loc_tree: LocationTree = {} - - # Set the location tree dictionary for this tree view and populate the top - # level entities - def SetLocationTree(self, loc_tree: LocationTree) -> None: - self.__loc_tree = loc_tree - assert self.__tree is not None - self.__tree.DeleteChildren(self.__root) - self.__tree_dict = {} - for key in self.__loc_tree.keys(): - child = self.__tree.AppendItem(self.__root, key) - if self.__loc_tree[key] != {}: - self.__tree.SetItemHasChildren(child, True) - self.__tree_dict[key] = child - - def Create(self, parent: ElementPropertyList) -> bool: - self.__tree = wx.TreeCtrl( - parent, - wx.NewId(), - style=wx.TR_HIDE_ROOT | wx.TR_HAS_BUTTONS | wx.TR_SINGLE - ) - # Ensure that the tree calculates its best size correctly - self.__tree.SetQuickBestSize(False) - # Add the invisible root node - self.__root = self.__tree.AddRoot('__root') - self.__tree.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.OnExpandItem) - self.__tree.Bind(wx.EVT_TREE_ITEM_EXPANDED, self.OnExpandedItem) - self.__tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.OnSelChanged) - self.__tree.Bind(wx.EVT_TREE_KEY_DOWN, self.OnKeyDown) - return True - - # Update the ComboCtrl text box with the selected value in the tree - def UpdateParentValue(self) -> None: - self.GetComboCtrl().SetValue(self.GetValue()) - - def OnSelChanged(self, evt: wx.TreeEvent) -> None: - if self.ShouldUpdateText(): - self.UpdateParentValue() - - # If the user presses Enter, update the value and close the popup - def OnKeyDown(self, evt: wx.TreeEvent) -> None: - if evt.GetKeyEvent().GetKeyCode() == wx.WXK_RETURN: - self.UpdateParentValue() - self.Dismiss() - else: - evt.Skip() - - def GetControl(self) -> wx.Window: - assert self.__tree is not None - return self.__tree - - # Resize the popup to account for what is being shown - def AutoSize(self) -> None: - assert self.__tree is not None - (width, height) = self.__tree.GetBestSize() - self.__tree.SetSize((width, -1)) - self.GetComboCtrl().GetPopupWindow().SetSize((width, -1)) - - def OnExpandedItem(self, evt: wx.TreeEvent) -> None: - self.AutoSize() - - # This intelligently populates the tree as its nodes are expanded instead - # of doing it all at once - reduces time to open the popup - def OnExpandItem(self, evt: wx.TreeEvent) -> None: - item = evt.GetItem() - assert self.__tree is not None - path = self.__tree.GetItemText(item) - curdict = self.__loc_tree - for token in path.split('.'): - curdict = curdict[token] - self.SetTree(curdict, item, path) - - def GetValue(self) -> str: - assert self.__tree is not None - curnode = self.__tree.GetSelection() - if not curnode.IsOk(): - retval = '' - else: - retval = self.__tree.GetItemText(curnode) - return retval - - def OnPopup(self) -> None: - self.AutoSize() - - # Sets the selected node in the tree - def SetValue(self, val: str) -> None: - assert self.__tree is not None - # If we haven't already added the node for this value - if val not in self.__tree_dict: - tokens = val.split('.') - curdict = self.__loc_tree - # Iterate through the tokens - if one of them doesn't appear at the - # correct level of the location tree dictionary, then this is an - # invalid value and we should stop processing. - for token in tokens: - if token not in curdict: - return - curdict = curdict[token] - # Otherwise, expand the tree from the deepest existing node until - # all necessary nodes are added - while val not in self.__tree_dict: - for i in range(len(tokens) - 1, -1, -1): - cur_path = '.'.join(tokens[:i]) - if cur_path in self.__tree_dict: - self.__tree.Expand(self.__tree_dict[cur_path]) - break - - self.__tree.SelectItem(self.__tree_dict[val]) - - # Adds every member at the top level of location dictionary "tree" to node - # "item" - def SetTree(self, - tree: LocationTree, - item: Optional[wx.TreeItemId] = None, - path: str = "") -> None: - if not item: - item = self.__root - - def cmp_pair(x: Tuple[str, LocationTree], - y: Tuple[str, LocationTree]) -> int: - # Fix missing cmp() function in Python 3 - def cmp(a: Union[int, str], b: Union[int, str]) -> int: - if isinstance(a, int): - assert isinstance(b, int) - return (a > b) - (a < b) - else: - assert isinstance(b, str) - return (a > b) - (a < b) - - return cmp(len(x[0]), len(y[0])) \ - if len(x[0]) != len(y[0]) else cmp(x[0], y[0]) - - # Sort keys by length ascending then string-comparison alphabetically - for k, v in sorted(tree.items(), - key=cmp_to_key(cmp_pair)): - if not path: - child_path = k - else: - child_path = path + '.' + k - - assert self.__tree is not None - if child_path in self.__tree_dict: - child = self.__tree_dict[child_path] - else: - child = self.__tree.AppendItem(item, child_path) - self.__tree_dict[child_path] = child - # Leaf nodes aren't expandable - if v != {}: - self.__tree.SetItemHasChildren(child, True) - - -# Location tree cell editor class -class TreeCellEditor(PopupCellEditor): - - def __init__(self, loc_tree: LocationTree) -> None: - PopupCellEditor.__init__(self, TreePopup) - self.__loc_tree = loc_tree - - def Create(self, - parent: ElementPropertyList, - id: int, - event_handler: wx.EvtHandler) -> None: - super().Create(parent, id, event_handler) - popup = self.GetPopup() - assert popup is not None - popup = cast(TreePopup, popup) - popup.SetLocationTree(self.__loc_tree) - chooser = self.GetChooser() - assert chooser is not None - chooser.Bind(wx.EVT_TEXT, self.OnTextChanged) - chooser.Bind(wx.EVT_TEXT_ENTER, self.OnTreeChanged) - - def Clone(self) -> TreeCellEditor: - return TreeCellEditor(self.__loc_tree.copy()) - - def OnTreeChanged(self, evt: wx.CommandEvent) -> None: - chooser = self.GetChooser() - assert chooser is not None - popup = self.GetPopup() - assert popup is not None - popup = cast(TreePopup, popup) - chooser.SetValue(popup.GetValue()) - self.UpdateGrid() - - def OnTextChanged(self, evt: wx.CommandEvent) -> None: - chooser = self.GetChooser() - if evt.GetEventObject() == chooser: - assert chooser is not None - popup = self.GetPopup() - assert popup is not None - popup = cast(TreePopup, popup) - popup.SetUpdateText(False) - popup.SetValue(chooser.GetTextCtrl().GetValue()) - popup.SetUpdateText(True) - else: - evt.Skip() diff --git a/helios/pipeViewer/pipe_view/gui/widgets/frame_playback_bar.py b/helios/pipeViewer/pipe_view/gui/widgets/frame_playback_bar.py deleted file mode 100644 index bd30c13ac1..0000000000 --- a/helios/pipeViewer/pipe_view/gui/widgets/frame_playback_bar.py +++ /dev/null @@ -1,1069 +0,0 @@ -from __future__ import annotations -import wx -import wx.lib.agw.hyperlink as hl -import logging -import time -from typing import Any, Optional, Tuple, Union, TYPE_CHECKING - -from ...model.clock_manager import ClockManager -from ..font_utils import ScaleFont - -if TYPE_CHECKING: - from ..layout_frame import Layout_Frame - from ...model.settings import ArgosSettings - -# @package frame_playback_bar.py -# @brief Contains FramePlaybackBar which holds all playback controls for a -# single LayoutFrame - - -# @brief A wide, stretchable panel containing playback controls for a single -# Argos Frame -class FramePlaybackBar(wx.Panel): - - # Label for start-of-database cycle number - START_TIME_FMT = 'cyc-start:{0}' - - # Label for end-of-database cycle number - END_TIME_FMT = 'cyc-end:{0}' - - # Label for tick number - CUR_TIME_FMT = 'tick:{0}' - - # Label for current-time current cycle number - CUR_CYCLE_FMT = 'cycle:{0}' - - # Range of time slider values (input resolution) - TIME_SLIDER_RANGE = 1000000 - - # Label for play button - LABEL_PLAY = 'play' - LABEL_PAUSE = 'pause' - - # Amount to step when a key is used to press the rewind or fast-forward - # buttons - COARSE_KEYPRESS_STEP = 5 - - # Default size of the query API's dynamic in-memory data window in terms of - # ticks - DB_PRELOAD_WINDOW_SIZE = 1000 - - # Rate of playback when the rewind or fast-forward key is held with the - # mouse (in current clock cycles per second) - COARSE_MOUSE_RATE = 10 - - # Maximum animated playback rate - MAX_PLAY_RATE = 1000 - - # Set up all the menus and embedded sub-menus, with all their - # bindings/callbacks - def __init__(self, frame: Layout_Frame) -> None: - self.__parent = frame - wx.Panel.__init__(self, self.__parent, wx.ID_ANY) - - context = self.__parent.GetContext() - self.__db = context.dbhandle.database - self.__qapi = context.dbhandle.api - - # Vars - # keep a list of clock instances that are referenced - self.__referenced_clocks = context.GetVisibleClocks() - self.__current_clock: Optional[ClockManager.ClockDomain] = None - self.__is_auto_clock = False - # Dummy to hold play rate when playing. If None, refers to the play - # speed spin control. Otherwise indicates an actual play speed. - # Set when starting and pausing playback - self.__play_rate: Optional[float] = None - # Timestamp of last play tick - self.__last_play_tick: Optional[float] = None - # Cumulative advance time across multiple cycles - self.__accum_play_cycle_fraction: Optional[float] = None - # Is user controlling the slider - # (True prevents auto-updates to the slider) - self.__slider_hooked = False - - # Colors - - self.__med_blue = wx.Colour(0, 0, 220) - - # Controls - # Placeholder. Will be created as needed - self.__play_timer = wx.Timer(self) - - self.__hl_start = hl.HyperLinkCtrl(self, wx.ID_ANY, label=' ', URL='') - self.__hl_start.SetToolTip( - 'Jump to first cycle in the transaction database' - ) - self.__hl_start.AutoBrowse(False) - self.__hl_end = hl.HyperLinkCtrl(self, wx.ID_ANY, label=' ', URL='') - self.__hl_end.SetToolTip( - 'Jump to final cycle in the transaction database' - ) - self.__hl_end.AutoBrowse(False) - - self.__time_slider = wx.Slider(self, wx.ID_ANY) - self.__time_slider.SetToolTip( - 'Displays the current position in the transaction database. ' - 'Slide this bar to quickly move around' - ) - # self.__time_slider.SetThumbLength(2) - # self.__time_slider.SetLineSize(1) - self.__time_slider.SetRange(0, self.TIME_SLIDER_RANGE) - self.__time_slider.SetForegroundColour(self.__med_blue) - - clocks = self.__db.clock_manager.getClocks() - clock_choices = [''] - self.__ALL_CLOCKS = 0 - for clk in clocks: - clock_choices.append(clk.name) - - self.__drop_clock = wx.ComboBox(self, choices=clock_choices, - size=(150, -1), - style=wx.CB_DROPDOWN | wx.CB_READONLY) - self.__drop_clock.SetToolTip( - 'Current clock domain. This dictates the units in which this ' - 'frame prints time. Playback and step controls will operate ' - 'on this clock domain. Select any clock domain from the list.' - ) - # self.__drop_clock.SetValue(ClockManager.HYPERCYCLE_CLOCK_NAME) - self.__drop_clock.SetSelection(0) - - curtime_msg = 'Current cycle in the current clock domain. Also ' \ - 'shows the hyper-tick (common time) tick count of ' \ - 'this frame' - - self.__static_curcycle = wx.StaticText(self, wx.ID_ANY, size=(90, -1)) - self.__static_curcycle.SetToolTip(curtime_msg) - self.__static_curcycle.SetForegroundColour(self.__med_blue) - - self.__static_curtick = wx.StaticText(self, wx.ID_ANY, size=(90, -1)) - self.__static_curtick.SetToolTip(curtime_msg) - self.__static_curtick.SetForegroundColour(self.__med_blue) - - PLAYBACK_BUTTON_WIDTH = 40 - self.__btn_rw_hold = ShyButton(self, - wx.ID_ANY, - '<<', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_rw_hold.SetToolTip( - 'Rewinds while left-mouse is held. Using space or enter to ' - 'activate this control also rewinds but rate is dictated by ' - 'keyboard repeat rate' - ) - self.__btn_back30 = ShyButton(self, - wx.ID_ANY, - '-30', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_back30.SetToolTip( - 'Steps back 30 cycles in the current clock domain' - ) - self.__btn_back10 = ShyButton(self, - wx.ID_ANY, - '-10', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_back10.SetToolTip( - 'Steps back 10 cycles in the current clock domain' - ) - self.__btn_back3 = ShyButton(self, - wx.ID_ANY, - '-3', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_back3.SetToolTip( - 'Steps back 3 cycles in the current clock domain' - ) - self.__btn_back1 = ShyButton(self, - wx.ID_ANY, - '-1', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_back1.SetToolTip( - 'Steps back 1 cycle in the current clock domain' - ) - self.__btn_forward1 = ShyButton(self, - wx.ID_ANY, - '+1', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_forward1.SetToolTip( - 'Steps forward 1 cycle in the current clock domain' - ) - self.__btn_forward3 = ShyButton(self, - wx.ID_ANY, - '+3', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_forward3.SetToolTip( - 'Steps forward 3 cycles in the current clock domain' - ) - self.__btn_forward10 = ShyButton(self, - wx.ID_ANY, - '+10', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_forward10.SetToolTip( - 'Steps forward 10 cycles in the current clock domain' - ) - self.__btn_forward30 = ShyButton(self, - wx.ID_ANY, - '+30', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_forward30.SetToolTip( - 'Steps forward 30 cycles in the current clock domain' - ) - self.__btn_ff_hold = ShyButton(self, - wx.ID_ANY, - '>>', - size=(PLAYBACK_BUTTON_WIDTH, -1)) - self.__btn_ff_hold.SetToolTip( - 'Fast-Forwards while left-mouse is held. Using space or enter to ' - 'activate this control also fast-forwards but rate is dictated by ' - 'keyboard repeat rate' - ) - - self.__txt_goto = wx.TextCtrl(self, - wx.ID_ANY, - size=(60, -1), - value='+10') - self.__txt_goto.SetToolTip( - 'Enter an absolute or relative cycle number (in the current clock ' - 'domain) to jump to. Decimal, octal (0NNN), hex (0xNNN), or ' - 'binary (0bNNN) literals are all acceptable inputs. Prefixing the ' - 'number with a + or - sign will result in this value being ' - 'interpreted as a relative value from the current cycle. Press ' - 'Enter or click "jump" to jump to the specified cycle' - ) - self.__btn_goto = ShyButton(self, wx.ID_ANY, 'jump', size=(60, -1)) - self.__btn_goto.SetToolTip( - 'Jump to the absolute or relative cycle specified in the jump ' - 'text control' - ) - - self.__static_playback_speed_units = wx.StaticText(self, - wx.ID_ANY, - 'cyc/\nsec') - self.__spin_playback_speed = wx.SpinCtrl(self, wx.ID_ANY, '1') - self.__spin_playback_speed.SetToolTip( - 'Set the number of cycles (in current clock domain) to display ' - 'per second during playback. This can be positive or negative. ' - 'A value of 0 prevents playing' - ) - self.__spin_playback_speed.SetRange(-self.MAX_PLAY_RATE, - self.MAX_PLAY_RATE) - self.__spin_playback_speed.SetValue(1) - self.__btn_playpause = ShyButton(self, - wx.ID_ANY, - self.LABEL_PLAY, - size=(45, -1)) - self.__btn_playpause.SetToolTip( - 'Automatically steps through cycles (in the current clock domain) ' - 'at the rate (positive or negative) specified by the "cyc/sec" ' - 'spin-control beside this button') - - # Setup Fonts - self.UpdateFontSize() - - # Event Bindings - - self.Bind(wx.EVT_TIMER, self.__OnPlayTimer, self.__play_timer) - self.Bind(wx.EVT_CHILD_FOCUS, self.__OnChildFocus) - - self.__hl_start.Bind(hl.EVT_HYPERLINK_LEFT, self.__OnGotoStart) - self.__hl_end.Bind(hl.EVT_HYPERLINK_LEFT, self.__OnGotoEnd) - self.__time_slider.Bind(wx.EVT_SLIDER, self.__OnTimeSlider) - self.__time_slider.Bind(wx.EVT_LEFT_DOWN, self.__OnTimeSliderLeftDown) - self.__time_slider.Bind(wx.EVT_LEFT_UP, self.__OnTimeSliderLeftUp) - self.__time_slider.Bind(wx.EVT_CHAR, self.__OnTimeSliderChar) - self.__drop_clock.Bind(wx.EVT_COMBOBOX, self.__OnClockSelect) - - self.__btn_rw_hold.Bind(wx.EVT_LEFT_DOWN, self.__OnRWButtonDown) - self.__btn_rw_hold.Bind(wx.EVT_LEFT_UP, self.__OnRWButtonUp) - self.__btn_rw_hold.Bind(wx.EVT_KEY_DOWN, self.__OnRWKeyDown) - self.__btn_back30.Bind(wx.EVT_BUTTON, self.__OnBack30) - self.__btn_back10.Bind(wx.EVT_BUTTON, self.__OnBack10) - self.__btn_back3.Bind(wx.EVT_BUTTON, self.__OnBack3) - self.__btn_back1.Bind(wx.EVT_BUTTON, self.__OnBack1) - self.__btn_back1.Bind(wx.EVT_LEFT_DOWN, self.__OnBack1LeftDown) - self.__btn_back1.Bind(wx.EVT_LEFT_UP, self.__OnBack1LeftUp) - self.__btn_forward1.Bind(wx.EVT_BUTTON, self.__OnForward1) - self.__btn_forward1.Bind(wx.EVT_LEFT_DOWN, self.__OnForward1LeftDown) - self.__btn_forward1.Bind(wx.EVT_LEFT_UP, self.__OnForward1LeftUp) - self.__btn_forward3.Bind(wx.EVT_BUTTON, self.__OnForward3) - self.__btn_forward10.Bind(wx.EVT_BUTTON, self.__OnForward10) - self.__btn_forward30.Bind(wx.EVT_BUTTON, self.__OnForward30) - self.__btn_ff_hold.Bind(wx.EVT_LEFT_DOWN, self.__OnFFButtonDown) - self.__btn_ff_hold.Bind(wx.EVT_LEFT_UP, self.__OnFFButtonUp) - self.__btn_ff_hold.Bind(wx.EVT_KEY_DOWN, self.__OnFFKeyDown) - self.__txt_goto.Bind(wx.EVT_TEXT, self.__OnChangeGotoValue) - self.__txt_goto.Bind(wx.EVT_CHAR, self.__OnGotoChar) - self.__btn_goto.Bind(wx.EVT_BUTTON, self.__OnGoto) - self.__spin_playback_speed.Bind(wx.EVT_SPINCTRL, - self.__OnPlaySpinValChange) - self.__spin_playback_speed.Bind(wx.EVT_CHAR, self.__OnPlaySpinValChar) - self.__btn_playpause.Bind(wx.EVT_BUTTON, self.__OnPlayPause) - - # Layout - - curticks = wx.BoxSizer(wx.VERTICAL) - curticks.Add(self.__static_curcycle, 1, wx.EXPAND) - curticks.Add(self.__static_curtick, 1, wx.EXPAND) - - row1 = wx.BoxSizer(wx.HORIZONTAL) - row1.Add(self.__hl_start, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT, - 2) - slider_sizer = wx.BoxSizer(wx.HORIZONTAL) - slider_sizer.Add(self.__time_slider, 1, wx.ALIGN_CENTER_VERTICAL) - row1.Add(slider_sizer, 1, wx.EXPAND) - row1.Add(self.__hl_end, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT, - 2) - - row2 = wx.FlexGridSizer(cols=6) - for i in range(5): - row2.AddGrowableCol(i) - row2.AddGrowableRow(0) - - clock_sizer = wx.FlexGridSizer(2) - clock_sizer.AddGrowableRow(0) - clock_sizer.Add(self.__drop_clock, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.SHAPED) - clock_sizer.Add(curticks, 0, wx.EXPAND | wx.LEFT, 3) - row2.Add(clock_sizer, 0, wx.ALIGN_CENTER_VERTICAL | wx.EXPAND) - - nav_sizer = wx.FlexGridSizer(10) - nav_sizer.AddGrowableRow(0) - nav_sizer.Add(self.__btn_rw_hold, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_back30, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_back10, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_back3, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_back1, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_forward1, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_forward3, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_forward10, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_forward30, 0, wx.ALIGN_CENTER_VERTICAL) - nav_sizer.Add(self.__btn_ff_hold, 0, wx.ALIGN_CENTER_VERTICAL) - row2.Add(nav_sizer, 0, wx.ALIGN_CENTER_VERTICAL | wx.EXPAND) - - line_sizer1 = wx.BoxSizer(wx.HORIZONTAL) - line_sizer1.Add(wx.StaticLine(self, wx.ID_ANY, style=wx.VERTICAL), - 0, - wx.SHAPED | wx.ALIGN_CENTER_VERTICAL) - row2.Add(line_sizer1, 0, wx.EXPAND | wx.ALIGN_CENTER_VERTICAL) - - goto_sizer = wx.FlexGridSizer(2) - goto_sizer.AddGrowableRow(0) - goto_sizer.Add(self.__txt_goto, 0, wx.ALIGN_CENTER_VERTICAL) - goto_sizer.Add(self.__btn_goto, 0, wx.ALIGN_CENTER_VERTICAL) - row2.Add(goto_sizer, 0, wx.ALIGN_CENTER_VERTICAL | wx.EXPAND) - - line_sizer2 = wx.BoxSizer(wx.HORIZONTAL) - line_sizer2.Add(wx.StaticLine(self, wx.ID_ANY, style=wx.VERTICAL), - 0, - wx.SHAPED | wx.ALIGN_CENTER_VERTICAL) - row2.Add(line_sizer2, 0, wx.EXPAND | wx.ALIGN_CENTER_VERTICAL) - - playback_sizer = wx.FlexGridSizer(2) - playback_sizer.AddGrowableRow(0) - playback_sizer.Add(self.__btn_playpause, 0, wx.ALIGN_CENTER_VERTICAL) - spinner_sizer = wx.BoxSizer(wx.HORIZONTAL) - spinner_sizer.Add(self.__static_playback_speed_units, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.LEFT, - 2) - spinner_sizer.Add(self.__spin_playback_speed, - 0, - wx.ALIGN_CENTER_VERTICAL | wx.LEFT, - 1) - playback_sizer.Add(spinner_sizer, 0, wx.ALIGN_CENTER_VERTICAL) - row2.Add(playback_sizer, 0, wx.ALIGN_RIGHT | wx.EXPAND) - - playback_controls = [ - c for c in self.GetChildren() - if not isinstance(c, wx.StaticText) and - row2.GetItem(c, recursive=True) is not None - ] - min_playback_height = max( - c.GetBestSize().GetHeight() for c in playback_controls - ) - for c in playback_controls: - orig_width, _ = c.GetMinSize() - c.SetMinSize((orig_width, min_playback_height)) - - rows = wx.BoxSizer(wx.VERTICAL) - rows.Add(row2, 0, wx.EXPAND | wx.TOP, 2) - rows.Add(wx.StaticLine(self, wx.ID_ANY), - 0, - wx.EXPAND | wx.TOP | wx.BOTTOM, - 4) - rows.Add(row1, 0, wx.EXPAND) - - self.SetSizer(rows) - - self.Fit() - self.SetAutoLayout(True) - - # Initialization - - self.__OnChangeGotoValue() # Validate initial GOTO value - self.__GetSelectedClock() - self.__Update() - - # Refreshes the content of this panel - def Refresh( - self, - eraseBackground: bool = True, - rect: Optional[Union[wx.Rect, Tuple[int, int, int, int]]] = None - ) -> None: - self.__Update() - - # Updates values displayed to match current context and selection - # @note Does not invoke FullUpdate on the associated layout canvas - def __Update(self) -> None: - # Compute cycle to display - assert self.__current_clock is not None - start_cycle = self.__current_clock.HypercycleToLocal( - self.__qapi.getFileStart() - ) - inc_end_cycle = self.__current_clock.HypercycleToLocal( - self.__qapi.getFileInclusiveEnd() - ) - hc = self.__parent.GetContext().GetHC() - cur_cycle = self.__current_clock.HypercycleToLocal(hc) - - self.__hl_start.SetLabel(self.START_TIME_FMT.format(start_cycle)) - self.__hl_end.SetLabel(self.END_TIME_FMT.format(inc_end_cycle)) - self.__static_curcycle.SetLabel(self.CUR_CYCLE_FMT.format(cur_cycle)) - self.__static_curtick.SetLabel(self.CUR_TIME_FMT.format(hc)) - - # Do not update if user is editing slider - if not self.__slider_hooked: - cyc_range = max(float(inc_end_cycle - start_cycle + 1), 0.1) - new_slider_cycle = min(1.0, (cur_cycle - start_cycle) / cyc_range) - # Do not update the slider unless the actual cycle has changed - prev_slider_cycle = self.__ComputeSliderCycle(cyc_range) - if prev_slider_cycle != new_slider_cycle: - self.__time_slider.SetValue( - int(new_slider_cycle * self.TIME_SLIDER_RANGE) - ) - - # Moves to the start cycle of this database - def __OnGotoStart(self, evt: hl.HyperLinkEvent) -> None: - hc = self.__qapi.getFileStart() - self.__parent.GetContext().GoToHC(hc) - - # Moves to the end cycle of this database - def __OnGotoEnd(self, evt: hl.HyperLinkEvent) -> None: - hc = self.__qapi.getFileEnd() - self.__parent.GetContext().GoToHC(hc) - - # Time slider being moved. - # Pauses playback - def __OnTimeSlider(self, evt: wx.ScrollEvent) -> None: - self.__PausePlaying() - - # Show current cycle - clk = self.__GetSelectedClock(auto_chosen=False) - - cur_cycle = self.__ComputeSliderLocalCycle(clk) - self.__static_curcycle.SetLabel(self.CUR_CYCLE_FMT.format(cur_cycle)) - self.__static_curtick.SetLabel(self.CUR_TIME_FMT.format( - clk.LocalToHypercycle(cur_cycle)) - ) - - # Mouse down even on time slider - def __OnTimeSliderLeftDown(self, evt: wx.MouseEvent) -> None: - evt.Skip() - - # Moves to a specific cycle of this database - # Also pauses playback - def __OnTimeSliderLeftUp(self, evt: wx.MouseEvent) -> None: - self.__PausePlaying() - clk = self.__GetSelectedClock() - cur_cycle = self.__ComputeSliderLocalCycle(clk) - - self.__slider_hooked = True - try: - hc = clk.LocalToHypercycle(cur_cycle) - - # Do not do this. It currently confuses the IntervalWindow - # Reduce preload window to 1 tick in either direction to maximize - # load speed - self.__parent.GetContext().GoToHC(hc) - except Exception: - raise - finally: - self.__slider_hooked = False - evt.Skip() - - # Handles keyboard input on the time slider - def __OnTimeSliderChar(self, evt: wx.KeyEvent) -> None: - pass # Do not forward the event - - # Updates information for new clock selection. - # Does not actually change context - def __OnClockSelect(self, evt: wx.CommandEvent) -> None: - self.__referenced_clocks = \ - self.__parent.GetContext().GetVisibleClocks() - self.__parent.Refresh() - - def __OnBack30(self, evt: wx.CommandEvent) -> None: - self.__StepBackward(30) - - def __OnBack10(self, evt: wx.CommandEvent) -> None: - self.__StepBackward(10) - - def __OnBack3(self, evt: wx.CommandEvent) -> None: - self.__StepBackward(3) - - def __OnBack1(self, evt: wx.CommandEvent) -> None: - self.__StepBackward(1) - - def __OnBack1LeftDown(self, evt: wx.MouseEvent) -> None: - self.__StartPlaying(rate=-1) - evt.Skip() - - def __OnBack1LeftUp(self, evt: wx.MouseEvent) -> None: - self.__PausePlaying() - evt.Skip() - - def __OnForward1(self, evt: wx.CommandEvent) -> None: - self.__StepForward(1) - - def __OnForward1LeftDown(self, evt: wx.MouseEvent) -> None: - self.__StartPlaying(rate=1) - evt.Skip() - - def __OnForward1LeftUp(self, evt: wx.MouseEvent) -> None: - self.__PausePlaying() - evt.Skip() - - def __OnForward3(self, evt: wx.CommandEvent) -> None: - self.__StepForward(3) - - def __OnForward10(self, evt: wx.CommandEvent) -> None: - self.__StepForward(10) - - def __OnForward30(self, evt: wx.CommandEvent) -> None: - self.__StepForward(30) - - def __OnRWButtonDown(self, evt: wx.MouseEvent) -> None: - self.__StartPlaying(rate=-self.COARSE_MOUSE_RATE) - evt.Skip() - - def __OnRWButtonUp(self, evt: wx.MouseEvent) -> None: - self.__PausePlaying() - evt.Skip() - - def __OnRWKeyDown(self, evt: wx.KeyEvent) -> None: - self.__StepBackward(self.COARSE_KEYPRESS_STEP) - evt.Skip() - - def __OnFFButtonDown(self, evt: wx.MouseEvent) -> None: - self.__StartPlaying(rate=self.COARSE_MOUSE_RATE) - evt.Skip() - - def __OnFFButtonUp(self, evt: wx.MouseEvent) -> None: - self.__PausePlaying() - evt.Skip() - - def __OnFFKeyDown(self, evt: wx.KeyEvent) -> None: - self.__StepForward(self.COARSE_KEYPRESS_STEP) - evt.Skip() - - def __OnChangeGotoValue(self, - evt: Optional[wx.CommandEvent] = None) -> None: - try: - _ = self.__GetGotoCycle() - except ValueError: - self.__btn_goto.Enable(False) - else: - self.__btn_goto.Enable(True) - - def __OnGotoChar(self, evt: wx.KeyEvent) -> None: - if evt.GetKeyCode() == wx.WXK_RETURN: - self.__OnGoto() - else: - evt.Skip() - - def __OnGoto(self, evt: Optional[wx.CommandEvent] = None) -> None: - try: - goto_cycle, relative = self.__GetGotoCycle() - except Exception: - wx.MessageBox( - f'Could not convert jump value "{self.__txt_goto.GetValue()}" ' - 'to an integer. This string must be a decimal or hexidecimal ' - '(prefixed with 0x) integer', - 'Could not jump to cycle', - style=wx.OK, - parent=self - ) - else: - clk = self.__GetSelectedClock() - cur_cycle = clk.HypercycleToLocal( - self.__parent.GetContext().GetHC() - ) - if relative == '+': - cur_cycle += goto_cycle - elif relative == '-': - cur_cycle -= goto_cycle - else: - cur_cycle = goto_cycle - - # Jump. Context must constrain - self.__parent.GetContext().GoToHC(clk.LocalToHypercycle(cur_cycle)) - self.__parent.GetCanvas().SetFocus() # go back to canvas - - # Changed spin value - # - # Enables or disables the play/pause button depending on current value - def __OnPlaySpinValChange(self, evt: wx.SpinEvent) -> None: - spin_val = self.__spin_playback_speed.GetValue() - self.__btn_playpause.Enable(spin_val != 0) - - # Keystroke on spin value - # - # Starts or stops playing if enter is pressed - def __OnPlaySpinValChar(self, evt: wx.KeyEvent) -> None: - if evt.GetKeyCode() == wx.WXK_RETURN: - self.__OnPlayPause() - else: - evt.Skip() - - # Handler for clicking the play-pause button - def __OnPlayPause(self, evt: Optional[wx.CommandEvent] = None) -> None: - if self.__btn_playpause.GetLabel() == self.LABEL_PLAY: - self.__StartPlaying() - elif self.__btn_playpause.GetLabel() == self.LABEL_PAUSE: - self.__PausePlaying() - else: - raise RuntimeError('Play button label was neither play nor pause') - - # If a ShyButton is being focused, focus the canvas instead - def __OnChildFocus(self, evt: wx.FocusEvent) -> None: - obj = evt.GetWindow() - if obj and not obj.AcceptsFocus(): - self.__parent.GetCanvas().SetFocus() - - # Helper methods - - # Steps current clock forward by a number of cycles on the current clock - # @param step Number of cycles on local clock to step forward - def __StepForward(self, step: int = 1) -> None: - # to avoid rounding errors - # not a perfect fix since playback rate will not be exact - step = int(step) - - clk = self.__GetSelectedClock() - cur_cycle = self.__GetNextCycle(clk, step) - self.__parent.GetContext().GoToHC(clk.LocalToHypercycle(cur_cycle)) - - # Steps current clock backward by a number of cycles on the current clock - # @param step Number of cycles on local clock to step backward (should be - # positive) - def __StepBackward(self, step: int = 1) -> None: - step = int(step) - clk = self.__GetSelectedClock(forward=False) - cur_cycle = self.__GetPrevCycle(clk, step) - - # because of floor and since the clock is already behind hc, add one - # @todo Stepping backward (and forward) should actually choose the - # closest clock edge-by-edge. It should never take the closest clock - # and add/subtract a number greater than 1. - self.__parent.GetContext().GoToHC(clk.LocalToHypercycle(cur_cycle)) - - # Gets the value of the goto (jump) text box and converts it to a tuple: - # (absolute_value, relative) where absolute_value is the integer value in - # the textbox and relative is the sign character extracted. If no sign - # character is found at the start of the string, the relative component - # of the result tuple will be None. - # @throw ValueError if number cannot be converted (see __StrintToInt) - # @note Supports all numeric types that __StringToInt does - def __GetGotoCycle(self) -> Tuple[int, Optional[str]]: - cyc_text = self.__txt_goto.GetValue() - cyc_text = cyc_text.strip() - if cyc_text.find('+') == 0: - relative = '+' - cyc_text = cyc_text[1:] - elif cyc_text.find('-') == 0: - relative = '-' - cyc_text = cyc_text[1:] - else: - relative = None - - cyc_text = cyc_text.strip() - return (self.__StringToInt(cyc_text), relative) - - # Computes the current cycle indicated by the slider in terms of the given - # clock. - def __ComputeSliderLocalCycle(self, clk: ClockManager.ClockDomain) -> int: - start_cycle = clk.HypercycleToLocal(self.__qapi.getFileStart()) - inc_end_cycle = clk.HypercycleToLocal( - self.__qapi.getFileInclusiveEnd() - ) - - cyc_range = max(float(inc_end_cycle - start_cycle), 0.1) - return int(self.__ComputeSliderCycle(cyc_range)) - - # Computes the current cycle based on the slider position interpolated - # within the given cycle range tuple representing the range of cycles - # addressable by the slider - def __ComputeSliderCycle(self, cyc_range: float) -> int: - # Scale into [0,1) range - portion = self.__time_slider.GetValue() / float(self.TIME_SLIDER_RANGE) - assert self.__current_clock is not None - return int( - (portion * cyc_range) + - self.__current_clock.HypercycleToLocal(self.__qapi.getFileStart()) - ) - - # Gets the current Clock object selected by the dropdown - def __GetSelectedClock( - self, - forward: bool = True, - auto_chosen: bool = False - ) -> ClockManager.ClockDomain: - # Find the current clock - clocks = self.__db.clock_manager.getClocks() - hc = self.__parent.GetContext().GetHC() - clock_selection = self.__drop_clock.GetCurrentSelection() - - if clock_selection == self.__ALL_CLOCKS: - auto_chosen = True - clk = self.__db.clock_manager.getClosestClock( - hc, - self.__referenced_clocks, - forward - ) - else: - auto_chosen = False - # bumps forward - clk = clocks[self.__drop_clock.GetCurrentSelection() - 1] - self.__current_clock = clk - self.__is_auto_clock = auto_chosen - return clk - - # Sets up timer to start playing with given rate in cycles/sec. - # @param rate Rate to attempt to refresh in current-clock-cycles/sec. - # If rate==None, uses the rate from self.__spin_playback_speed control - def __StartPlaying(self, - rate: Optional[float] = None, - delay: int = 0) -> None: - self.__play_rate = rate - self.__last_play_tick = time.time() + delay - self.__accum_play_cycle_fraction = 0 - self.__btn_playpause.SetLabel(self.LABEL_PAUSE) - self.__UpdatePlayTimer(delay=delay) - - # Handle clicking on the pause button - def __PausePlaying(self) -> None: - self.__play_timer.Stop() - self.__play_rate = None - self.__last_play_tick = None - self.__accum_play_cycle_fraction = None - - # Update label if not already play. Causes flicker if set to same label - if self.__btn_playpause.GetLabel() != self.LABEL_PLAY: - self.__btn_playpause.SetLabel(self.LABEL_PLAY) - - # Handler for __play_timer event - def __OnPlayTimer(self, evt: wx.TimerEvent) -> None: - # Determine play rate - if self.__play_rate is not None: - play_rate = self.__play_rate - else: - play_rate = self.__spin_playback_speed.GetValue() - - # Determine step size based on rate - assert self.__last_play_tick is not None - cur_time = time.time() - if cur_time < self.__last_play_tick: - # Someone moved __last_play_tick ahead. This timer callback could - # have happened when someone was trying to cancel it, resulting in - # an extra event (e.g. two steps-forward when arrow-right was - # pressed but not held). Simply ignore this event. - msg = 'Playback not advancing' - logging.getLogger('FramePlayback').debug(msg) - return - - delta_time = cur_time - self.__last_play_tick - self.__last_play_tick = cur_time - # How many ticks should be advanced in this time delta for this - # play_rate - advance_raw = play_rate * delta_time - - if play_rate > 0: - assert self.__accum_play_cycle_fraction is not None - self.__accum_play_cycle_fraction += advance_raw - if self.__accum_play_cycle_fraction > 1: # Greater than threshold - advance = int(self.__accum_play_cycle_fraction) - self.__accum_play_cycle_fraction %= 1 # Fractional portion - else: - advance = 0 - pass # Do no update - elif play_rate < 0: - assert self.__accum_play_cycle_fraction is not None - self.__accum_play_cycle_fraction += advance_raw - if self.__accum_play_cycle_fraction < -1: - advance = int(self.__accum_play_cycle_fraction) - self.__accum_play_cycle_fraction %= -1 # Fractional portion - else: - advance = 0 - pass # Do no update - else: - assert 0, 'Invalid play rate of 0' - - # Move to next (back or forward) cycles - context = self.__parent.GetContext() - if advance < -1: - self.__StepBackward(-advance) - if context.GetHC() <= self.__qapi.getFileStart(): - # Pause if we've exceeded the duration of the data. - self.__PausePlaying() - else: - self.__UpdatePlayTimer() - self.__parent.Update() # Force redraw - elif advance >= 1: - self.__StepForward(advance) - if context.GetHC() >= self.__qapi.getFileInclusiveEnd(): - # Pause if we've reached the duration of the data. - self.__PausePlaying() - else: - self.__UpdatePlayTimer() - # print 'UPDATING', self.__parent - # import pdb; pdb.set_trace() - # Force redraw NOW. Refresh does not cause redraws - self.__parent.Update() - else: - self.__UpdatePlayTimer() - # No advance because it was too small. - # self.__accum_play_cycle_fraction was incremented above - pass - - logging.getLogger('FramePlayback').debug( - 'Playback cycle advance=%s advance_frac=%s dt=%s cur_t=%s', - advance, - self.__accum_play_cycle_fraction, - delta_time, - cur_time - ) - - # Updates the play timer value based on the play speed spin ctrl - # @pre Do not invoke if spin ctrl value is 0 - def __UpdatePlayTimer(self, delay: int = 0) -> None: - # Determine play rate - if self.__play_rate is not None: - play_rate = self.__play_rate - else: - play_rate = self.__spin_playback_speed.GetValue() - - # Clamp play_rate to be safe - play_rate = max(-self.MAX_PLAY_RATE, - min(self.MAX_PLAY_RATE, play_rate)) - - # Start up a new timer with current play speed if possible - try: - cps = float(play_rate) - except Exception: - self.__PausePlaying() - wx.MessageBox( - f'Could not convert cycles/second string "{play_rate}" to a ' - 'float. This string must be a decimal floating poing or ' - 'integer value', - 'Could play', - style=wx.OK, - parent=self - ) - else: - if cps == 0: - self.__PausePlaying() - else: - - # Actually play (time in milliseconds) - next_timer = abs(1000.0 / cps) + delay * 1000 - logging.getLogger('FramePlayback').debug( - 'Playback cps=%s delay=%s next_timer=%s', - cps, - delay, - next_timer - ) - - # Delay scheduling of timer and always schedule immediately. - # This lets us redraw as fast as possible. May need to - # specially handle case __OnPlayTimer wher the tick advances - # less than 1 cycle due to quick refresh - # - # TODO: Compute proper refresh rate based on observed timer - # speed while properly considering redraw time cost between the - # most recent timer event and now. - def TimerStarter() -> None: - # If not paused/stopped - if self.__last_play_tick is not None: - self.__play_timer.Start(10, oneShot=True) - - wx.CallAfter(TimerStarter) - - # Converts a string to an integer allowing for hex, binary, and octal - # prefixes - # @note Does NOT support negative or positive prefixes - # @throw ValueError if number cannot be converted - def __StringToInt(self, num_str: str) -> int: - num_str = num_str.strip() - if (num_str.find('-') == 0) or (num_str.find('+') == 0): - raise ValueError('__StringToInt does not expect a -/+ prefix') - if num_str.find('0x') == 0 or num_str.find('0X') == 0: - num = int(num_str, 16) - elif num_str.find('0b') == 0 or num_str.find('0b') == 0: - num = int(num_str, 2) - elif num_str.find('0') == 0 or num_str.find('0') == 0: - num = int(num_str, 8) - else: - num = int(num_str, 10) - - return num - - # Returns the current cycle in the selected clock domain - def __GetCurrentCycle(self, clk: ClockManager.ClockDomain) -> int: - return clk.HypercycleToLocal(self.__parent.GetContext().GetHC()) - - # Determines the next clock cycle within the database file extents. - # @param step Size of forward step (should be a positive number) - # @return an integer with the next local cycle if valid. This may be - # the same as the current value if the current value is at the right - # edge of the database transaction range. - def __GetNextCycle(self, - clk: ClockManager.ClockDomain, - step: int = 1) -> int: - cur = self.__GetCurrentCycle(clk) - next = step + cur - return next - - # Determines the prev clock cycle within the database file extents. - # @param step Size of backward step (should be a positive number) - # @return an integer with the next local cycle if valid. This may be - # the same as the current value if the current value is at the left - # edge of the database transaction range. - def __GetPrevCycle(self, - clk: ClockManager.ClockDomain, - step: int = 1) -> int: - cur = self.__GetCurrentCycle(clk) - prev = cur - step - return prev - - # Steps current clock forward by a number of cycles on the current clock - # @param step Number of cycles on local clock to step forward - # Called by external element - def StepForward(self, step: int = 1) -> None: - self.__StepForward(step) - - # Steps current clock backward by a number of cycles on the current clock - # @param step Number of cycles on local clock to step backward (should be - # positive) - # Called by external element - def StepBackward(self, step: int = 1) -> None: - self.__StepBackward(step) - - # Used by input decoder when arrow key is held down - def StartPlaying(self, step: int = 1, delay: int = 0) -> None: - self.__StartPlaying(step, delay=delay) - - # Used by input decoder when arrow key is released - # Public interface to pause playing (from Frame) - # @note This exists to stop playing when the owning frame is trying to - # close using a wx.CallAfter but playback is not allowing the event queue - # to completely drain - def PausePlaying(self) -> None: - self.__PausePlaying() - - # Attempt to select the given clock by name - def SetDisplayClock(self, - clock_name: str, - error_if_not_found: bool = True) -> bool: - cn = clock_name.lower() - for idx, item in enumerate(self.__drop_clock.GetItems()): - if item.lower() == cn: - self.__drop_clock.SetSelection(idx) - self.__parent.Refresh() # Allow update of the cycle printout - return True - - if error_if_not_found: - clock_items = ', '.join(x for x in self.__drop_clock.GetItems()) - raise IndexError( - f'No clock known for frame "{self.__parent.GetTitle()}" ' - f'with (case insensitive) name "{clock_name}". ' - f'Options are: {clock_items}' - ) - - return False - - # To to a specific cyle on the currently displayed clock for this frame - def GoToCycle(self, cycle: int) -> None: - assert cycle is not None - clk = self.__GetSelectedClock() - assert clk is not None - self.__parent.GetContext().GoToHC(clk.LocalToHypercycle(cycle)) - - # Sets focus to the jump-to-time text entry box - def FocusJumpBox(self) -> None: - self.__txt_goto.SetFocus() - - # Gets the global settings object - def GetSettings(self) -> ArgosSettings: - return self.__parent.GetSettings() - - # Updates font sizes for all of the controls - def UpdateFontSize(self) -> None: - self.__fnt_tiny = wx.Font( - ScaleFont(self.GetSettings().playback_font_size), - wx.NORMAL, - wx.NORMAL, - wx.NORMAL - ) - self.__fnt_bold_med = wx.Font( - ScaleFont(self.GetSettings().playback_font_size), - wx.NORMAL, - wx.NORMAL, - wx.BOLD - ) - self.__fnt = wx.Font( - ScaleFont(self.GetSettings().playback_font_size), - wx.NORMAL, - wx.NORMAL, - wx.NORMAL - ) - self.__fnt_hl = self.__fnt.Underlined() - - self.SetFont(self.__fnt) - self.__drop_clock.SetFont(self.__fnt) - self.__static_curcycle.SetFont(self.__fnt_bold_med) - self.__static_curtick.SetFont(self.__fnt_tiny) - self.__hl_start.SetFont(self.__fnt_hl) - self.__hl_end.SetFont(self.__fnt_hl) - self.__btn_rw_hold.SetFont(self.__fnt) - self.__btn_back30.SetFont(self.__fnt) - self.__btn_back10.SetFont(self.__fnt) - self.__btn_back3.SetFont(self.__fnt) - self.__btn_back1.SetFont(self.__fnt) - self.__btn_forward1.SetFont(self.__fnt) - self.__btn_forward3.SetFont(self.__fnt) - self.__btn_forward10.SetFont(self.__fnt) - self.__btn_forward30.SetFont(self.__fnt) - self.__btn_ff_hold.SetFont(self.__fnt) - self.__txt_goto.SetFont(self.__fnt) - self.__btn_goto.SetFont(self.__fnt) - self.__static_playback_speed_units.SetFont(self.__fnt) - self.__spin_playback_speed.SetFont(self.__fnt) - self.__btn_playpause.SetFont(self.__fnt) - - self.Layout() - - -# A button which does not accept focus -class ShyButton(wx.Button): - - def __init__(self, *args: Any, **kwargs: Any) -> None: - wx.Button.__init__(self, *args, **kwargs) - - # overriding this method should do the trick, but although it doesn't work - # we use this method to identify shy button objects and de-focus them if - # they get focused - def AcceptsFocus(self) -> bool: - return False diff --git a/helios/pipeViewer/pipe_view/gui/widgets/location_entry.py b/helios/pipeViewer/pipe_view/gui/widgets/location_entry.py deleted file mode 100644 index 42db0655e3..0000000000 --- a/helios/pipeViewer/pipe_view/gui/widgets/location_entry.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations -from typing import Any, List, Optional - -import wx - -from ...model.location_manager import LocationTree - - -# An Auto-suggesting Box for Entry of Location Strings -class LocationEntry(wx.ComboBox): - def __init__(self, - parent: wx.Panel, - value: str, - location_tree: LocationTree, - style: int = 0, - **kwargs: Any) -> None: - wx.ComboBox.__init__(self, - parent, - wx.ID_ANY, - value, - style=style | wx.CB_DROPDOWN, - choices=[], - **kwargs) - self.__location_tree = location_tree - self.AppendItems(self.GetChoices(value)) - self.Bind(wx.EVT_TEXT, self.EvtText) - - def GetChoices(self, string: str) -> List[str]: - keys = string.split('.') - current_level: Optional[LocationTree] = self.__location_tree - last_level = keys.pop() - base = '' - while keys and current_level: - key = keys.pop(0) - if base: - base += '.' + key - else: - base = key - current_level = current_level.get(key) - if current_level is None: - return [] - else: - if not base: - return list(current_level.keys()) - else: - return_list = [] - for p_entry in current_level.keys(): - if p_entry.startswith(last_level): - return_list.append(base + '.' + p_entry) - return return_list - - def EvtText(self, event: wx.CommandEvent) -> None: - currentText = event.GetString() - self.Clear() - self.AppendItems(self.GetChoices(currentText)) diff --git a/helios/pipeViewer/pipe_view/gui/widgets/transaction_list.py b/helios/pipeViewer/pipe_view/gui/widgets/transaction_list.py deleted file mode 100644 index 4ae4826732..0000000000 --- a/helios/pipeViewer/pipe_view/gui/widgets/transaction_list.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations -from typing import Any, Dict, List, Tuple, Union, TYPE_CHECKING -import wx -from ..font_utils import GetMonospaceFont - -if TYPE_CHECKING: - from ..layout_canvas import Layout_Canvas - from ..dialogs.search_dlg import SearchResult - -TransactionListBaseEntry = Dict[str, Any] -TransactionListEntry = Union['SearchResult', TransactionListBaseEntry] - - -# This class is a GUI list control element that shows transactions. -class TransactionList(wx.ListCtrl): - def __init__(self, - parent: wx.Frame, - canvas: Layout_Canvas, - name: str = '') -> None: - wx.ListCtrl.__init__(self, - parent=parent, - id=wx.NewId(), - name=name, - style=wx.LC_REPORT | wx.SUNKEN_BORDER) - self.SetFont(GetMonospaceFont(canvas.GetSettings().layout_font_size)) - - # used for coloring - self.__canvas = canvas - # list of dictionary of properties - self.__transactions: List[TransactionListEntry] = [] - # properties to show. - self.__properties: Tuple[str, ...] = ('start', # must have at least 1 - 'location', - 'annotation') - # insertion point for elements at end - self.__current_new_idx = 0 - self.__colorize = True - self.RefreshAll() - - def GetProperties(self) -> Tuple[str, ...]: - return tuple(self.__properties) - - def SetProperties(self, properties: Tuple[str, ...]) -> None: - self.__properties = properties - self.RefreshAll() - - def Clear(self) -> None: - self.__transactions = [] - - def Colorize(self, colorize: bool) -> None: - self.__colorize = colorize - - # Destroy all graphical elements and recreate them - def RefreshAll(self) -> None: - self.ClearAll() - # make header - width = int(self.GetClientSize()[0] / len(self.__properties)) - if width < 100: - width = 100 - for col_idx, column in enumerate(self.__properties): - self.InsertColumn(col_idx, column) - self.SetColumnWidth(col_idx, width) - self.__current_new_idx = 0 - for transaction in self.__transactions: - self.__AddGraphicalTransaction(transaction) - - def RefreshTransaction(self, index: int) -> None: - transaction = self.__transactions[index] - for col_idx, prop in enumerate(self.__properties): - self.SetItem(index, col_idx, str(transaction.get(prop))) - annotation = transaction.get('annotation') - if self.__colorize and annotation: - color = self.__canvas.GetAutocolorColor(annotation) - else: - color = wx.Colour(255, 255, 255) - self.SetItemBackgroundColour(index, color) - - def GetTransaction(self, index: int) -> TransactionListEntry: - return self.__transactions[index] - - def __AddGraphicalTransaction(self, - transaction: TransactionListEntry) -> None: - self.InsertItem(self.__current_new_idx, - str(transaction.get(self.__properties[0]))) - self.RefreshTransaction(self.__current_new_idx) - self.__current_new_idx += 1 - - # Add a new element to bottom of list - # New item must be dictionary of properties - def Add(self, transaction_properties: TransactionListEntry) -> int: - self.__transactions.append(transaction_properties) - self.__AddGraphicalTransaction(transaction_properties) - return self.__current_new_idx - 1 - - def Remove(self, index: int) -> None: - del self.__transactions[index] - self.DeleteItem(index) - self.__current_new_idx -= 1 - - # Attempts to resize the columns based on size - # @pre All content should be added before this is called (once) - def FitColumns(self) -> None: - for idx, _ in enumerate(self.__properties): - self.SetColumnWidth(idx, wx.LIST_AUTOSIZE) - self.Layout() diff --git a/helios/pipeViewer/pipe_view/logsearch/.gitignore b/helios/pipeViewer/pipe_view/logsearch/.gitignore deleted file mode 100644 index b798b4762a..0000000000 --- a/helios/pipeViewer/pipe_view/logsearch/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/logsearch.cpp diff --git a/helios/pipeViewer/pipe_view/logsearch/__init__.pyi b/helios/pipeViewer/pipe_view/logsearch/__init__.pyi deleted file mode 100644 index 25ac02cb54..0000000000 --- a/helios/pipeViewer/pipe_view/logsearch/__init__.pyi +++ /dev/null @@ -1,11 +0,0 @@ -# Stub definitions for the logsearch library -# Cython isn't very good at generating these automatically yet - -from typing import Any - -_DUMMY: str - -class LogSearch: - BAD_LOCATION: Any - def __init__(self, *args: Any) -> None: ... - def getLocationByTick(self, tick: int, earlier_loc: int = 0) -> int: ... diff --git a/helios/pipeViewer/pipe_view/logsearch/src/log_search.cpp b/helios/pipeViewer/pipe_view/logsearch/src/log_search.cpp deleted file mode 100644 index 2eafb5c39b..0000000000 --- a/helios/pipeViewer/pipe_view/logsearch/src/log_search.cpp +++ /dev/null @@ -1,101 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -/*! - * \brief Helper for quickly scanning through a SPARTA log file based on - * the "{TICK CYCLE ..." line format. - * - * Example - * \code - * LogSearch s(myfile); - * uint64_t tick = 12345678; - * uint64_t loc = s.getLocationByTick(tick); - * if(loc == s.BAD_LOCATION){ - * // do nothing - * }else{ - * // open myfile, seekg to loc, and then readline() until done - * } - * loc = s.getLocationByTick(tick + 10, loc); // Can starting from previous file loc to save time. - * // reject BAD_LOCATION return value, read from file, repeat above... - * \endcode - * - * \todo This should support reading lines directly from the file as well - */ -class LogSearch -{ - std::ifstream in_; - uint64_t file_bytes_ = 0; - -public: - - static const uint64_t BAD_LOCATION = std::numeric_limits::max(); - - explicit LogSearch(const std::string& filename) - : in_(filename) - { - file_bytes_ = 0; - if(in_.is_open()){ - in_.seekg(0, in_.end); - const auto len = in_.tellg(); - if(len >= 0){ - file_bytes_ = static_cast(len); - } - } - } - - uint64_t getLocationByTick(uint64_t tick, uint64_t earlier_location=0) - { - // Early out for no file or empty file - if(in_.is_open() == false || file_bytes_ == 0){ - return BAD_LOCATION; - } - - const uint32_t BUFFER_SIZE = 512; - char buffer[BUFFER_SIZE]; - - if(earlier_location >= file_bytes_){ - return BAD_LOCATION; - } - - in_.seekg((std::streampos)earlier_location); - - uint64_t last_line_pos = earlier_location; // Assume at start of line - bool got_newline = true; - while(in_.good()){ - char chr; - in_.get(chr); - if(chr == '\n'){ - last_line_pos = in_.tellg(); - got_newline = true; - continue; - } - // If prev char was newline - if(got_newline){ - if(chr == '{'){ - // Parse out the tick value - char* bufptr = buffer; - while(!in_.eof()){ - in_.get(chr); - if(chr == ' '){ - break; - } - *bufptr = chr; - bufptr++; - } - const uint64_t tickval = strtoull(buffer, &bufptr, 10); - if(tickval >= tick){ - return last_line_pos; // Found a line containing the chosen tick or later! - } - } - } - got_newline = false; - } - - return BAD_LOCATION; - } -}; diff --git a/helios/pipeViewer/pipe_view/logsearch/src/logsearch.pyx b/helios/pipeViewer/pipe_view/logsearch/src/logsearch.pyx deleted file mode 100644 index ec995c736b..0000000000 --- a/helios/pipeViewer/pipe_view/logsearch/src/logsearch.pyx +++ /dev/null @@ -1,51 +0,0 @@ - - -## @package Argos search for SPARTA logs - -import sys - -from common cimport * -from libc.stdlib cimport strtoul - -_DUMMY="dummy" - -cimport cython - -cdef extern from "log_search.cpp": - - cdef cppclass c_LogSearch "LogSearch": - c_LogSearch(string filename) - - uint64_t getLocationByTick(uint64_t tick, uint64_t earlier_location) - - uint64_t BAD_LOCATION - -cdef class LogSearch: - - cdef c_LogSearch* c_logsearch - - def __cinit__(self, filename): - self.c_logsearch = new c_LogSearch(filename) - - def __init__(self, *args): - pass - - def __dealloc__(self): - del self.c_logsearch - - property BAD_LOCATION: - "Represents bad result from getLocationByTick" - def __get__(self): - return self.c_logsearch.BAD_LOCATION - - def __str__(self): - return '' - - def __repr__(self): - return self.__str__() - - def getLocationByTick(self, long tick, long earlier_loc=0): - assert self.c_logsearch != NULL, 'Null logsearch object within wrapper. It may never have been allocated' - - return self.c_logsearch.getLocationByTick(tick, earlier_loc) - diff --git a/helios/pipeViewer/pipe_view/misc/__init__.py b/helios/pipeViewer/pipe_view/misc/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/misc/version.py b/helios/pipeViewer/pipe_view/misc/version.py deleted file mode 100644 index 04fd7d3be1..0000000000 --- a/helios/pipeViewer/pipe_view/misc/version.py +++ /dev/null @@ -1,10 +0,0 @@ -# @package version.py -# @brief Argos version finder - -from __future__ import annotations -from .. import core # Argos core - - -# Attempts to determine the version of this argos by its .VERSION file -def get_version() -> int: - return core.get_argos_version() diff --git a/helios/pipeViewer/pipe_view/model/__init__.py b/helios/pipeViewer/pipe_view/model/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helios/pipeViewer/pipe_view/model/clock_manager.py b/helios/pipeViewer/pipe_view/model/clock_manager.py deleted file mode 100644 index 250a41193a..0000000000 --- a/helios/pipeViewer/pipe_view/model/clock_manager.py +++ /dev/null @@ -1,300 +0,0 @@ -# @package clock_manager.py -# @brief Consumes argos clock files through ClockManager class - -from __future__ import annotations -import math -from typing import Dict, List, Optional, TextIO, Tuple, Union - - -# Consumes an Argos clock file and provides a means of lookup up clock info -# via clock IDs -# -# Also allows browsing of availble clocks -class ClockManager: - # Clock domain owned by a ClockManager - class ClockDomain: - def __init__(self, - clk_id: int, - clk_name: str, - hc_tick_period: int, - hc_ratio_num: int, - hc_ratio_denom: int) -> None: - self.__clk_id = clk_id - self.__clk_name = clk_name - self.__hc_tick_period = hc_tick_period - self.__hc_ratio_num = hc_ratio_num - self.__hc_ratio_denom = hc_ratio_denom - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() - - @property - def ID(self) -> int: - return self.__clk_id - - @property - def name(self) -> str: - return self.__clk_name - - # Period of this clock in hypercycle ticks - @property - def tick_period(self) -> int: - return self.__hc_tick_period - - # Convert a cycle in terms of this clock domain into a hypercycle tick - # count - # @note The input to this should never be used as a delta and the - # output should not be used as a relative cycle because inaccuracies - # can accumulate. - # @param local_cycle Cycle number on this clock domain - # @return integer number of hypercycle ticks elapsed up to - # \a local_cycle. This is ABSOLUTE cycles. - # Relative cycles are computed differently - def LocalToHypercycle(self, local_cycle: int) -> int: - assert local_cycle is not None - # Absolute cycle counts begin at 0. - return int(math.floor((local_cycle) * self.__hc_tick_period)) - - # Convert a cycle in terms of hypercycle ticks into number of cycles on - # this clock domain. - # @note The input to this should never be used as a delta and the - # output should not be used as a relative cycle because inaccuracies - # can accumulate. - # @param hc Hypercycle tick count. - # @return integer number of cycles on this clock domain - def HypercycleToLocal(self, hc: int) -> int: - assert hc is not None - # Absolute cycle counts begin at 0. - return int(math.floor(hc / self.__hc_tick_period)) - - # With a given hypercycle, compute a hypercycle that represents the - # next local cycle. - # @param hc Hypercycle to use for computation - # @return hypercycle representing (local cycle) where local_cycle - # is computed by the \a hc argument. - # - # Implemented as LocalToHypercycle(HypercycleToLocal(hc)) - def NextLocalCycle(self, hc: int) -> int: - assert hc is not None - return self.LocalToHypercycle(self.HypercycleToLocal(hc)) - - # Describes the clock file - CLOCK_FILE_EXTENSION = 'clock.dat' - - # Expeted version number from file. Otherwise the data cannot be consumed - VERSION_NUMBER = 1 - - # ID associated with built-in hypercycle clock - HYPERCYCLE_CLOCK_ID = 0 - - # Name associated with built-in hypercycle clock - HYPERCYCLE_CLOCK_NAME = 'ticks' - - # Constructor - # @param prefix Argos transaction database prefix to open. - # CLOCK_FILE_EXTENSION will be appended to determine the actual - # filename to open - # @note Prevents duplicate names - def __init__(self, prefix: str) -> None: - # { ClockID : ClockDomain } - self.__clocks: Dict[int, ClockManager.ClockDomain] = {} - # { Clock name : ClockID } - self.__clock_names: Dict[str, int] = {} - # [ClockDomain0, ClockDomain1, ... ] - self.__clock_list: List[ClockManager.ClockDomain] = [] - self.__hc_frequency = 0 # Frequency of hypercycle ticks in actual Hz - - with open(prefix + self.CLOCK_FILE_EXTENSION, 'r') as f: - - # Find version information - while 1: - first = self.__findNextLine(f) - assert first is not None - if first == '': - continue - - try: - els = first.split(' \t#') - ver = int(els[0]) - except Exception: - raise ValueError( - 'Found an unparsable (non-integer) version number ' - f'string: "{first}". Expected "{self.VERSION_NUMBER}".' - ) - - if ver != self.VERSION_NUMBER: - raise ValueError( - f'Found incorrect version number: "{ver}". Expected ' - f'"{self.VERSION_NUMBER}". This reader may need to be ' - 'updated' - ) - break - - # Find hypercycle tick frequency information - # - while 1: - first = self.__findNextLine(f) - assert first is not None - if first == '': - continue - - try: - self.__hc_frequency = int(first) - except Exception: - raise ValueError( - 'Found an unparsable (non-integer) frequency string: ' - f'"{first}".' - ) - - self.__addClockDomain(self.HYPERCYCLE_CLOCK_ID, - self.HYPERCYCLE_CLOCK_NAME, - 1, - 1, - 1) - break - - # Read subsequent location lines - # ,,,, # noqa: E501 - while 1: - ln = self.__findNextLine(f) - if ln == '': - continue - if ln is None: - break - - els = ln.split(',') - - try: - uid_str, name, period_str, rat_num_str, rat_denom_str = els[:5] # noqa: E501 - except Exception: - raise ValueError(f'Failed to parse line "{ln}"') - - uid = int(uid_str) - period = int(period_str) - rat_num = int(rat_num_str) - rat_denom = int(rat_denom_str) - - self.__addClockDomain(uid, name, period, rat_num, rat_denom) - - # Checks if a ClockDomain object with the given name exists. - # @param clock_name Name of clock to lookup. - # @return Whether clock name is present - def doesClockNameExist(self, clock_name: str) -> bool: - if not isinstance(clock_name, str): - raise TypeError( - f'clock_name must be a string, is a {type(clock_name)}' - ) - return clock_name in self.__clock_names - - # Gets a ClockDomain object with the given name. - # @param clock_name Name of clock to lookup. - # @throw KeyError if name is not found in this manager. - # @return ClockDomain object - def getClockDomainByName(self, - clock_name: str) -> ClockManager.ClockDomain: - if not isinstance(clock_name, str): - raise TypeError( - f'clock_name must be a string, is a {type(clock_name)}' - ) - # Throw KeyError if not found - return self.getClockDomain(self.__clock_names[clock_name]) - - # Gets a ClockDomain object associated with the given clock ID. - # @param clock_id ID of clock to lookup. - # @throw KeyError if ID is not found in this manager. - # @return ClockDomain object - def getClockDomain(self, clock_id: int) -> ClockManager.ClockDomain: - if not isinstance(clock_id, int): - raise TypeError( - f'clock_id must be an integer, is a {type(clock_id)}' - ) - return self.__clocks[clock_id] # Throw KeyError if not found - - # Gets sequence of all clock domains in no particular order - # @note This is slow because this list is generated on request - # @note Order of results is guaranteed to be consistent - # @return List of ClockDomain instances representing all known - # clock domains - def getClocks(self) -> Tuple[ClockManager.ClockDomain, ...]: - # Convert to tuple to prevent modification to internal list - return tuple(self.__clock_list) - - # Of all clocks, which is closest to our current time - def getClosestClock(self, - hc: int, - clocks: Optional[Union[Tuple[int, ...], List[int]]], - forward: bool = True) -> ClockManager.ClockDomain: - if not clocks: - return self.__clock_list[0] - closest_clock = self.__clocks[clocks[0]] - if not forward: - hc -= 1 - min_divergence = hc % closest_clock.tick_period - if forward: - min_divergence = closest_clock.tick_period-min_divergence - for clk_id in clocks: - if clk_id == -1: - continue - clk = self.__clocks[clk_id] - delta = hc % clk.tick_period - if forward: - delta = clk.tick_period-delta - if delta < min_divergence: - min_divergence = delta - closest_clock = clk - return closest_clock - - # Adds a clock domain with given parameters. - # @note Refer to ClockDomain class for parameter semantics - # @throw If any clock domain arguments are unacceptable for ClockDomain or - # if \a clkname has already been added to this manager - def __addClockDomain(self, - clkid: int, - clkname: str, - hc_period: int, - rat_num: int, - rat_denom: int) -> None: - if clkname in self.__clock_names: - raise ValueError( - f'clkname "{clkname}" is already present in this clock ' - 'manager. It cannot be re-added' - ) - cd = self.ClockDomain(clkid, clkname, hc_period, rat_num, rat_denom) - self.__clock_names[clkname] = clkid - self.__clocks[clkid] = cd - self.__clock_list.append(cd) - - # Gets next line from file which is not a comment. - # Strips comments on the line and whitespace from each end - # @param f File to read next line from - # @return '' if line is empty or comment, None if EOF is reached - def __findNextLine(self, f: TextIO) -> Optional[str]: - ln = f.readline().strip() - if ln == '': - return None - - if ln.find('#') == 0: - return '' - - pos = ln.find('#') - if pos >= 0: - ln = ln[:pos] - - ln = ln.strip() - - return ln - - def __len__(self) -> int: - return len(self.__clocks) - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() diff --git a/helios/pipeViewer/pipe_view/model/content_options.py b/helios/pipeViewer/pipe_view/model/content_options.py deleted file mode 100644 index bd33540990..0000000000 --- a/helios/pipeViewer/pipe_view/model/content_options.py +++ /dev/null @@ -1,286 +0,0 @@ -# @package contentoptions -# This module exists to provide a centralized listing of all content options -# an Element can display, combined with descriptions of those content -# options and the logic to extract the specified content from a transaction - -# Transaction type strings known to the viewer -from __future__ import annotations -from typing import (Any, - Callable, - Dict, - List, - Optional, - Tuple, - cast, - TYPE_CHECKING) - -if TYPE_CHECKING: - from .database import Transaction - from .database_handle import DatabaseHandle - from .element import Element - - -TRANSACTION_TYPES = ['Annotation', 'Instruction', 'MemoryOp'] - - -# These methods are used to actually acquire the data specified by the -# various content options -# -# TODO: Implement checks on each of these Get* methods to see if there is -# valid data, otherwise return a string corresponding to 'no data' from -# __DISPLAY_STATES -def GetType(trans: Transaction, e: Element, *args: Any) -> str: - return TRANSACTION_TYPES[trans.getType()] - - -def GetStart(trans: Transaction, e: Element, *args: Any) -> int: - return trans.getLeft() - - -def GetEnd(trans: Transaction, e: Element, *args: Any) -> int: - return trans.getRight() - - -def GetTransactionID(trans: Transaction, - e: Element, - *args: Any) -> Optional[int]: - return trans.getTransactionID() - - -def GetLocation(trans: Transaction, e: Element, *args: Any) -> str: - return cast(str, e.GetProperty('LocationString')) - - -def GetLocationTrunc(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> str: - return dbhandle.database.location_manager.replaceLocationVariables( - GetLocation(trans, e), - loc_vars - ).split('.')[-1] - - -def GetLocationID(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> int: - return dbhandle.database.location_manager.getLocationInfo( - GetLocation(trans, e), - loc_vars - )[0] - - -def __check_optional_int(val: Optional[int]) -> int: - assert val is not None - return val - - -def GetFlags(trans: Transaction, *args: Any) -> str: - return hex(__check_optional_int(trans.getFlags())) - - -def GetParent(trans: Transaction, *args: Any) -> Optional[int]: - return trans.getParentTransactionID() - - -def GetOpcode(trans: Transaction, *args: Any) -> str: - return hex(__check_optional_int(trans.getOpcode())) - - -def GetVAddr(trans: Transaction, *args: Any) -> str: - return hex(__check_optional_int(trans.getVirtualAddress())) - - -def GetRAddr(trans: Transaction, *args: Any) -> str: - return hex(__check_optional_int(trans.getRealAddress())) - - -def GetAnnotation(trans: Transaction, *args: Any) -> Optional[str]: - return trans.getAnnotation() - - -def GetCaption(trans: Transaction, e: Element, *args: Any) -> str: - return cast(str, e.GetProperty('caption')) - - -def GetImage(*args: Any) -> str: - return '' - - -def GetClock(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> str: - clock_id = dbhandle.database.location_manager.getLocationInfo( - GetLocation(trans, e), - loc_vars - )[2] - if clock_id != dbhandle.database.location_manager.NO_CLOCK: - clk = dbhandle.database.clock_manager.getClockDomain(clock_id) - return clk.name - return "#NO! " - - -def GetCycle(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> str: - clock_id = dbhandle.database.location_manager.getLocationInfo( - GetLocation(trans, e), loc_vars - )[2] - if clock_id != dbhandle.database.location_manager.NO_CLOCK: - clk = dbhandle.database.clock_manager.getClockDomain(clock_id) - return str(clk.HypercycleToLocal(tick)) - return "#NO! " - - -def GetStartCycle(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> str: - return GetCycle(trans, e, dbhandle, trans.getLeft(), loc_vars, *args) - - -def GetEndCycle(trans: Transaction, - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str], - *args: Any) -> str: - return GetCycle(trans, e, dbhandle, trans.getRight(), loc_vars, *args) - - -def GetTickDuration(trans: Transaction, *args: Any) -> int: - return trans.getRight() - trans.getLeft() - - -def IsContinued(trans: Transaction, e: Element, *args: Any) -> bool: - return trans.isContinued() - - -# The dictionary with all the content options, how to get the specified -# content, and a description of each content option. Add additional content -# options in this dictionary, and write the related function above -__CONTENT_PROC: Dict[str, Tuple[Callable, str]] = { - 'type': (GetType, - "Whether the transaction is an annotation, " - "instruction, or memory_op"), - 'start': (GetStart, "Start time in ticks"), - 'end': (GetEnd, "End time in ticks"), - 'start_cycle': (GetStartCycle, - "Start time in location's clock domain"), - 'end_cycle': (GetEndCycle, - "End time in location's clock domain"), - 'transaction': (GetTransactionID, "Transaction ID"), - 'loc': (GetLocation, "Location Name (string)"), - 'loc_id': (GetLocationID, "Internal Location ID (int)"), - 'truncated_location': (GetLocationTrunc, - "Location Name Truncated (string)"), - 'flags': (GetFlags, "Flags from transaction"), - 'parent': (GetParent, "Parent Transaction ID"), - 'opcode': (GetOpcode, - "instruction opcode (\'-\' if transaction is " - "a memory op"), - 'vaddr': (GetVAddr, - "instruction/mem-op virtual address"), - 'paddr': (GetRAddr, - "instruction/mem-op physical address"), - 'annotation': (GetAnnotation, - "Show annotation selected by \'annotation\' " - "property"), - 'auto_color_annotation': (GetAnnotation, - "Show annotation selected by \'annotation\' " - "property colored based on seq ID"), - 'auto_color_anno_notext': (GetAnnotation, - "Color element based on seq ID, but do not " - "display annotation"), - 'auto_color_anno_nomunge': (GetAnnotation, - "Color element based on seq ID, but do not " - "munge text"), - 'caption': (GetCaption, - "Caption (static text) from Element " - "\'caption\' property"), - 'image': (GetImage, - "Image (currently hardcoded to nothing)"), - 'clock': (GetClock, - "Name of clock associated with this location"), - 'cycle': (GetCycle, - "Current Cycle of clock associated with this " - "location"), - 'tick_duration': (GetTickDuration, - "Duration of this transaction in ticks"), - 'continued': (IsContinued, - "Whether the transaction is continued over a " - "heartbeat interval") -} - -# The dictionary with all display states and the string which will be used -# to override the val of an Element Value -__DISPLAY_STATES = { - 'normal': '', # Display whatever belongs inside the element - 'no trans': '', # Display empy space - 'no data': '#NO! ', # Display a MS Excel-style "#NO! ContentOption" Where - # ContentOption is some inappropriate value for - # content for a transaction - 'no loc': '?' # Display when the element has a location string that does - # not refer to an actual location in the locations file -} - -# A listing of the content options which do not require a transaction -# from a stabbing query in order to be determined. Use a dictionary for faster -# lookup in the critical path -NO_TRANSACTIONS_REQUIRED = {'caption': None, - 'clock': None, - 'loc': None, - 'loc_id': None, - 'truncated_location': None, - 'cycle': None} - -# A listing of the content options which do not require a database at all. -# Use a dictionary for faster lookup in the critical path -NO_DATABASE_REQUIRED = {'caption': None} - -# Check NO_DATABASE_REQUIRED table -for x in NO_DATABASE_REQUIRED: - assert x in NO_TRANSACTIONS_REQUIRED, \ - 'All content types specified in NO_DATABASE_REQUIRED must also be ' \ - f'found in NO_TRANSACTIONS_REQUIRED. "{x}" violated this' - - -def OverrideState(key: str) -> str: - return __DISPLAY_STATES[key] - - -# This method is used by an Ordered Dictionary to populate it's -# Element_Value's with the data -def ProcessContent(content: str, - trans: Optional[Transaction], - e: Element, - dbhandle: DatabaseHandle, - tick: int, - loc_vars: Dict[str, str]) -> Any: - return __CONTENT_PROC[content][0](trans, e, dbhandle, tick, loc_vars) - - -# Returns the string describing the specified content option -def ContentDescription(content: str) -> str: - return __CONTENT_PROC[content][1] - - -__CONTENT_OPTIONS = list(__CONTENT_PROC.keys()) - - -# Returns a listing of the available content options -def GetContentOptions() -> List[str]: - return __CONTENT_OPTIONS diff --git a/helios/pipeViewer/pipe_view/model/database.py b/helios/pipeViewer/pipe_view/model/database.py deleted file mode 100644 index b7ab5bf2fc..0000000000 --- a/helios/pipeViewer/pipe_view/model/database.py +++ /dev/null @@ -1,126 +0,0 @@ -# @package database.py -# @brief Consumes argos database files based on prefix - -from __future__ import annotations -import logging -from logging import info, error -from types import ModuleType -from typing import Any, Dict, Optional - -from .location_manager import LocationManager -from .clock_manager import ClockManager - -try: - from .. import transactiondb -except ImportError as e: - error('Argos failed to import module: "transactiondb"') - raise e - -# Inform users which transactiondb interface they are getting -info('Using transactiondb at "%s"', transactiondb.__file__) - -Transaction = transactiondb.Transaction -TransactionDatabase = transactiondb.TransactionDatabase - - -# Consumes an Argos database and creates a location manager, a clock manager, -# and a transaction database API -# -# Also allows browsing of database static structure as well as creating -# handles to database transaction info. -# -# The static database structure exposed through this class and its children is -# can be shared between many DatabaseHandle instances -class Database: - - # Constructor - # @param prefix Argos transaction database prefix to open. - # Transaction database file extensions will be appended to determine the - # actual filenames to open - def __init__(self, prefix: str, update_enabled: bool) -> None: - - self.__filename = prefix - self.__loc_mgr = LocationManager(prefix, update_enabled) - self.__clk_mgr = ClockManager(prefix) - - # stores extra data shared between objects across layout contexts using - # the same database - self.__metadata: Dict[str, Dict[str, Any]] = {} - self.__metadata_tick: Optional[int] = None - - # Note that this will need to move if multiple layout contexts access - # the same database sporadically - logging.getLogger('Database').debug( - 'Database %s about to open query API', - self - ) - self.__dbapi = transactiondb.TransactionDatabase( - self.filename, - 1 + self.location_manager.getMaxLocationID(), - update_enabled - ) - logging.getLogger('Database').debug( - 'Database opened with node length %s, heartbeat size %s', - self.__dbapi.getNodeLength(), - self.__dbapi.getChunkSize() - ) - - # Gets the database implementation module - @property - def dbmodule(self) -> ModuleType: - return transactiondb - - # Query API (TransactionDatabase) - @property - def api(self) -> transactiondb.TransactionDatabase: - return self.__dbapi - - # ClockManager object containing all clocks in this database - @property - def clock_manager(self) -> ClockManager: - return self.__clk_mgr - - # LocationManager object containing all locations in this database - @property - def location_manager(self) -> LocationManager: - return self.__loc_mgr - - # Filename (prefix) of the database referenced by this object - @property - def filename(self) -> str: - return self.__filename - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() - - # Returns the tick at which the current metadata was written. - # This is used to determine whether to keep or purge the metadata during - # an element_set Update - def GetMetadataTick(self) -> Optional[int]: - return self.__metadata_tick - - # Sets the metadata tick. See GetMetadataTick - def SetMetadataTick(self, tick: int) -> None: - self.__metadata_tick = tick - - # Returns values stored under a string 'name' - # One current use: coloring and generated info on graph nodes - def GetMetadata(self, objname: str) -> Optional[Dict[str, Any]]: - return self.__metadata.get(objname) - - def DumpMetadata(self) -> Dict[str, Dict[str, Any]]: - return self.__metadata - - def PurgeMetadata(self) -> None: - self.__metadata.clear() - - # Add or update specified key-value pair(s) - def AddMetadata(self, objname: str, data: Dict[str, Any]) -> None: - old_data = self.__metadata.get(objname) - if old_data: - old_data.update(data) - else: - self.__metadata[objname] = data diff --git a/helios/pipeViewer/pipe_view/model/database_handle.py b/helios/pipeViewer/pipe_view/model/database_handle.py deleted file mode 100644 index 9c79891ed0..0000000000 --- a/helios/pipeViewer/pipe_view/model/database_handle.py +++ /dev/null @@ -1,79 +0,0 @@ -# @package database_handle.py -# @brief Structure that associates an ISL API and a Database instance - -from __future__ import annotations -import logging -from typing import Callable - -from .database import Database, TransactionDatabase - - -# Associates a database with a query method and some result caching -# -# Also allows browsing of database static structure as well as creating -# handles to database transaction info. -# -# The static database structure exposed through this class and its children is -# can be shared between many DatabaseHandle instances -class DatabaseHandle: - - # Constructor - # @param db argos Database instance - def __init__(self, db: Database) -> None: - - if not isinstance(db, Database): - raise TypeError( - f'db must be an instance of database.Database, is a {type(db)}' - ) - - self.__db = db - self.__dbapi = self.__db.api - - if logging.root.isEnabledFor(logging.DEBUG): - self.__dbapi.setVerbose(True) - - # Associated database object which provides access to filename, clocks, - # and locations - # - # Transaction information refering to the locations and clockos in this - # database is available through the 'api' attribute - @property - def database(self) -> Database: - return self.__db - - def query(self, - start_inc: int, - end_inc: int, - cb: Callable, - mod_tracking: bool = True) -> None: - assert end_inc >= start_inc, \ - 'Query range must be ordered such that start <= end' - - # Handles queries with negative starts by invoking the callback for - # each one because the update algorithm requires a callback at each - # tick - if start_inc < 0: - # Allow fake callbacks to be made here which contain no - # transactions - self.__dbapi.clearCurrentTickContent() - for t in range(start_inc, min(0, end_inc + 1)): - cb(t, self.__dbapi) - - # Warning. Do NOT clamp range here. Update algorithm expects callbacks - # through entire range. Do skip over negative absolute times - query_start_inc = max(0, start_inc) - if end_inc >= 0: - self.__dbapi.query(query_start_inc, - end_inc, - cb, - modify_tracking=mod_tracking) - - @property - def api(self) -> TransactionDatabase: - return self.__dbapi - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() diff --git a/helios/pipeViewer/pipe_view/model/element.py b/helios/pipeViewer/pipe_view/model/element.py deleted file mode 100644 index 58f3e83e67..0000000000 --- a/helios/pipeViewer/pipe_view/model/element.py +++ /dev/null @@ -1,1115 +0,0 @@ -from __future__ import annotations - -import os -import re -import sre_constants -from typing import (Any, - Callable, - Dict, - List, - Optional, - TextIO, - Tuple, - TypeVar, - Union, - cast, - TYPE_CHECKING) -import weakref -import yaml - -from . import element_propsvalid as valid - -# Though this is part of the model-side, it is easier to place some -# drawing routines here - there is a precedent for that already -# anyway -import wx - -# Another view-side import since elements here have rendering code embedded. -from ..gui import autocoloring - -from ..logsearch import LogSearch # Argos module for searching logfiles - -if TYPE_CHECKING: - from .layout import Layout - from .element_value import Element_Value - from ..gui.layout_canvas import Layout_Canvas - -T = TypeVar('T') -PropertyValue = Optional[Union[str, - int, - float, - Tuple[int, int], - Tuple[float, float], - Tuple[int, int, int], - List['PropertyValue']]] -PropertyDict = Dict[str, PropertyValue] -ValidatedPropertyDictElement = Tuple[T, Callable] -ValidatedPropertyDict = Dict[str, ValidatedPropertyDictElement[PropertyValue]] - - -# The building blocks of a Layout -class Element: - __FRAME_COLOR = (128, 128, 128) - - # Set the default values for each property of an Element - _ALL_PROPERTIES: ValidatedPropertyDict = { - 'type': ('', valid.validateString), - 'position': ((10, 7), valid.validatePos), - 'dimensions': ((120, 14), valid.validateDim), - 'color': (__FRAME_COLOR, valid.validateColor), - 't_offset': (0, valid.validateOffset), # Clock cycles, not HCs - 'caption': ('', valid.validateString), - 'on_update': ('', valid.validateString), - 'on_init': ('', valid.validateString), - 'on_cycle_changed': ('', valid.validateString), - # This is a virtual property that enables auto-scaling layouts to fit - # arbitrary font sizes - 'scale_factor': ((1, 1), valid.validateScale), - } - - # Properties that get auto-scaled with scale_factor - _SCALED_PROPERTIES = set(('position', 'dimensions')) - - # Properties that should never be written to a layout file - _VIRTUAL_PROPERTIES = set(('scale_factor',)) - - # Name of the property to use as a metadata key - _METADATA_KEY_PROPERTY = '' - - # Additional metadata properties that should be associated with this - # element type - _AUX_METADATA_PROPERTIES: List[str] = [] - - __CONTENT_OPTIONS = valid.GetContentOptions() - - __cur_pin = -1 - - # Store PIN in root class - @staticmethod - def Gen_PIN() -> int: - Element.__cur_pin += 1 - return Element.__cur_pin - - # called to see if we element should be considered for rendering - # Currently there is no 'ghost' type - @staticmethod - def IsDrawable() -> bool: - return False - - # called to see if we element will be making queries - @staticmethod - def NeedsDatabase() -> bool: - return False - - # called to see if we element is selectable in playback mode - @staticmethod - def IsSelectable() -> bool: - return True - - # called to see if we element will gather data from additional'meta' - # table of data from database - @staticmethod - def UsesMetadata() -> bool: - return False - - # called when drawn is not null - @staticmethod - def GetDrawRoutine() -> Optional[Callable]: - return None - - # indicates if this is a possible container - # if it is then the following methods must be added: - # GetChildren(self) -> [Element, ...] - # AddChild(self, Element) - # RemoveChild(self, Element) - @staticmethod - def HasChildren() -> bool: - return False - - @staticmethod - def GetElementProperties() -> ValidatedPropertyDict: - return Element._ALL_PROPERTIES - - @staticmethod - def GetType() -> str: - return '' - - # returns a list of property names (keys) to exclude from - # element property dialog - @staticmethod - def GetHiddenProperties() -> List[str]: - return ['scale_factor'] - - # returns a list of property names (keys) to mark read-only in - # element property dialog - @staticmethod - def GetReadOnlyProperties() -> List[str]: - return [] - - # The constructor - # if an Element is passed in, then a duplicate is created, otherwise the - # default values are loaded - # @param initial_properties Properties dictionary that overrides any - # defaults - # @param force_pin Force the PIN to be a specific value. Caller is - # responsible for prevenging layout duplicates - # @profile - # Must be run by subclass implementing _ALL_PROPERTIES static varible - def __init__(self, - duplicate: Optional[Element] = None, - container: Optional[Layout] = None, - initial_properties: Optional[PropertyDict] = None, - parent: Optional[Element] = None, - force_pin: Optional[int] = None) -> None: - self._parent = parent - self.__draw = True - self.__needs_redraw = True - self._properties = {} - self.__pin = force_pin if force_pin is not None else self.Gen_PIN() - if force_pin is not None and container is not None: - assert container.FindByPIN(force_pin) is None, \ - f'forced PIN {force_pin} already exists in layout' - - # layout needs to be None to begin with, to allow for proper sorting - # in Layout & Layout Context the first time after the other - # properties are set - self._layout = None - - if duplicate is None: - for key in self._ALL_PROPERTIES: - self._properties[key] = self._ALL_PROPERTIES[key][0] - - if initial_properties: - # Overwrite defaults with any initial settings we have. - for key in initial_properties.keys(): - if key in self._VIRTUAL_PROPERTIES: - raise Exception( - f'Property {key} is a virtual property and cannot ' - 'be initialized' - ) - - prop = self._ALL_PROPERTIES.get(key) - if prop: - self._properties[key] = prop[1]( - key, - initial_properties[key] - ) - elif key == 'annotation': - # Suppress error, but also further generation of this - # element unused--this property is dynamically - # generated. this code can be removed when annotation - # fields are completely phased out. - # -N.S. 06/21/13 - pass - else: - raise Exception( - f'Trying to add unknown property type: {key}' - ) - else: - self._properties = duplicate._properties.copy() - - # Unescape input. This must be symmetric with _GetYAMLEvents - for k, v in self._properties.items(): - if isinstance(v, str): - self._properties[k] = v.replace('\\"', '"').replace('\\n', '\n').replace('\\r', '\r') # noqa: E501 - - # These need to be set after the properties are set - self._layout = container - - # At end of construction, mark as clean - self.__changed = False - - pen_colors = cast(Tuple[int, int, int], self._properties['color']) - self._pen = wx.Pen([int(c) for c in pen_colors], 1) - - # Return the unique identifier of this element - def GetPIN(self) -> int: - return self.__pin - - # Return parent for this object - def GetParent(self) -> Optional[Element]: - return self._parent - - # Return the layout which this element belongs to - def GetLayout(self) -> Optional[Layout]: - return self._layout - - def GetBounds(self) -> Tuple[int, int, int, int]: - pos = cast(Tuple[int, int], self.GetProperty('position')) - size = cast(Tuple[int, int], self.GetProperty('dimensions')) - return pos[0], pos[1], pos[0] + size[0], pos[1] + size[1] - - # Set a flag that the element needs to be redrawn - def SetNeedsRedraw(self) -> None: - self.__needs_redraw = True - - # Unset the redraw flag - def UnsetNeedsRedraw(self) -> None: - self.__needs_redraw = False - - # Query the redraw flag - def NeedsRedraw(self) -> bool: - return self.__needs_redraw - - # Set the Element's property key to the value val, if it passes the - # validation effort - # @param key Property to change - # @param val New value for the property. Note that val can always be a - # string regardless of property type - def SetProperty(self, key: str, val: PropertyValue) -> None: - if key not in self._properties: - raise ValueError( - f'Attempting to set a non-existent property: {key}' - ) - - orig_val = self._properties[key] - - props = self._ALL_PROPERTIES[key] - val = props[1](key, val) - if (key == 'caption'): - self._properties[key] = val - assert self._layout is not None - self._layout.Refresh(self) - elif key == 'position' or key == 'dimensions': - self._properties[key] = val - assert self._layout is not None - self._layout.ElementsMoved(self) - self._layout.Refresh(self) - elif key == 'color': - val = cast(Tuple[int, int, int], val) - self._properties[key] = val - self._pen = wx.Pen([int(c) for c in val], 1) - else: - self._properties[key] = val - - # Setting a property is a change if the new value is actually different - if val != orig_val and key != 'scale_factor': - self.__changed = True - assert self._layout is not None - self._layout.SetChanged() - - def SetX(self, x: int) -> None: - current = cast(Tuple[int, int], self._properties['position']) - orig_val = current[0] - val = self._ALL_PROPERTIES['position'][1]('position', (x, current[1])) - self._properties['position'] = val - assert self._layout is not None - self._layout.ElementsMoved(self) - - # Setting a property is a change if the new value is actually different - if val[0] != orig_val: - self.__changed = True - self._layout.SetChanged() - - def SetY(self, y: int) -> None: - current = cast(Tuple[int, int], self._properties['position']) - orig_val = current[1] - val = self._ALL_PROPERTIES['position'][1]('position', (current[0], y)) - self._properties['position'] = val - assert self._layout is not None - self._layout.ElementsMoved(self) - - # Setting a property is a change if the new value is actually different - if val[0] != orig_val: - self.__changed = True - self._layout.SetChanged() - - # Fetch the value for the given key - def GetProperty(self, - key: str, - period: Optional[int] = None) -> PropertyValue: - val = self._properties[key] - - if key not in self._SCALED_PROPERTIES: - return val - - if key == 'position' or key == 'dimensions': - val = cast(Tuple[int, int], val) - x_scale, y_scale = cast(Union[Tuple[float, float], Tuple[int, int]], self._properties['scale_factor']) # noqa: E501 - return (round(val[0] * x_scale), round(val[1] * y_scale)) - else: - raise NotImplementedError( - f"Scaling not implemented for property '{key}'." - ) - - # These shortcut functions were added in order to improve performance for - # some derived types - # Shortcut to get X dimension - def GetXDim(self) -> int: - return cast(Tuple[int, int], self.GetProperty('dimensions'))[0] - - # Shortcut to get Y dimension - def GetYDim(self) -> int: - return cast(Tuple[int, int], self.GetProperty('dimensions'))[1] - - # Shortcut to get X position - def GetXPos(self) -> int: - return cast(Tuple[int, int], self.GetProperty('position'))[0] - - # Shortcut to get Y position - def GetYPos(self) -> int: - return cast(Tuple[int, int], self.GetProperty('position'))[1] - - # Return the entire dict of properties - def GetProperties(self) -> PropertyDict: - return self._properties - - # Return the entire dict of properties - def GetSerializableProperties(self) -> PropertyDict: - return { - k: v for k, v in self._properties.items() - if k not in self._VIRTUAL_PROPERTIES - } - - # Does this element have a particular property - def HasProperty(self, key: str) -> bool: - return key in self._properties - - # Return a listing of the valid options for the 'Content' property - def GetContentOptions(self) -> List[str]: - return self.__CONTENT_OPTIONS - - # Override and return offset times to query range of data at update. - def GetQueryFrame(self, period: int) -> Optional[Tuple[int, int]]: - return None - - # Indicates whether this element has been modified in any way that could - # affect the save state on disk - # @return True if the element has changed - # @see _MarkAsUnchanged - def HasChanged(self) -> bool: - return self.__changed - - # Mark the element for update. - def SetChanged(self) -> None: - self.__changed = True - - # Sets if element should be drawn. - # Useful after a parent has handled the draw for the children. - def EnableDraw(self, state: bool) -> None: - self.__draw = state - - # Returns if draw is enabled on this element. - def ShouldDraw(self) -> bool: - return self.__draw - - def GetChildren(self) -> List[Element]: - raise NotImplementedError('GetChildren not implemented for this class') - - # Append YAML content of this element to a YAML event list - # @pre Assumes this element can just append a self-contained YAML - # block-style map right here (no leading map key/scalar or - # sequence start events will be emitted). - # @param events Llist of YAML events to which new events should be - # appended describing this Element. - # @return None - # - # Intended to be called by a Layout during saving - def _GetYAMLEvents(self) -> List[yaml.Event]: - events: List[yaml.Event] = [] - events.append(yaml.MappingStartEvent(anchor=None, - tag=None, - implicit=True, - flow_style=False)) - - # Serialize all properties to yaml pairs - sorted_keys = sorted(k for k in self._ALL_PROPERTIES.keys() - if k not in self._VIRTUAL_PROPERTIES) - for k in sorted_keys: - if k == 'children' and self.HasChildren(): - events.append(yaml.ScalarEvent(anchor=None, - tag=None, - implicit=(True, True), - value='children')) - events.append(yaml.SequenceStartEvent(anchor=None, - tag=None, - implicit=(True, True))) - for child in self.GetChildren(): - events.extend(child._GetYAMLEvents()) - - events.append(yaml.SequenceEndEvent()) - else: - val = str(self._properties[k]) - # Escape input. This must be symmetric with __init__ when - # loading initial properties - val = val.replace('"', '\\"').replace('\n', '\\n').replace('\r', '\\r') # noqa: E501 - - events.extend([ - yaml.ScalarEvent(anchor=None, - tag=None, - implicit=(True, True), - value=str(k)), - yaml.ScalarEvent(anchor=None, - tag=None, - implicit=(True, True), - value=val), - ]) - - events.append(yaml.MappingEndEvent()) - return events - - # Flags this layout and all comprising elements as Not Changed. - # This is intended to be called by the owning Layout - def _MarkAsUnchanged(self) -> None: - self.__changed = False - - def LocationHasVars(self) -> bool: - return False - - # Will return a hash of the Elements PIN, which is a unique identifier - # for each Element (and therefore each Element Value) - def __hash__(self) -> int: - return hash(self.GetPIN()) - - # Determine if two Element instances are equivalent, as determined by - # sharing the same PIN or the other element being None. - # Comparison with other types is not allowed - def __eq__(self, other: object) -> bool: - if not isinstance(other, Element): - return False - - return self.GetPIN() == other.GetPIN() - - -Element._ALL_PROPERTIES['type'] = (Element.GetType(), valid.validateString) - - -# Generic element capable of holding other elements. -class MultiElement(Element): - _MULTI_PROPERTIES: ValidatedPropertyDict = { - 'children': ([], valid.validateList) - } - _ALL_PROPERTIES = Element._ALL_PROPERTIES.copy() - _ALL_PROPERTIES.update(_MULTI_PROPERTIES) - - @staticmethod - def HasChildren() -> bool: - return True - - def __init__(self, *args: Any, **kwargs: Any) -> None: - Element.__init__(self, *args, **kwargs) - # store child elements - self.__children: List[Element] = [] - # map PINs to child elements - self.__children_by_pin: Dict[int, Element] = {} - - @staticmethod - def GetHiddenProperties() -> List[str]: - # we use pixel_offset instead - return ['scale_factor', 'children'] - - def GetChildren(self) -> List[Element]: - return self.__children - - def GetChildByPIN(self, pin: int) -> Optional[Element]: - return self.__children_by_pin.get(pin, None) - - def RemoveChild(self, e: Element) -> None: - self.__children.remove(e) - del self.__children_by_pin[e.GetPIN()] - - def AddChild(self, e: Element) -> None: - self.__children.append(e) - self.__children_by_pin[e.GetPIN()] = e - - def SetNeedsRedraw(self) -> None: - for child in self.GetChildren(): - child.SetNeedsRedraw() - - -# Element that gets queries from database. -class LocationallyKeyedElement(Element): - _LOC_KEYED_PROPERTIES = { - 'LocationString': ('top', valid.validateLocation), - # See content_options.py - 'Content': ('loc', valid.validateContent), - 'auto_color_basis': ('', valid.validateString), - # needs better validator eventually - 'color_basis_type': ('string_key', valid.validateString), - 'tooltip': ('', valid.validateString), - } - - _ALL_PROPERTIES = Element._ALL_PROPERTIES.copy() - _ALL_PROPERTIES.update(_LOC_KEYED_PROPERTIES) - COLOR_BASIS_TYPES = ['string_key', 'python_exp', 'python_func'] - - @staticmethod - def NeedsDatabase() -> bool: - return True - - __brush_cache_needs_purging = False - - def __init__(self, *args: Any, **kwargs: Any) -> None: - Element.__init__(self, *args, **kwargs) - self.__has_vars = False - # keep track of state among all LocationallyKeyed Elements - - def SetProperty(self, key: str, val: PropertyValue) -> None: - Element.SetProperty(self, key, val) - val = self.GetProperty(key) - if (key == 'LocationString' or key == 't_offset') \ - and (self._layout is not None and self._layout.HasContext()): - if key == 'LocationString': - loc = cast(str, val) - # This allows us to avoid using an expensive regex later on if - # there aren't any variables in the location string - if '{' in loc: - self.__has_vars = True - t_off = cast(int, self._properties['t_offset']) - else: - t_off = cast(int, val) - loc = cast(str, self._properties['LocationString']) - self._layout.ReSort(self, t_off, loc) - self._properties[key] = val - self._layout.Refresh(self) - elif (key == 'Content'): - assert self._layout is not None - self._properties[key] = val - self._layout.Refresh(self) - elif key == 'auto_color_basis' or key == 'color_basis_type': - self.__brush_cache_needs_purging = True - - def SetBrushesWerePurged(self) -> None: - self.__brush_cache_needs_purging = False - - def BrushCacheNeedsPurging(self) -> bool: - return self.__brush_cache_needs_purging - - # Return if the location string has any variables - def LocationHasVars(self) -> bool: - return self.__has_vars - - # Debug purposes only - def __str__(self) -> str: - return f"<{self._properties['type']} " \ - f"element: loc={self._properties['LocationString']} " \ - f"toff={self._properties['t_offset']}>" - - # Debug purposes only - def __repr__(self) -> str: - return cast(str, self._properties['LocationString']) + \ - ' ' + \ - cast(str, self._properties['Content']) - - -# For purposes of consistency, we don't use the base type Element directly. -class BoxElement(LocationallyKeyedElement): - - @staticmethod - def GetType() -> str: - return 'box' - - @staticmethod - def IsDrawable() -> bool: - return True - - -BoxElement._ALL_PROPERTIES['type'] = (BoxElement.GetType(), - valid.validateString) - - -# For purposes of consistency, we don't use the base type Element directly. -class ImageElement(LocationallyKeyedElement): - - # Manages loaded images - # @note This is required as an optimization for undo/redo/scale operations - # so that images needn't be reloaded. - class ImageManager: - - class BitmapOriginalPair: - - def __init__(self, - bmp_wref: weakref.ReferenceType[wx.Bitmap], - original_img: wx.Image) -> None: - self.bmp_wref = bmp_wref - self.original_img = original_img - - def __init__(self) -> None: - self.__originals: weakref.WeakValueDictionary = \ - weakref.WeakValueDictionary() # {(filename): wx.Image} - # {(filename,(w,h): (wx.Bitmap, original wx.Image)} - self.__bmps: Dict[Tuple[str, Tuple[int, int]], ImageElement.ImageManager.BitmapOriginalPair] = {} # noqa: E501 - - # A bit map being tracked expired. Free it - def __OnBmpExpire(self, - wref: weakref.ReferenceType[wx.Bitmap]) -> None: - print(wref) - for k, v in self.__bmps.items(): - if v.bmp_wref == wref: - del self.__bmps[k] - break - - # Get an item from the manager, loading if necessary - # @param info Info about the item to load including filename and - # dimensions as nested tuples: (filename, (w,h)) - def __getitem__( - self, - info: Tuple[str, Tuple[int, int]] - ) -> Optional[wx.Bitmap]: - # See if a scaled image matching the info exists already - bmp_and_img = self.__bmps.get(info) - if bmp_and_img is not None: - # Use the existing scaled bitmap. Will expedite copy/paste - # undo/redo - return bmp_and_img.bmp_wref() - else: - pass - - # Unpack args for new image - filename, dims = info - w, h = dims - - # See if a full-size image exists - img = self.__originals.get(filename) - - if img is None: - bmp = wx.Bitmap(filename, wx.BITMAP_TYPE_ANY) - if not bmp.IsOk(): - raise IOError(f'Bitmap {filename} could not be loaded') - img = wx.ImageFromBitmap(bmp) - self.__originals[filename] = img - - # Scale the new image - img2 = img.Scale(w, h, wx.IMAGE_QUALITY_HIGH) - bmp = wx.BitmapFromImage(img2) - - # Track in list - self.__bmps[info] = self.BitmapOriginalPair(weakref.ref(bmp), img) - - return bmp - - _IMAGE_PROPERTIES = {'filename': ('logo.png', valid.validateString)} - - _ALL_PROPERTIES = LocationallyKeyedElement._ALL_PROPERTIES.copy() - _ALL_PROPERTIES.update(_IMAGE_PROPERTIES) - _ALL_PROPERTIES['dimensions'] = ((135, 135), - _ALL_PROPERTIES['dimensions'][1]) - - # Static image manager/loader for ImageElements per process - _IMAGE_MANAGER = ImageManager() - - @staticmethod - def GetType() -> str: - return 'image' - - @staticmethod - def IsDrawable() -> bool: - return True - - # called when drawn is not null - @classmethod - def GetDrawRoutine(cls) -> Callable: - return cls.DrawRoutine - - def __init__(self, *args: Any, **kwargs: Any) -> None: - LocationallyKeyedElement.__init__(self, *args, **kwargs) - - self.__image: Optional[wx.Bitmap] = None - self.__loaded_filename: Optional[str] = None - - self.ScaleImage(cast(str, self.GetProperty('filename')), - cast(Tuple[int, int], self.GetProperty('dimensions'))) - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int, - time_range: Optional[Tuple[int, int]] = None, - render_box: Optional[Tuple[int, int, int, int]] = None, - fixed_offset: Optional[Tuple[int, int]] = None) -> None: - - border_color = cast(Tuple[int, int, int], self.GetProperty('color')) - pen = wx.Pen(border_color, 1) # TODO: Use a pen cache - dc.SetPen(pen) - - (x, y) = cast(Tuple[int, int], self.GetProperty('position')) - (w, h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - xoff, yoff = canvas.GetRenderOffsets() - if not fixed_offset: - (x, y) = (x - xoff, y - yoff) - else: - x = x - fixed_offset[0] - y = y - fixed_offset[1] - - brush = wx.Brush((0, 0, 0), style=wx.SOLID) # TODO: use a brush cache - dc.SetBrush(brush) - - dc.SetBackground(wx.Brush('WHITE')) - - if self.__image is not None: - dc.DrawBitmap(self.__image, int(x), int(y)) - else: - # Draw border and X through box - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - dc.DrawLine(int(x), int(y), int(x + w), int(y + h)) - dc.DrawLine(int(x + w), int(y), int(x), int(y + h)) - - self.UnsetNeedsRedraw() - - def SetProperty(self, key: str, val: PropertyValue) -> None: - LocationallyKeyedElement.SetProperty(self, key, val) - - val = self.GetProperty(key) - if key == 'filename': - val = cast(str, val) - self.ScaleImage( - val, - cast(Tuple[int, int], self.GetProperty('dimensions')) - ) - if key == 'dimensions': - val = cast(Tuple[int, int], val) - self.ScaleImage(cast(str, self.GetProperty('filename')), val) - - # Scale image, reloading if necessary - def ScaleImage(self, filename: str, dims: Tuple[int, int]) -> None: - imgpath = self.__ChooseImage(filename) - - if imgpath is None or not os.path.exists(imgpath): - print('Image does not exist in known resource dirs: ' - f'"{filename}"->"{imgpath}". Use -R to specify others') - layout = self.GetLayout() - assert layout is not None - ws = layout.GetWorkspace() - print('Resource dirs:') - if ws is not None: - for rd in self.__GetResourceDirs(): - print(f' {rd}') - else: - print(' None') - - self.__image = None - self.__loaded_filename = imgpath - return - - try: - self.__image = self._IMAGE_MANAGER[(imgpath, dims)] - except Exception as ex: - print(f'Error loading bitmap: "{imgpath}": {ex}') - self.__images = None - - self.__loaded_filename = imgpath - - def __GetResourceDirs(self) -> List[str]: - dirs = [] - - layout = self.GetLayout() - assert layout is not None - - lfn = layout.GetFilename() - # import pdb; pdb.set_trace() - import traceback - traceback.print_stack() - # print self.GetLayout() - print(lfn) - if lfn is not None: - dirs.append(os.path.dirname(lfn)) # Use layout file path - - ws = layout.GetWorkspace() - if ws is not None: - dirs.extend(ws.GetResourceResolutionList()) # Get resource paths - - return dirs - - # Determines a path to an image given a relative path to this session's - # resource directories - def __ChooseImage(self, filename: str) -> Optional[str]: - layout = self.GetLayout() - assert layout is not None - lfn = layout.GetFilename() - ws = layout.GetWorkspace() - if ws is not None: - imgpath = ws.LocateResource( - filename, - try_first=(os.path.dirname(lfn) if lfn is not None else None) - ) - else: - # Maybe we'll get lucky. Shouldn't really be caring about resources - # without a workspace though. - imgpath = filename - - return imgpath - - -ImageElement._ALL_PROPERTIES['type'] = (ImageElement.GetType(), - valid.validateString) - - -# For purposes of consistency, we don't use the base type Element directly. -class LogElement(LocationallyKeyedElement): - - _LOG_PROPERTIES = {'filename': ('logo.png', valid.validateString), - 'regex': ('', valid.validateString), - 'sub_pattern': ('', valid.validateString), - 'color_group_pattern': ('', valid.validateString), } - - _ALL_PROPERTIES = { - key: LocationallyKeyedElement._ALL_PROPERTIES[key] - for key in LocationallyKeyedElement._ALL_PROPERTIES - if key not in ('auto_color_annotation', - 'auto_color_basis', - 'on_update', - 'on_init', - 'on_cycle_changed', - 'caption') - } - _ALL_PROPERTIES.update(_LOG_PROPERTIES) - _ALL_PROPERTIES['dimensions'] = ((200, 200), - _ALL_PROPERTIES['dimensions'][1]) - - @staticmethod - def GetType() -> str: - return 'log' - - @staticmethod - def IsDrawable() -> bool: - return True - - # called when drawn is not null - @classmethod - def GetDrawRoutine(cls) -> Callable: - return cls.DrawRoutine - - def __init__(self, *args: Any, **kwargs: Any) -> None: - LocationallyKeyedElement.__init__(self, *args, **kwargs) - - self.__def_background_color = wx.Colour(255, 255, 255) - - # Searcher (logsearcher) for scanning log for ticks - self.__searcher: Optional[LogSearch] = None - self.__file: Optional[TextIO] = None # File handle for reading log - # Log lines for current cycle - self.__line_cache: Optional[List[Tuple[str, wx.Colour]]] = [] - self.__tick: Optional[int] = None - self.__prev_loc: Optional[int] = None - - self.LoadLog(cast(str, self.GetProperty('filename'))) - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - render_tick: int, - time_range: Optional[Tuple[int, int]] = None, - render_box: Optional[Tuple[int, int, int, int]] = None, - fixed_offset: Optional[Tuple[int, int]] = None) -> None: - - pen_colors = cast(Tuple[int, int, int], self.GetProperty('color')) - border_color = [int(c) for c in pen_colors] - normal_pen = wx.Pen(border_color, 1) # TODO: Use a pen cache - dc.SetPen(normal_pen) - - (x, y) = cast(Tuple[int, int], self.GetProperty('position')) - (w, h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - xoff, yoff = canvas.GetRenderOffsets() - if not fixed_offset: - (x, y) = (x - xoff, y - yoff) - else: - x = x - fixed_offset[0] - y = y - fixed_offset[1] - - period = pair.GetClockPeriod() - t_offset = cast(int, self.GetProperty('t_offset')) - - dc.SetClippingRegion(x, y, w, h) - - filename = cast(str, self.GetProperty('filename')) - if not self.__searcher: - # TODO: use a brush cache - brush = wx.Brush((200, 200, 200), style=wx.SOLID) - dc.SetBrush(brush) - - dc.SetBackground(wx.Brush(self.__def_background_color)) - - # Draw border and X through box - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - - # Draw x through box - dc.DrawLine(int(x), int(y), int(x + w), int(y + h)) - dc.DrawLine(int(x + w), int(y), int(x), int(y + h)) - dc.DrawText('No such file:\n' + filename, int(x), int(y)) - elif t_offset != 0 and period == -1: - # TODO: use a brush cache - brush = wx.Brush((200, 200, 200), style=wx.SOLID) - dc.SetBrush(brush) - - dc.SetBackground(wx.Brush(self.__def_background_color)) - - # Draw border and X through box - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - - # Draw x through box - dc.DrawLine(int(x), int(y), int(x + w), int(y + h)) - dc.DrawLine(int(x + w), int(y), int(x), int(y + h)) - dc.DrawText(f't_offset is nonzero ({t_offset}) but this element\n' - 'element is not associated with a database\n' - 'location which has a clock. Change location\n' - f'"{self.GetProperty("LocationString")}"\n' - 'to something else so that this element can\n' - 'compute the offset in that location\'s clock\n' - 'cycles. Or change t_offset to 0\n', - int(x), - int(y)) - else: - regex_str = cast(str, self.GetProperty('regex')) - sub_pat_str = cast(str, self.GetProperty('sub_pattern')) - color_pat_str = cast(str, self.GetProperty('color_group_pattern')) - if regex_str == '': - expr = None - else: - expr = re.compile(regex_str) - # TODO: use a brush cache - brush = wx.Brush(border_color, style=wx.TRANSPARENT) - dc.SetBrush(brush) - dc.DrawRectangle(int(x), int(y), int(w), int(h)) - - tick = render_tick + (t_offset * period) - - assert self.__file is not None - - if self.__tick != tick or self.__line_cache is None: - if self.__tick is None or tick > self.__tick: - after_last = True - else: - after_last = False - self.__tick = tick - self.__line_cache = [] - - if self.__prev_loc is None or \ - self.__prev_loc == self.__searcher.BAD_LOCATION: - self.__prev_loc = 0 # Reset for a new search - assert self.__tick is not None - self.__prev_loc = self.__searcher.getLocationByTick( - self.__tick, - self.__prev_loc if after_last else 0 - ) - if self.__prev_loc == self.__searcher.BAD_LOCATION: - pass - # Nothing in the log for this tick - else: - self.__file.seek(self.__prev_loc) - while 1: - t = self.__file.readline() - if not t.startswith('{'): - break # Not in the middle of the line - try: - tick_str = t[1:t.find(' ')] - line_tick = int(tick_str) - except Exception: - break # Unable to parse SPARTA line prefix - if line_tick > self.__tick: - break # Reached the next tick - - if expr is None: - self.__line_cache.append( - (t, self.__def_background_color) - ) - else: - m = expr.search(t) - if m is not None: - # Determine Line Color - if color_pat_str == '': - color = self.__def_background_color - else: - try: - col_str = re.sub(expr, - color_pat_str, - t) - except sre_constants.error as ex: - print('Unable to generate color value ' - 'from regex groups using ' - f'"{color_pat_str}". Error={ex}') - color = self.__def_background_color - else: - try: - idx = int(col_str) % len(autocoloring.BACKGROUND_BRUSHES) # noqa: E501 - except ValueError: - try: - idx = int(col_str, 16) % len(autocoloring.BACKGROUND_BRUSHES) # noqa: E501 - except ValueError: - idx = hash(col_str) % len(autocoloring.BACKGROUND_BRUSHES) # noqa: E501 - - color = autocoloring.BACKGROUND_BRUSHES[idx].GetColour() # noqa: E501 - - # Determine Content - if sub_pat_str == '': # Do no replacement - self.__line_cache.append((t, color)) - else: - try: - t = re.sub(expr, sub_pat_str, t) - except sre_constants.error as ex: - print('Unable to generate displayed ' - 'string from regex groups using ' - f'"{color_pat_str}". Error={ex}') - - self.__line_cache.append((t, color)) - - lx = x + 1 - ly = y + 1 - dc.SetBackgroundMode(wx.SOLID) - for t, color in self.__line_cache: - dc.DrawTextList([t.rstrip()], - [(int(lx), int(ly))], - backgrounds=color) - ly += 12 # Figure out line height - - dc.DestroyClippingRegion() - self.UnsetNeedsRedraw() - - def SetProperty(self, key: str, val: PropertyValue) -> None: - LocationallyKeyedElement.SetProperty(self, key, val) - - if key == 'filename': - val = cast(str, self.GetProperty(key)) - self.LoadLog(val) - elif key == 'regex': - self.__line_cache = None # Force reloading - elif key == 'sub_pattern': - self.__line_cache = None # Force reloading - elif key == 'color_group_pattern': - self.__line_cache = None # Force reloading - - def LoadLog(self, filename: str) -> None: - if os.path.exists(filename): - self.__searcher = LogSearch(filename) - self.__file = open(filename, 'r') - else: - self.__searcher = None - self.__file = None - - # Clear line cache and force reload next time - self.__tick = None - self.__line_cache = None - self.__prev_loc = 0 - - -LogElement._ALL_PROPERTIES['type'] = (LogElement.GetType(), - valid.validateString) - - -# Class which only is used to identify transactions. -# Used for collision events for mouse-over. -class FakeElement(Element): - - def __init__(self) -> None: - self._properties: PropertyDict = {} - - @staticmethod - def IsSelectable() -> bool: - return True - - def GetProperty(self, - key: str, - period: Optional[int] = None) -> PropertyValue: - # try...except is faster than dict.get() - try: - return self._properties[key] - except Exception: - return None - - def SetProperty(self, key: str, value: PropertyValue) -> None: - self._properties[key] = value - - # Does this element have a particular property - def HasProperty(self, key: str) -> bool: - return key in self._properties diff --git a/helios/pipeViewer/pipe_view/model/element_propsvalid.py b/helios/pipeViewer/pipe_view/model/element_propsvalid.py deleted file mode 100644 index c471524289..0000000000 --- a/helios/pipeViewer/pipe_view/model/element_propsvalid.py +++ /dev/null @@ -1,238 +0,0 @@ -# @package elementpropertiesvalidation -# This module exists to pair with element.py and validate anything that is -# being attempted to set as a value for one of an Element's properties - -from __future__ import annotations -from . import content_options as content -# used for tuplifying -import string -import ast -from typing import List, Optional, Tuple, TypeVar, Union, cast - -# A listing of the available options for the 'Content' field of an Element -__CONTENT_OPTIONS = content.GetContentOptions() - - -# Returns the listing of options for the 'Content' field of an Element -def GetContentOptions() -> List[str]: - return __CONTENT_OPTIONS - - -StringTuple = Union[str, Tuple[int, int]] - - -# Confirms that a position is in the form (int, int) for (x,y) -def validatePos(name: str, raw: StringTuple) -> Tuple[int, int]: - if isinstance(raw, str): - val = tuplify(raw) - else: - val = raw - if not isinstance(val, tuple): - raise TypeError(f'Parameter {name} must be a 2-tuple of ints') - if len(val) != 2: - raise ValueError(f'Parameter {name} must be 2-tuple of ints') - if not isinstance(val[0], int): - raise TypeError( - f'Parameter {name}: only integers allowed for x-coords' - ) - if not isinstance(val[1], int): - raise TypeError( - f'Parameter {name}: only integers allowed for y-coords' - ) - val = cast(Tuple[int, int], val) - return val - - -# Confirms that dimensions are in the form (int, int) for (width, height) -def validateDim(name: str, raw: StringTuple) -> Tuple[int, int]: - if isinstance(raw, str): - val = tuplify(raw) - else: - val = raw - if not isinstance(val, tuple): - raise TypeError(f'Parameter {name} must be a 2-tuple of ints') - if len(val) != 2: - raise ValueError(f'Parameter {name} must be 2-tuple of ints') - if not (val[0] > 0 and val[1] > 0): - raise ValueError(f'Parameter {name} must only have positive values') - if not isinstance(val[0], int): - raise TypeError(f'Parameter {name}: only integers allowed for width') - if not isinstance(val[1], int): - raise TypeError(f'Parameter {name}: only integers allowed for height') - val = cast(Tuple[int, int], val) - return val - - -# Confirms that a color is in the form (int, int, int) for (R,G,B) -def validateColor( - name: str, - raw: Union[str, Tuple[int, int, int]] -) -> Tuple[int, int, int]: - if isinstance(raw, str): - val = tuplify(raw) - else: - val = raw - if not isinstance(val, tuple): - raise TypeError(f'Parameter {name} must be a 3-tuple of ints') - if len(val) != 3: - raise ValueError(f'Parameter {name} must be 3-tuple of ints') - if not isinstance(val[0], int): - raise TypeError(f'Parameter {name}: only integers allowed for red') - if not isinstance(val[1], int): - raise TypeError(f'Parameter {name}: only integers allowed for green') - if not isinstance(val[2], int): - raise TypeError(f'Parameter {name}: only integers allowed for blue') - if not 0 <= val[0] <= 255: - raise ValueError(f'Parameter {name}: red must be between 0 & 255') - if not 0 <= val[1] <= 255: - raise ValueError(f'Parameter {name}: green must be between 0 & 255') - if not 0 <= val[2] <= 255: - raise ValueError(f'Parameter {name}: blue must be between 0 & 255') - val = cast(Tuple[int, int, int], val) - return val - - -# Confirms that an LocationString is a str -def validateLocation(name: str, val: str) -> str: - val = str(val) - if not isinstance(val, str): - raise TypeError(f'Parameter {name} must be an str') - return val - - -# Confirms that the Content specification is a str corresponding to the -# available options -def validateContent(name: str, val: str) -> str: - if not isinstance(val, str): - raise TypeError(f'Parameter {name} must be a str') - if val not in __CONTENT_OPTIONS: - raise ValueError( - f'Parameter {name} must be one of the options: {__CONTENT_OPTIONS}' - ) - return val - - -# Confirms that the clock offset is valid -def validateClockOffset(name: str, raw: StringTuple) -> Tuple[int, int]: - if isinstance(raw, str): - val = tuplify(raw) - else: - val = raw - - if not isinstance(val, tuple): - raise TypeError(f'Parameter {name} must be a tuple of (clock, cycles)') - - val = cast(Tuple[int, int], val) - return val - - -# Confirms that scale factor is an int -def validateTimeScale(name: str, raw: Union[str, float, int]) -> float: - try: - val = float(raw) - except ValueError: - raise TypeError(f'Parameter {name} must be a number') - return val - - -# Confirms that an offset is an int -def validateOffset(name: str, raw: Union[str, int]) -> int: - val: Union[str, int] - if isinstance(raw, str) and isNumeral(raw): - val = int(raw) - else: - val = raw - if not isinstance(val, int): - raise TypeError(f'Parameter {name} must be a int') - return val - - -# Confirms this is a string -# Treats None objects as empty string -def validateString(name: str, val: Optional[str]) -> str: - if val is None: - val = '' - else: - val = str(val) - if not isinstance(val, str): - raise TypeError(f'Parameter {name} must be a str') - return val - - -# Confirms this is a bool and converts if necessary -def validateBool(name: str, val: Optional[bool]) -> bool: - if val is None: - val = False - else: - val = not not val - return val - - -T = TypeVar('T') - - -# Confirms this is list -def validateList(name: str, val: Union[str, List[T]]) -> List[T]: - if isinstance(val, str): - val = ast.literal_eval(val) - else: - val = list(val) - if not isinstance(val, list): - raise TypeError(f'Parameter {name} must be a list') - return val - - -# Confirms that an X/Y scale value is in the form (number, number) -def validateScale( - name: str, - raw: Union[Tuple[float, float], StringTuple] -) -> Union[Tuple[float, float], Tuple[int, int]]: - val: Union[Tuple[float, float], Tuple[int, int]] - if isinstance(raw, str): - val = cast(Tuple[int, int], tuplify(raw)) - else: - val = raw - if not isinstance(val, tuple): - raise TypeError(f'Parameter {name} must be a 2-tuple of numbers') - if len(val) != 2: - raise ValueError(f'Parameter {name} must be 2-tuple of numbers') - if not isinstance(val[0], (int, float)): - raise TypeError( - f'Parameter {name}: only numbers allowed for x-scale factors' - ) - if not isinstance(val[1], (int, float)): - raise TypeError( - f'Parameter {name}: only numbers allowed for y-scale factors' - ) - return val - - -# Takes a string (of supposed user input) and converts it, if possible, to a -# tuple of ints. Floats are currently discarded and disregarded -def tuplify(raw: str) -> Tuple[int, ...]: - # strip away any leading characters that are not numeric digits - strip = string.whitespace + string.ascii_letters + string.punctuation - strip = strip.replace('-', '') - strip = strip.replace(',', '') - temp_str = raw.strip(strip) - temp = [t.split(' ') for t in temp_str.split(',')] - nums = [] - for element in temp: - for s in element: - if isNumeral(s): - nums.append(int(s)) - val = tuple(nums) - return val - - -# A simple helper method for tuplify(), providing a level of abstraction for -# checking that every character within a string is a digit (base 10) -def isNumeral(s: str) -> bool: - options = string.digits + '-' - res = True - for char in s: - if char not in options: - res = False - if len(s) == 0: - res = False - return res diff --git a/helios/pipeViewer/pipe_view/model/element_set.py b/helios/pipeViewer/pipe_view/model/element_set.py deleted file mode 100644 index 93b73cd26c..0000000000 --- a/helios/pipeViewer/pipe_view/model/element_set.py +++ /dev/null @@ -1,288 +0,0 @@ -from __future__ import annotations -from .element_value import Element_Value -from .query_set import QuerySet -from .quad_tree import QuadTree - -from typing import Callable, Dict, List, Optional, Tuple, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from .element import Element - from .extension_manager import ExtensionManager - from .layout_context import Layout_Context - - -# ElementSet stores all elements in a LayoutContext and bins them by time -# appropriately -# Purely drawable objects are only put in a draw list. -# Objects that request data from the database are put in a QuerySet, -# Objects with both are placed in both. -class ElementSet: - def __init__(self, - layout_context: Layout_Context, - extensions: ExtensionManager) -> None: - self.__draw_set: List[Element_Value] = [] - self.__meta_set: List[Element_Value] = [] - self.__query_set = QuerySet(layout_context) - # used to quickly find pairs from elements - self.__elements_to_pairs: Dict[Element, Element_Value] = {} - self.__layout_context = layout_context - self.__extensions = extensions - # tree used for display calls - self.__tree = QuadTree() - self.__vis_tick = 0 - - # Adds a new element to the set - # @param e Element to add - # @param after_pins ordered PINs of elements after which to insert this - # element - def AddElement(self, - e: Element, - after_pins: List[Optional[int]] = [None]) -> None: - pair = Element_Value(e) - self.__elements_to_pairs[e] = pair - if e.NeedsDatabase(): - self.__query_set.AddPair(pair) - if e.IsDrawable(): - self.__InsertDrawableAfterPIN(pair, after_pins=after_pins) - if e.UsesMetadata(): - self.__meta_set.append(pair) - if e.HasProperty('on_init'): - on_init = cast(str, e.GetProperty('on_init')) - if on_init and on_init != 'None': - func = self.__extensions.GetFunction(on_init) - if func: - func(pair, self.__layout_context, 0) - else: - print('Warning: unable to call \"%s\"' % on_init) - print(func) - - # Removes an element from the set - def RemoveElement(self, e: Element) -> None: - pair = self.__elements_to_pairs[e] - if e.NeedsDatabase(): - self.__query_set.DeletePair(pair) - if e.IsDrawable(): - self.__draw_set.remove(pair) - self.__tree.RemoveObject(pair) - if e.UsesMetadata(): - self.__meta_set.remove(pair) - del self.__elements_to_pairs[e] - - # Resorts an element. Mostly for query-type objects. - def ReSort(self, e: Element, t_offs: int, id: int) -> None: - # q-frame objects need no resorting. make a bogus call to GetQueryFrame - if e.NeedsDatabase() and not e.GetQueryFrame(1): - pair = self.__elements_to_pairs[e] - self.__query_set.ReSort(pair, t_offs, id) - # Don't currently do anything for pure drawables. - - # Resort all elements which depend on database queries - def ReSortAll(self) -> None: - for e in list(self.__elements_to_pairs.keys()): - if e.NeedsDatabase() and not e.GetQueryFrame(1): - loc = cast(str, e.GetProperty("LocationString")) - t_off = cast(int, e.GetProperty("t_offset")) - loc_mgr = self.__layout_context.dbhandle.database.location_manager # noqa: E501 - id = loc_mgr.getLocationInfo(loc, self.__layout_context.GetLocationVariables())[0] # noqa: E501 - self.ReSort(e, t_off, id) - - # Update value - def ReValue(self, e: Element) -> None: - if e.NeedsDatabase(): - pair = self.__elements_to_pairs[e] - self.__query_set.ReValue(pair) - - # Used in the event that many elements were changed (e.g. a location string - # variable was updated) - def ReValueAll(self) -> None: - for pair in self.__elements_to_pairs.values(): - self.__query_set.ReValue(pair) - - def RefreshPair(self, pair: Element_Value) -> None: - self.__tree.RefreshObject(pair) - - # Called every major render update. This makes sure objects are updated - # when they are moved. - def MicroUpdate(self) -> None: - self.__tree.Update() - if self.__layout_context.IsElementMoved(): - frame = self.__layout_context.GetFrame() - if frame: - for el in frame.GetCanvas().GetSelectionManager().GetSelection(): # noqa: E501 - self.RefreshPair(self.__elements_to_pairs[el]) - - def HandleCycleChangedEvent(self) -> None: - # call custom updates - # keyed by function, stores number of times function accessed in Update - indices: Dict[Callable, int] = {} - - for element in self.GetElements(): - if element.HasProperty('on_cycle_changed'): - on_cycle_changed = \ - cast(str, element.GetProperty('on_cycle_changed')) - if on_cycle_changed and on_cycle_changed != 'None': - func = self.__extensions.GetFunction(on_cycle_changed) - if func: - if indices.get(func) is None: - indices[func] = 0 - func(self.__elements_to_pairs[element], - self.__layout_context, indices[func]) - indices[func] += 1 - else: - print(f'Warning: unable to call \"{on_cycle_changed}\"') # noqa: E501 - - # Update what needs to be updated. - # @param tick Tick at which update is happening (Helps in maintaining - # metadata between layout windows) - def Update(self, tick: int) -> None: - - # Update element content before updating meta-data - # Could go into a slower-updating loop - self.__query_set.Update() - - # call custom updates - # keyed by function, stores number of times function accessed in Update - indices: Dict[Callable, int] = {} - - # Purge metadata at the start of a new tick - if self.__layout_context.dbhandle.database.GetMetadataTick() != tick: - self.__layout_context.dbhandle.database.PurgeMetadata() - self.__layout_context.dbhandle.database.SetMetadataTick(tick) - - for element in self.GetElements(): - if element.HasProperty('on_update'): - on_update = cast(str, element.GetProperty('on_update')) - if on_update and on_update != 'None': - func = self.__extensions.GetFunction(on_update) - if func: - if indices.get(func) is None: - indices[func] = 0 - func(self.__elements_to_pairs[element], - self.__layout_context, indices[func]) - indices[func] += 1 - else: - print('Warning: unable to call \"%s\"' % on_update) - - # Updates meta-data for elements in this element set - # @note Call immediately before drawing - # @param tick Tick at which update/redraw is happening - def MetaUpdate(self, tick: int) -> None: - db = self.__layout_context.dbhandle.database - assert db.GetMetadataTick() == tick, \ - 'MetaUpdate called where meta data was stale' - - # update pairs with current metadata (happens anyway with pointers?) - for pair in self.__meta_set: - el = pair.GetElement() - pair.SetMetaEntries( - db.GetMetadata(cast(str, - el.GetProperty(pair.GetMetadataKey()))) - ) - for aux_prop in pair.GetAuxMetadataProperties(): - pair.UpdateMetaEntries( - db.GetMetadata(cast(str, el.GetProperty(aux_prop))) - ) - - # Always refreshes all objects and clears local buffers - def FullUpdate(self) -> None: - for element in self.GetElements(): - element.SetChanged() - self.Update(self.__layout_context.GetHC()) - - # Force a redraw of all elements that have changed their highlighting state - def RedrawHighlighted(self) -> None: - for element, pair in self.__elements_to_pairs.items(): - if element.GetType() == 'schedule_line_ruler': - continue - for key in pair.GetTimedValues().keys(): - redraw_set = False - if element.HasProperty('LocationString'): - loc_id = self.__layout_context.GetLocationId(element) - if (self.__layout_context.IsSearchResult(key, loc_id) or - self.__layout_context.WasSearchResult(key, loc_id)): - element.SetNeedsRedraw() - redraw_set = True - - if not redraw_set: - uop_uid = pair.GetTimedValUopUid(key) - if uop_uid is not None: - if (self.__layout_context.IsUopUidHighlighted(uop_uid) or # noqa: E501 - self.__layout_context.WasUopUidHighlighted(uop_uid)): # noqa: E501 - element.SetNeedsRedraw() - - # Force a full redraw of the elements. - def RedrawAll(self) -> None: - for element in self.GetElements(): - element.SetNeedsRedraw() - - # Force a DB update. - def DBUpdate(self) -> None: - self.RedrawAll() - self.Update(self.__layout_context.GetHC()) - - # Get depth-sorted pairs with the intention of drawing them while updating - # each ElementValue's visibility tick for filtering by renders/selectors - # @return All drawables pairs sorted by draw depth (back first). Caller - # must filter out-of-bounds elements using ElementValue.GetVisibilityTick() - def GetDrawPairs( - self, - bounds: Optional[Tuple[int, int, int, int]] = None - ) -> List[Element_Value]: - if bounds is not None: - # Use quadtree - self.__vis_tick += 1 - if self.__vis_tick > 10000: - self.__vis_tick = 0 - # TODO: invalidate vis tick on all elements! - self.__tree.SetVisibilityTick(bounds, self.__vis_tick) - else: - pass # Do not update visibility tick. Caller doesn't care - - # Use ordered draw set. Caller must filter elements based on - # GetVisibilityTick() comparison with ElementValue.GetVisibilityTick - return self.__draw_set - - def GetVisibilityTick(self) -> int: - return self.__vis_tick - - # Get Element_Value pairs from set. - def GetPairs( - self, - bounds: Optional[Tuple[int, int]] = None - ) -> List[Element_Value]: - return list(self.__elements_to_pairs.values()) - - def GetPair(self, e: Element) -> Optional[Element_Value]: - return self.__elements_to_pairs.get(e) - - # Get Elements from set. - def GetElements(self) -> List[Element]: - return list(self.__elements_to_pairs.keys()) - - # Insert a drawable after the given PIN - def __InsertDrawableAfterPIN( - self, - pair: Element_Value, - after_pins: List[Optional[int]] = [None] - ) -> None: - self.__tree.AddObject(pair) # No pin ordering support yet - - # Care only about element since the draw list will contain all elements - # we care about - after_pin = after_pins[-1] - - if after_pin is None: - self.__draw_set.append(pair) - elif after_pin == -1: - self.__draw_set.insert(0, pair) - else: - for idx, op in enumerate(self.__draw_set): - if op.GetElement().GetPIN() == after_pin: - self.__draw_set.insert(idx+1, pair) - break - pass - else: - self.__draw_set.append(pair) - - def UpdateElementBounds(self) -> None: - self.__tree.UpdateBounds() diff --git a/helios/pipeViewer/pipe_view/model/element_types.py b/helios/pipeViewer/pipe_view/model/element_types.py deleted file mode 100644 index 45b7e72b33..0000000000 --- a/helios/pipeViewer/pipe_view/model/element_types.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations -from .element import FakeElement, BoxElement, ImageElement -from .schedule_element import (ScheduleElement, - ScheduleLineElement, - ScheduleLineRulerElement) -from .rpc_element import RPCElement -from .speedo_element import SpeedoElement - -# This module serves as a unified place to import Elements from - -creatables = { - BoxElement.GetType(): BoxElement, - ScheduleElement.GetType(): ScheduleElement, - ScheduleLineElement.GetType(): ScheduleLineElement, - ScheduleLineRulerElement.GetType(): ScheduleLineRulerElement, - ImageElement.GetType(): ImageElement, - RPCElement.GetType(): RPCElement, - SpeedoElement.GetType(): SpeedoElement, -} -# Disabled until complete: LogElement.GetType() : LogElement} - -special = {'fake': FakeElement} diff --git a/helios/pipeViewer/pipe_view/model/element_value.py b/helios/pipeViewer/pipe_view/model/element_value.py deleted file mode 100644 index 4bf78fef88..0000000000 --- a/helios/pipeViewer/pipe_view/model/element_value.py +++ /dev/null @@ -1,257 +0,0 @@ -# A convenient wrapper for individual Elements and the [Layout] -# Context-specific value they should display when drawn with a Layout -# Canvas. For all intents and purposes, this could be accomplished with -# [Element, val]. -from __future__ import annotations -from bisect import bisect, bisect_left -from typing import Any, Dict, List, Optional, Tuple, Union -from . import content_options as content -from . import highlighting_utils -from .element import Element, FakeElement, PropertyValue -from .schedule_element import ScheduleLineElement - - -TimedVal = Tuple[Optional[Union[int, str]], Tuple[int, int]] -TimedDict = Dict[int, TimedVal] - - -class Element_Value: - __TIMED_VAL_MAX_CAPACTIY = 1200 - - # number of periods on either side of edge of window to leave transactions - __EVICT_MARGIN = 3 - - def __init__(self, e: Element, val: str = '') -> None: - self.__element = e - self.__val = val - self.__clock_period = -1 - self.__timed_vals: TimedDict = {} - self.__timed_vals_uop_uids: Dict[int, Optional[int]] = {} - self.__update_index = -1 - self.__metas: Optional[Dict[str, Any]] = None - - # Missing location. This differs from __resolved_location in that it - # implies that the element NEEDS a location to display anyting but has - # none - self.__missing_loc = False - - # Cached display data - self.__resolved_location: Optional[str] = None - self.__display_t_offset: Optional[int] = None - - # Tick ID of last update - self.__vis_tick = -1 - - # Returns keys stored for this object in the database - # if nothing stored, returns None - def GetMetaEntries(self) -> Optional[Dict[str, Any]]: - return self.__metas - - # Sets the metadata to reference the \a entries argument - # @param entries Dictionary of meta-data - def SetMetaEntries(self, entries: Optional[Dict[str, Any]]) -> None: - assert entries is None or isinstance(entries, dict), \ - f'Attempted to set metadata which was not a dict: {entries}' - self.__metas = entries - - # Updates the current meta-entries, overwriting only keys in added. This is - # simply a dictionary update and should only be used if the SetMetaEntries - # method has already been used this tick - # @param entries Dictionary of meta-data. If None, has no effect - def UpdateMetaEntries(self, entries: Optional[Dict[str, Any]]) -> None: - assert entries is None or isinstance(entries, dict), \ - f'Attempted to set metadata which was not a dict: {entries}' - if self.__metas is None: - self.__metas = entries - elif entries is not None: - self.__metas.update(entries) - - # Gets the auxilliary metadata properties for this element - def GetAuxMetadataProperties(self) -> List[str]: - return self.__element._AUX_METADATA_PROPERTIES - - # Gets the property to use as the metadata key for this element - def GetMetadataKey(self) -> str: - return self.__element._METADATA_KEY_PROPERTY - - # Returns the value (str) that should be drawn with a Layout Canvas - def GetVal(self) -> str: - return self.__val - - # Sets the value after confirming it is a str - # @post Marks this element value as NOT missing its location - def SetVal(self, val: str, up_idx: Optional[int] = None) -> None: - if not isinstance(val, str): - raise TypeError('Value must be a string, was a ' + type(val)) - self.__val = val - if up_idx: - self.__update_index = up_idx - self.__missing_loc = False - - # Marks this element value as missing a location - # @post Updates value to be the content_options 'no trans' string - def SetMissingLocation(self) -> None: - self.SetVal(content.OverrideState('no trans')) - self.__missing_loc = True - - # Is this element value missing location - def IsMissingLocation(self) -> bool: - return self.__missing_loc - - # Set value to values keyed by time - def SetTimedVal(self, time: int, val: TimedVal) -> None: - if len(self.__timed_vals) > self.__TIMED_VAL_MAX_CAPACTIY: - q_frame = self.__element.GetQueryFrame(self.__clock_period) - if q_frame and isinstance(self.__element, ScheduleLineElement): - margin = self.__EVICT_MARGIN*self.__clock_period - hc = self.__element.GetTime() - q_frame = q_frame[0]+hc-margin, q_frame[1]+hc+margin - # clear everything not in q frame - new = {} - new_uop_uids = {} - keys = self.__GetKeysInRange(q_frame) - for key in keys: - new[key] = self.__timed_vals[key] - new_uop_uids[key] = self.__timed_vals_uop_uids[key] - del self.__timed_vals - del self.__timed_vals_uop_uids - self.__timed_vals = new - self.__timed_vals[time] = val - self.__timed_vals_uop_uids = new_uop_uids - self.__timed_vals_uop_uids[time] = \ - highlighting_utils.GetUopUid(val[0]) - else: - raise Exception( - 'Should not be setting timed value on item without ' - 'time-frame.' - ) - else: - self.__timed_vals[time] = val - self.__timed_vals_uop_uids[time] = \ - highlighting_utils.GetUopUid(val[0]) - - def SetClockPeriod(self, period: int) -> None: - self.__clock_period = period - - def GetClockPeriod(self) -> int: - return self.__clock_period - - # purge cache - def ClearTimedValues(self) -> None: - self.__timed_vals.clear() - self.__timed_vals_uop_uids.clear() - - # Get value at a particular time - def GetTimedVal(self, time: int) -> Optional[TimedVal]: - return self.__timed_vals.get(time) - - def GetTimedValues(self) -> TimedDict: - return self.__timed_vals - - # Get the UID for the uop at the given time - def GetTimedValUopUid(self, time: int) -> Optional[int]: - return self.__timed_vals_uop_uids.get(time) - - def __GetKeysInRange(self, interval: Tuple[int, int]) -> List[int]: - start_tick, end_tick = interval - # fast method for returning range of data - times = list(self.__timed_vals.keys()) - times = sorted(times) - start_ind = bisect_left(times, start_tick) - end_ind = bisect(times, end_tick) - - if start_ind > 0: - # Since we are now merging transactions across heartbeat intervals, - # we need to check if the previous interval overlaps the current - # query range - last_int_start, last_int_end = self.__timed_vals[times[start_ind-1]][1] # noqa: E501 - # Iterate backwards until we no longer overlap - while (start_tick >= last_int_start and - start_tick <= last_int_end and - start_ind > 0): - start_ind -= 1 - last_int_start, last_int_end = self.__timed_vals[times[start_ind-1]][1] # noqa: E501 - - return times[start_ind:end_ind] - - # Gets all values stored on a given time (units: ticks) interval. - def GetTimeRange(self, interval: Tuple[int, int]) -> List[TimedVal]: - keys_in_range = self.__GetKeysInRange(interval) - return [self.__timed_vals[key] for key in keys_in_range] - - # Used to determine if an Element Value has an up-to-date val - @property - def updateindex(self) -> int: - return self.__update_index - - # Returns the Element belonging to this instance - def GetElement(self) -> Element: - return self.__element - - # Used so that Element_Values can be compared to Elements - def GetPIN(self) -> int: - return self.__element.GetPIN() - - # Set tick ID of last visibility upadte for this element - # The renderer will query this to determine if the element should be draw - # this update - def SetVisibilityTick(self, t: int) -> None: - self.__vis_tick = t - - # Get tick ID of last visibility upadte for this element - def GetVisibilityTick(self) -> int: - return self.__vis_tick - - # Set the location and timing information used to order this object in a - # QuerySet. This must be cached in case the t_offset or location is changed - # (e.g. by changing location variables) before it is deleted. This is - # because deleting the element from the QuerySet requires knowledge of the - # clock (through location) and t_offset - # @param t_offset Current value of t_offset property - # @param location Location string with location-string variables resolved - def SetLocationAndTimingInformation(self, - t_offset: int, - location: Optional[str]) -> None: - self.__display_t_offset = t_offset - self.__resolved_location = location - - # Get the fully-resolved location string (no variables) as set by - # SetLocationAndTimingInformation. If none, this object did not correspond - # to a real location. - def GetDisplayLocationString(self) -> Optional[str]: - return self.__resolved_location - - # Get the t_offset as set by - # SetLocationAndTimingInformation - def GetDisplayTOffset(self) -> Optional[int]: - return self.__display_t_offset - - # Used so that Element_Values can be compared to Elements or a None object - # Comparison with other types is not allowed - def __eq__(self, other: object) -> bool: - if not isinstance(other, (Element, Element_Value)): - return False - return self.GetPIN() == other.GetPIN() - - def __hash__(self) -> int: - return self.GetPIN() - - -# For same purpose as fake element, just plays a different role. -class FakeElementValue(Element_Value): - def __init__(self, element: FakeElement, value: PropertyValue) -> None: - self.__element = element - self.__value = str(value) - self.__clock_period = 1 - - def GetVal(self) -> str: - return self.__value - - def GetElement(self) -> Element: - return self.__element - - def GetClockPeriod(self) -> int: - return self.__clock_period - - def SetClockPeriod(self, period: int) -> None: - self.__clock_period = period diff --git a/helios/pipeViewer/pipe_view/model/extension_manager.py b/helios/pipeViewer/pipe_view/model/extension_manager.py deleted file mode 100644 index aae299b40c..0000000000 --- a/helios/pipeViewer/pipe_view/model/extension_manager.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations -from types import ModuleType -from typing import Callable, Dict, List, Optional -from importlib.machinery import SourceFileLoader -import os - - -# Dynamically imports requested scripts and returns requested functions -class ExtensionManager: - def __init__(self) -> None: - self.__modules: Dict[str, ModuleType] = {} # keyed by name - self.__paths: List[str] = [] - - def __FindOrImportModule(self, name: str) -> Optional[ModuleType]: - if name in list(self.__modules.keys()): - return self.__modules[name] - else: - for path in self.__paths: - path_to_py = os.path.join(path, name+'.py') - try: - loader = SourceFileLoader(name, path_to_py) - mod = ModuleType(loader.name) - loader.exec_module(mod) - self.__modules[name] = mod - return mod - except Exception: - pass - return None - - # Adds a path to manager's list of paths to import from - def AddPath(self, path: str) -> None: - self.__paths.append(path) - - # Parses a string module:function, loads the module if needed, and returns - # the function - def GetFunction(self, string: str) -> Optional[Callable]: - try: - module_name, func_name = string.split(':') - module = self.__FindOrImportModule(module_name) - if module: - return getattr(module, func_name) - except Exception: - pass - return None diff --git a/helios/pipeViewer/pipe_view/model/group.py b/helios/pipeViewer/pipe_view/model/group.py deleted file mode 100644 index 2dd86402f7..0000000000 --- a/helios/pipeViewer/pipe_view/model/group.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations -import logging -from typing import List, Optional -import weakref - -from .layout_context import Layout_Context - - -class Group: - def __init__(self) -> None: - # Managed contexts (weak refs) - self.__contexts: List[weakref.ReferenceType[Layout_Context]] = [] - - def AddContext(self, frame: Layout_Context) -> None: - self.__contexts.append(weakref.ref(frame, self.__RemoveContext)) - - def GetContexts(self) -> List[Layout_Context]: - return [c for ctxt in self.__contexts if (c := ctxt()) is not None] - - # Set current tick in the context of - def MoveTo(self, - tick: int, - context: Layout_Context, - no_broadcast: bool = False) -> None: - # \todo All contexts in this group are synced at the same tick. - # In the future, they will be synced with relative offsets (per - # context). Support for this must be added. - - lg = logging.getLogger('Group') - lg.debug('Moving tick of group %s to %s. Caused by %s', - self, - tick, - context) - - assert context is not None, \ - 'Cannot invoke Group.MoveTo with a context argument of None' - - context.SetHC(tick, no_broadcast=no_broadcast) # Implies Update - - # Move other contexts - for ctxtref in self.__contexts: - ctxt = ctxtref() - if ctxt is not None and ctxt is not context: - lg.debug(' Updating %s', ctxt) - ctxt.SetHC(tick, no_broadcast=no_broadcast) - - # Refresh the invoking context LAST so that its draws "faster". - # This is unintuitive, but maybe wx events are stored in a stack? - for ctxtref in self.__contexts: - ctxt = ctxtref() - if ctxt is not None and ctxt is not context: - lg.debug(' Refreshing %s', ctxt) - ctxt.RefreshFrame() - context.RefreshFrame() - - def RemoveContext(self, context: Optional[Layout_Context]) -> None: - if context is not None: - for ctxtref in self.__contexts: - ctxt = ctxtref() - if ctxt is context: - logging.getLogger('Group').debug(' Removing %s', ctxt) - self.__RemoveContext(ctxtref) - break - - def __RemoveContext( - self, - context: weakref.ReferenceType[Layout_Context] - ) -> None: - assert isinstance(context, weakref.ref) - assert context in self.__contexts - self.__contexts.remove(context) diff --git a/helios/pipeViewer/pipe_view/model/highlighting_utils.py b/helios/pipeViewer/pipe_view/model/highlighting_utils.py deleted file mode 100644 index bb19f27522..0000000000 --- a/helios/pipeViewer/pipe_view/model/highlighting_utils.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations -import re -from typing import Optional, Union - -__UOP_UID_REGEX = re.compile(r'.*\buid\(([0-9]+)\)') - - -# @profile -def GetUopUid(anno_string: Optional[Union[int, str]]) -> Optional[int]: - if not isinstance(anno_string, str): - return None - - uid_match = __UOP_UID_REGEX.search(anno_string) - - if uid_match is not None: - return int(uid_match.group(1)) - else: - return None diff --git a/helios/pipeViewer/pipe_view/model/layout.py b/helios/pipeViewer/pipe_view/model/layout.py deleted file mode 100644 index db04f3f10d..0000000000 --- a/helios/pipeViewer/pipe_view/model/layout.py +++ /dev/null @@ -1,663 +0,0 @@ -from __future__ import annotations - -import yaml -import hashlib as md5 -import os -import time -import logging -import pickle -from typing import Any, Dict, List, Optional, TextIO, cast, TYPE_CHECKING - -from .import element_types as etypes - -if TYPE_CHECKING: - from .element import Element, MultiElement, PropertyDict - from .layout_context import Layout_Context - from .workspace import Workspace - - -# The reason this application exists, Layouts allow users to group together -# the elements they are viewing -# -# Layouts can only be loaded at Layout.__init__. To load a new layout, a new -# Layout instance must be created. -class Layout: - - # File name and extension for layout files - LAYOUT_FILE_EXTENSION = '.alf' - - # Layout file wildcard for use in file selection dialog boxes - LAYOUT_FILE_WILDCARD = \ - 'Argos layout files ' \ - f'(*{LAYOUT_FILE_EXTENSION})|*{LAYOUT_FILE_EXTENSION}' - - # Code at start of each precompiled alf file (alfc) indicating its version. - # @note This Layout class can only load specific versions. - # @note Must not contain any whitespace! - # @note There is likely no need to support historic versions assuming - # people don't delete the source alf's. Worst case scenary is some - # additional reload time each time this code is updated. - PRECOMPILED_VERSION_CODE = 'argos-alfc-v3'.strip() - - # Initialize two lists to store Elements and registerd Layout_Contexts - # @param filename Layout file to parse and open. This filename and a - # checksum will be stored in case the file is to be stored again following - # changes. If None or '', no file is loaded - # @param workspace Argos Workspace object if one is available - # @note Layout files can only be loaded during construction - def __init__(self, - filename: Optional[str] = None, - workspace: Optional[Workspace] = None) -> None: - if filename is not None and not isinstance(filename, str): - raise TypeError( - f'filename must be None or a str, is {type(filename)}' - ) - - logging.info('Constructing layout "%s"', filename) - t_start = time.time() - - self.__elements: List[Element] = [] - self.__elements_by_pin: Dict[int, Element] = {} - self.__elements_with_children: List[MultiElement] = [] - self.__workspace = workspace - self.lay_cons: List[Layout_Context] = [] - - # Checksum of file as loaded - self.__file_checksum: Optional[str] = None - self.__filename: Optional[str] = None # Name of file loaded - - if filename: - # Load elements - # (only called during init where no elements are present) - self.__LoadLayout(str(filename)) - # __filename & __file_checksum are already set - - # At end of construction, mark as clean - self.__changed = False - - logging.info("Layout loaded with %s elements in %ss", - len(self.__elements), - time.time() - t_start) - - # Returns the workspace associated with this layout - def GetWorkspace(self) -> Optional[Workspace]: - return self.__workspace - - # This should only be called by a Layout_Context - # Causes the Layout_Context to be registered with the Layout - def LinkLayoutContext(self, lay_con: Layout_Context) -> None: - self.lay_cons.append(lay_con) - - # This should only be called by a Layout_Context - # Unregisters the Layout_Context from the Layout - def UnLinkLayoutContext(self, lay_con: Layout_Context) -> None: - if lay_con in self.lay_cons: - self.lay_cons.remove(lay_con) - - # Gets whether any Layout_Contexts are registered with this Layout - def HasContext(self) -> bool: - return not not self.lay_cons - - # Determine PINs of elements preceding the given list of elements' pins. - # @return List of sequences of all preceeding pins: - # e.g. [[1,2], [1,2,3,4], [1,2,3,4,5,6,7]] - # Result is in same order as input pins. May contain [-1] to indicate that - # an input pin refers to an element at the start of the element list. May - # contain None to indicate that a pin was never found. - # - # It is important to return a list of ALL pins leading up to each input - # PIN because some element sets (such as a quad_tree) will contain nodes - # having only some elements within it. Inserting an element at the right - # point in that list requires finding a point after all known preceeding - # pins available in that list. - # - # @warning This may use lots of memory for large displays and should be - # tested for scalability - # @profile - def DeterminePriorPins(self, els: List[Element]) -> List[List[int]]: - pin_map = {} # {pin:pin_index} - # Pins preceeding pin of each element in els - # Initialize with empty values - results: List[List[int]] = [[]] * len(els) - - for idx, e in enumerate(els): - pin_map[e.GetPIN()] = idx - parent = e.GetParent() - if parent is not None: - # Previous pin of this element is parent's pin - # Should look for siblings though - results[idx].append(parent.GetPIN()) - - # Fill in prev_pin values - prev_pin = -1 # Indicates start of list - for e in self.__elements: - p = e.GetPIN() - if p in pin_map: - results[pin_map[p]].append(prev_pin) - prev_pin = p - - assert [] not in results, \ - f'{results.count([])} pins not found when determining prior pins' \ - f'for {[e.GetPIN() for e in els]}. Results={results}' - return results - - # Creates and returns a new element. Does not add it to the layout. - # @profile - def CreateElement(self, - duplicate: Optional[Element] = None, - initial_properties: Optional[PropertyDict] = None, - element_type: Optional[str] = None, - parent: Optional[Element] = None, - **kwargs: Any) -> Element: - if not element_type: - # Default for compatibility - element_type = 'box' - el_class = etypes.creatables.get(element_type) - if el_class: - e = el_class(duplicate, self, initial_properties, parent, **kwargs) - else: - raise Exception( - f'CreateElement: Unknown Element Type "{element_type}"' - ) - - return e - - # Adds a new element to the list and informs any subscribed Layout_Contexts - # supports duplicating an Element that is passed in as a parameter - # @param duplicate Optional element from which this element will copy - # properties - # @param initial_properties Properties dictionary that overrides any - # defaults - # @return Element created - # @see Element.__init__ - # @profile - def CreateAndAddElement(self, - duplicate: Optional[Element] = None, - initial_properties: Optional[PropertyDict] = None, - element_type: Optional[str] = None, - parent: Optional[Element] = None, - **kwargs: Any) -> Element: - add_kwargs = {} - if 'follows_pins' in kwargs: - # Forward argument - add_kwargs['follows_pins'] = kwargs['follows_pins'] - del kwargs['follows_pins'] - if 'skip_list' in kwargs: - # Forward argument - add_kwargs['skip_list'] = kwargs['skip_list'] - del kwargs['skip_list'] - - e = self.CreateElement(duplicate, - initial_properties, - element_type, - parent, - **kwargs) - - self.AddElement(e, **add_kwargs) - - return e - - # Removes the Element from the list and instructs all subscribed Layout - # Contexts to do the same - def RemoveElement(self, e: Element) -> None: - for element in self.__elements: - # initialize recurisve search and if we've already found something - # in base level - # indicate that by setting chop_all - pruned_lst = self.__PruneOnElement(e, element) - if pruned_lst: - pruned = set(pruned_lst) # don't delete things twice - break # found - else: - raise Exception('Zombie objects: unable to remove %s' % e) - for lay_con in self.lay_cons: - for el in pruned: - lay_con.RemoveElement(el) - self.__changed = True - if e in self.__elements: - self.__elements.remove(e) - del self.__elements_by_pin[e.GetPIN()] - if (parent := e.GetParent()) is not None: - parent.SetNeedsRedraw() - - # Adds the given element to the list and instructs all subscribed Layout - # Contexts to do the same - # @param follows_pin=PIN Pin of element after which to insert this element - # @param skip_list=Skips adding element to the local list if true - # @profile - def AddElement(self, e: Element, **kwargs: Any) -> None: - follows_pins = kwargs.get('follows_pins', [None]) - skip_list = kwargs.get('skip_list', False) - if not skip_list: - self.__InsertElementAfterPIN(e, follows_pins[-1]) - self.__elements_by_pin[e.GetPIN()] = e - if e.HasChildren(): - self.__elements_with_children.append(cast('MultiElement', e)) - for lay_con in self.lay_cons: - lay_con.AddElement(e, after_pins=follows_pins) - self.__changed = True - - # If a property (t_offset or LocationString) for an Element is changed - # such that it will no longer be correctly sorted in the OrderedDict, - # this method will figure out where it goes - def ReSort(self, e: Element, t_off: int, loc: str) -> None: - for lay_con in self.lay_cons: - lay_con.ReSort(e, t_off, loc) - - # If a property for an Element indicating position or size is changed, - # this method is invoked. Alerts all layout contexts110 - def ElementsMoved(self, e: Element) -> None: - for lay_con in self.lay_cons: - lay_con.ElementMoved(e) - - def Refresh(self, e: Element) -> None: - for lay_con in self.lay_cons: - lay_con.ReValue(e) - - # Find an element by its PIN - # @return The element with the given pin if found. Otherwise returns None - def FindByPIN(self, pin: int) -> Optional[Element]: - el = self.__elements_by_pin.get(pin, None) - # If we didn't find the element, it might be because it's a child of - # another element - # Search those next - if el is None: - for parent_el in self.__elements_with_children: - el = parent_el.GetChildByPIN(pin) - if el is not None: - break - return el - - # Returns the list (un-sorted) of all Elements in the Layout - # It is not safe to modify this list - def GetElements(self) -> List[Element]: - return self.__elements - - # @ Returns filename used by latest SaveToFile() either explicitly or - # implicitly. Or, if SaveToFile has not yet been called, returns the - # construction filename. If no construction filename was specified either, - # returns None. - def GetFilename(self) -> Optional[str]: - return self.__filename - - # @ Returns file checksum obtained by latest SaveToFile call. Or, if - # SaveToFile has not yet been called, returns the construction filename. - # If no construction filename was specified either, returns None. - def GetFileChecksum(self) -> Optional[str]: - return self.__file_checksum - - # Store the layout to a file as YAML. Preserves draw order - # @param filename. If None, stores to the current filename returned by - # GetFilename. If there is no current filename, raises ValueError. - # If a str, attempts to store to this file. If None, effectively behaves - # like a typical "save" feature. If a str, behaves like a "save-as". - def SaveToFile(self, filename: Optional[str] = None) -> None: - if filename is None: - filename = self.__filename - elif not isinstance(filename, str): - raise TypeError( - f'filename must be None or a str, is {type(filename)}' - ) - if filename == '': - raise ValueError('filename cannot be an empty string') - if self.__filename == filename: - if not self.CanSaveToFile(): - raise IOError( - f'Cannot save layout {self} back to file "{filename}" ' - 'without overwriting other changes in that file' - ) - assert filename is not None - - try: - with open(filename, 'w') as f: - self._StoreYAMLToStream(f) - except Exception: - # Failed to save. Allow this to propagate up - raise - - # Store new file info - self.__filename = os.path.abspath(filename) - self.__file_checksum = self.__ComputeChecksum(filename) - - # Write the updated precompiled layout file - self.__WritePrecompiledLayout( - self.__GenPrecompiledLayoutFilename(filename) - ) - - # Just saved, layout is now unchanged - self.__MarkAsUnchanged() - - # Determines if this can be stored to the current GetFilename() without - # overwriting changes to that file. If GetFilename returns None, this - # always returns False. - # @note Does not determine write-access for destination directory - # @return True if SaveToFile() can safely be called without a filename - # and False if SaveToFile must specify a filename different from the - # current filename or delete the current filename. - # - # For filenames other than the current GetFilename(), this Layout just - # assumes it can write without consequence - def CanSaveToFile(self) -> bool: - if self.__filename is None: - return False - if not os.path.exists(self.__filename): - return True # File was deleted? - digest = self.__ComputeChecksum(self.__filename) - if digest != self.__file_checksum: - return False # File has changed - - return True - - # Indicates whether this Layout or any of its comprising elements have - # been modified in any way that could affect the save state on disk - # @return True if this layout or elements have changed - def HasChanged(self) -> bool: - if self.__changed: - return True - - for c in self.__elements: - if c.HasChanged(): - return True - - return False - - # Mark the layout as changed. Ideally, this is not needed since HasChanged - # inspects elements for their changes. - def SetChanged(self) -> None: - self.__changed = True - - # Insert an element following an element with the given pin - def __InsertElementAfterPIN(self, e: Element, after_pin: int = -1) -> None: - if after_pin is None: - self.__elements.append(e) - elif after_pin == -1: - self.__elements.insert(0, e) - else: - for idx, el in enumerate(self.__elements): - if el.GetPIN() == after_pin: - self.__elements.insert(idx + 1, e) - break - else: - self.__elements.append(e) - - # Recursive removal of element and its children, returns the pruned - # elements - def __PruneOnElement(self, - e: Element, - element: Element, - chop_all: bool = False) -> List[Element]: - if e == element or chop_all: - chop_all = True - chopped = [element] - else: - chopped = [] - if element.HasChildren() and element.GetChildren(): - element = cast('MultiElement', element) - children = element.GetChildren() - for child in children: - # collection mode used on nodes subordinate to target node - if chop_all: - chopped.extend(self.__PruneOnElement(e, - child, - chop_all=True)) - else: # keep going - if child == e: # found our query - element.RemoveChild(child) - return self.__PruneOnElement(e, child, chop_all=True) - else: # not found yet, delve deeper - out = self.__PruneOnElement(e, child) - if out: # found something - return out - return chopped - - # Store the layout to a given stream as a new YAML stream without any - # safety checks about the stream itself. - # Preserves draw order - # @param stream Stream with writeability. Layout representation will be - # written to this stream. - # @note Does not update filename or file_checksum attributes - # @note Emits YAML stream start/end events, so storing multiple layouts in - # 1 file might not be parseable - def _StoreYAMLToStream(self, stream: TextIO) -> None: - events = [ - yaml.StreamStartEvent(encoding='ascii'), - yaml.DocumentStartEvent(explicit=True), - yaml.SequenceStartEvent(anchor=None, - tag=None, - implicit=True, - flow_style=False), - ] - - # Serialize all elements in order - for e in self.__elements: - events.extend(e._GetYAMLEvents()) - - events.extend([yaml.SequenceEndEvent(), - yaml.DocumentEndEvent(explicit=True), - yaml.StreamEndEvent()]) - - yaml.emit(events, stream) - - # Computes a checksum of the file specified - @staticmethod - def __ComputeChecksum(filename: str) -> str: - with open(filename, 'r') as f: - return md5.md5(f.read().encode('utf-8')).hexdigest() - - # Loads an element from YAML, returns element - def __LoadElement(self, - elinfo: PropertyDict, - idx: int, - filename: str, - parent: Optional[Element] = None) -> Element: - if not isinstance(elinfo, dict): - raise ValueError( - f'Element {idx} in Argos layout file "{filename}" was not a ' - f'map structure, was a {type(elinfo)}' - ) - - # Peek at type before further reading - el_type = cast(str, elinfo.get('type')) - # TODO: reintroduce unknown property checking - e = self.CreateElement(initial_properties=elinfo, - element_type=el_type, - parent=parent) - # load children if any exist - children = cast(List['PropertyDict'], elinfo.get('children')) - if e.HasChildren() and children: - e = cast('MultiElement', e) - for child in children: - e.AddChild(self.__LoadElement(child, idx, filename, parent=e)) - return e - - # Loads layout from a YAML file - # @param filename File to load - # @throw Raises an Exception if file cannot be opened or there are errors - # parsing - # @pre This layout must have no elements. They will not be cleared. - # @note If this layout has layout contexts referencing it, they will be - # notified of added elements - # @note Sets __filename and __file_checksum - # @todo Parse YAML events or tokens with Marks so that Layout-file - # semantic errors in the layout file can be pinpointed to a line/col. - def __LoadLayout(self, filename: str) -> None: - # Store filename before loading the layout since some elements care - # about this for determining resource locations - self.__filename = os.path.abspath(filename) # Name of file loaded - - precompiled_filename = self.__GenPrecompiledLayoutFilename(filename) - loaded_precompiled = self.__TryLoadPrecompiledLayout( - filename, - precompiled_filename - ) - - if not loaded_precompiled: - # No precompiled file (or out of date) - with open(filename, 'r') as f: - file_data = f.read() - try: - d = yaml.safe_load(file_data) - except Exception as exc: - raise Exception( - f'Failed to load yaml file "{filename}":\n{str(exc)}' - ) - - if not hasattr(d, '__iter__'): - raise ValueError( - f'Data retrieved from Argos layout file "{filename}" is ' - f'not an iterable structure : \n{d}' - ) - # Children of elements stored inside elements in Layout. - # LayoutContext flattens Layout for ease-of-rendering and querying. - # Get each element - for idx, elinfo in enumerate(d): - e = self.__LoadElement(elinfo, idx, filename) - self.AddElement(e) - - # @todo Use file date modified instead - probably faster - # Checksum of file as loaded - self.__file_checksum = self.__ComputeChecksum(filename) - - # Just loaded - No changes - self.__MarkAsUnchanged() - - # Load individual element from pickle - def __LoadElementFromPickle(self, - pdict: PropertyDict, - parent: Optional[Element] = None) -> Element: - if 'children' in pdict: - # child property only is generated at compile time. Strip out. - children = cast(List['PropertyDict'], pdict.pop('children')) - else: - children = None - - e = self.CreateElement(initial_properties=pdict, - element_type=cast(Optional[str], - pdict.get('type')), - parent=parent) - if children: - assert isinstance(e, MultiElement) - - for child in children: - e.AddChild(self.__LoadElementFromPickle(child, parent=e)) - - return e - - # Loads a precompiled layout file - # @return True if load was successful and False if not. - # @note Logs to warning and debug about loading issues - # @profile - def __TryLoadPrecompiledLayout(self, - filename: str, - precompiled_filename: str) -> bool: - if not os.path.exists(precompiled_filename): - return False - - logging.info('Attempting to precompiled layout %s', - precompiled_filename) - - # Open precompiled alf and load header lines - f = open(precompiled_filename, 'rb') - - # Compare - pc_ver_code = f.readline().strip().decode('utf-8') - if pc_ver_code != self.PRECOMPILED_VERSION_CODE: - logging.info( - 'Precompiled alf "%s" is in an older format. Version code "%s"' - ' differs from current precompiled layout version code "%s"', - precompiled_filename, - pc_ver_code, - self.PRECOMPILED_VERSION_CODE - ) - return False - - # Compare checksum - pc_checksum = f.readline().strip().decode('utf-8') - alf_checksum = self.__ComputeChecksum(filename) - if pc_checksum != alf_checksum: - logging.info( - 'Precompiled alf "%s" is out of date. checksum %s differs ' - 'from checksum %s of alf file %s:', - precompiled_filename, - pc_checksum, - alf_checksum, - filename - ) - return False - - try: - # Load rest of the precompiled file - properties = pickle.loads(f.read()) - except Exception as ex: - logging.warn('Found but failed to load "%s": %s', - precompiled_filename, - str(ex)) - return False - - for pdict in properties: - e = self.__LoadElementFromPickle(pdict) - self.AddElement(e) - return True - - def __MakeSerializable(self, el: Element) -> PropertyDict: - output = {} - output = el.GetSerializableProperties() # get all fields - if el.HasChildren(): - children = el.GetChildren() - output['children'] = [] - for child in children: - cast(List['PropertyDict'], output['children']).append( - self.__MakeSerializable(child) - ) - - return output - - # Writes a precompiled layout file having the given name - def __WritePrecompiledLayout(self, precompiled_filename: str) -> None: - try: - f = open(precompiled_filename, 'wb') - except IOError as ex: - if ex.errno != 13: # Permission error - raise # Only permission errors are expected - logging.info('Failed to open precompiled alf file for writing %s', - precompiled_filename) - else: - try: - f.write((self.PRECOMPILED_VERSION_CODE + '\n').encode('utf-8')) - f.write((str(self.__file_checksum) + '\n').encode('utf-8')) - f.write(pickle.dumps( - [self.__MakeSerializable(el) for el in self.__elements], - pickle.HIGHEST_PROTOCOL) - ) - except Exception as ex: - logging.info( - 'Encountered error writing content to precompiled alf ' - 'file %s. File was opened, but the error occurred when ' - 'writing content: %s', - precompiled_filename, - ex - ) - else: - logging.info('Successfully wrote precompiled alf: %s', - precompiled_filename) - - # Generates precompiled filename for a given layout filename - def __GenPrecompiledLayoutFilename(self, filename: str) -> str: - return filename + 'c' # e.g. layout.alfc - - # Flags this layout and all comprising elements as Not Changed. - # This is intended to be called when a layout is being loaded or saved - def __MarkAsUnchanged(self) -> None: - self.__changed = False - for c in self.__elements: - c._MarkAsUnchanged() - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() - - # Add a region of two offsets from current time to ask for when updated. - def GetElementDump(self) -> str: - return repr(self.__elements) diff --git a/helios/pipeViewer/pipe_view/model/layout_context.py b/helios/pipeViewer/pipe_view/model/layout_context.py deleted file mode 100644 index dca19a14bd..0000000000 --- a/helios/pipeViewer/pipe_view/model/layout_context.py +++ /dev/null @@ -1,733 +0,0 @@ -from __future__ import annotations -import logging -import weakref -import os -from typing import (Any, - Dict, - List, - Optional, - Set, - Tuple, - Union, - cast, - overload, - TYPE_CHECKING) -import wx - -from .layout import Layout -from .element_set import ElementSet -from .database_handle import DatabaseHandle -from .search_handle import SearchHandle -from .element import Element, FakeElement -from .extension_manager import ExtensionManager -from .location_manager import LocationManager, LocationType -from . import content_options as content -from . import highlighting_utils as highlighting_utils - - -if TYPE_CHECKING: - from ..gui.layout_frame import Layout_Frame - from ..gui.dialogs.search_dlg import SearchResult - from .database import Database, TransactionDatabase - from .element_value import Element_Value - from .group import Group - from .schedule_element import ScheduleLineElement, ScheduleElement - - -class Layout_Context: - EXTENT_L = 0 # Left - EXTENT_T = 1 # Top - EXTENT_R = 2 # Right - EXTENT_B = 3 # Bottom - - # Dummy class for showing that extents are invalid - class InvalidExtents: - pass - - # Creates an OrderedDict for the Layout - # @param loc_vars location string variables dictionary. This reference is - # used (not copied) Created if None - def __init__(self, - layout: Layout, - db: Database, - hc: Optional[int] = None, - loc_vars: Optional[Dict[str, str]] = None) -> None: - if hc is None: - hc = 0 - # Will be updated later. Allows __str__ to succeed - self.__layout = None - self.__group: Optional[Group] = None # Sync group - # Parent Layout_Frame (weak reference) - self.__frame: Optional[weakref.ReferenceType[Layout_Frame]] = None - self.__db = db - self.__dbhandle = DatabaseHandle(db) - self.__searchhandle = SearchHandle(self) - self.__qapi = self.__dbhandle.api - start = self.__qapi.getFileStart() - inc_end = self.__qapi.getFileInclusiveEnd() - if hc < start: - self.__hc = start - elif hc > inc_end: - self.__hc = inc_end - else: - self.__hc = hc - self.__extents: Union[Layout_Context.InvalidExtents, List[int]] = \ - [0, 0, 1, 1] - - # Location-string variables - self.__loc_variables = loc_vars if loc_vars is not None else {} - self.__loc_variables_changed = True if loc_vars is not None else False - - self.__extensions = ExtensionManager() - filename = layout.GetFilename() - if filename: - self.__extensions.AddPath(os.path.dirname(filename)) - - self.__elements = ElementSet(self, self.__extensions) - self.SetHC(self.__hc) - - self.__layout = layout - self.__layout.LinkLayoutContext(self) - self.__PullFromLayout() - - # Number of stabbing query results to store at a time - self.__cache_size = 20 - - # get visible clock stuff - self.__number_elements = 0 - self.__visible_clocks: Optional[Tuple[int, ...]] = None - - # Uop highlighting list - self.__highlighted_uops: Set[int] = set() - self.__previously_highlighted_uops: Set[int] = set() - self.__search_results: Set[int] = set() - self.__previous_search_results: Set[int] = set() - - # Returns the SearchHandle held by this context. - @property - def searchhandle(self) -> SearchHandle: - return self.__searchhandle - - # Returns the DatabaseHandle held by this context. - @property - def dbhandle(self) -> DatabaseHandle: - return self.__dbhandle - - # Returns the hypercycle this Context is currently centered on - @property - def hc(self) -> int: - return self.__hc - - # Populate our elements from current layout - # At the moment, call this only once. - def __PullFromLayout(self) -> None: - assert self.__layout is not None - # Flatten out struture of layout for easy access - for e in self.__layout.GetElements(): - self.AddElement(e) - if e.HasChildren(): - children = e.GetChildren() - for child in children: - self.AddElement(child) - - # Returns the location-string variables dictionary for this layout context - def GetLocationVariables(self) -> Dict[str, str]: - return self.__loc_variables - - def GetLocationVariablesChanged(self) -> bool: - return self.__loc_variables_changed - - # Updates location variables based on any new element locations created - # with variables in them - def UpdateLocationVariables(self) -> None: - assert self.__layout is not None - for el in self.__layout.GetElements(): - if el.HasProperty('LocationString'): - loc_str = cast(str, el.GetProperty('LocationString')) - loc_vars = LocationManager.findLocationVariables(loc_str) - for k, v in loc_vars: - if k in self.__loc_variables and \ - self.__loc_variables[k] != v: - # Do not add these to the table since there is a - # conflict - pass - # @todo Represent these conflicting variable defaults - # somehow so that they can be resolved in the location - # variables dialog - else: - self.__loc_variables[k] = v - self.__loc_variables_changed = True - - def AckLocationVariablesChanged(self) -> None: - self.__loc_variables_changed = False - - # Adds a new element to the OrderedDict - # @param e Element to add - # @param after_pin PIN of element after which to insert this element - # @profile - def AddElement(self, - e: Element, - after_pins: List[Optional[int]] = [None]) -> Element: - self.__elements.AddElement(e, after_pins=after_pins) - - # Update extents. This can only increase - if isinstance(self.__extents, self.InvalidExtents): - self.__extents = [0, 0, 1, 1] - - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - r = x + w - b = y + h - self.__extents[self.EXTENT_R] = max(r, self.__extents[self.EXTENT_R]) - self.__extents[self.EXTENT_B] = max(b, self.__extents[self.EXTENT_B]) - - return e - - # Remove an element from the ElementSet - def RemoveElement(self, e: Element) -> None: - self.__elements.RemoveElement(e) - - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - r = x + w - b = y + h - - if isinstance(self.__extents, self.InvalidExtents): - return # Already invalid, nothing to do here. Will recalc later - - # If this element was at the edge, we need a full recalc. - # Invalidate extents and recalc when asked - # Just in case extents were wrong and r was greater, use >= cur extent - if r >= self.__extents[self.EXTENT_R]: - self.__extents = self.InvalidExtents() - elif b >= self.__extents[self.EXTENT_B]: - self.__extents = self.InvalidExtents() - - # Move a set of elements to above the highest entry in a list - # @above_pin_list List of pins above which to move each element in the - # elements list. May contain None at the end to indicate "move to top" - # @note elements will retain their own ordering - def MoveElementsAbovePINs(self, - elements: List[Element], - above_pin_list: List[Optional[int]]) -> None: - assert self.__layout is not None - for e in elements: - self.__layout.RemoveElement(e) - - if not above_pin_list: - above_pin_list = [-1] - - prev_pins_list = above_pin_list[:] - for e in elements: - self.__layout.AddElement(e, follows_pins=prev_pins_list) - prev_pins_list.append(e.GetPIN()) - - # Move a set of elements to below the lowest entry in a list - # @below_pin_list List of pins below which to move each element in the - # elements list. May contain -1 at the start to indicate "move to bottom" - # @note elements will retain their own ordering - def MoveElementsBelowPINs(self, - elements: List[Element], - below_pin_list: List[Optional[int]]) -> None: - assert self.__layout is not None - for e in elements: - self.__layout.RemoveElement(e) - - if not below_pin_list: - below_pin_list = [None] - - prev_pins_list = below_pin_list[:] - for e in elements: - self.__layout.AddElement(e, follows_pins=prev_pins_list) - prev_pins_list.append(e.GetPIN()) - - # If a property for an Element is changed such that it will no longer be - # correctly sorted in the ElementSet, this method will figure out where - # it goes - def ReSort(self, e: Element, t_off: int, loc: str) -> None: - id = self.__dbhandle.database.location_manager.getLocationInfo(loc, self.__loc_variables)[0] # noqa: E501 - self.__elements.ReSort(e, t_off, id) - - # Resort all elements because some locations variable has changed and all - # elements may be effected - def ReSortAll(self) -> None: - self.__elements.ReSortAll() - - # An element in the layout has moved - def ElementMoved(self, e: Element) -> None: - # Because this can be called many times during a mass-move or resize, - # simply invalidate the extents - self.__extents = self.InvalidExtents() - - # Returns True if element was moved - def IsElementMoved(self) -> bool: - return isinstance(self.__extents, self.InvalidExtents) - - # Used in the event that a property was changed for an element which may - # require an updated value to be displayed - def ReValue(self, e: Element) -> None: - self.__elements.ReValue(e) - - # NOTE: This would be a good place to update any variables extracted - # from this element. Note that this might be very slow here since this - # method can be invoked during mass-updates - - # Used in the event that many elements were changed (e.g. a location string - # variable was updated) - def ReValueAll(self) -> None: - self.__elements.ReValueAll() - - # Updates this context's elements for the curent cycle - def Update(self) -> None: - self.__elements.Update(self.__hc) - - # update that is called every major display update - def MicroUpdate(self) -> None: - self.__elements.MicroUpdate() - - # Force a DB update. - def DBUpdate(self) -> None: - self.__elements.DBUpdate() - - # Force a full update. - def FullUpdate(self) -> None: - self.__elements.FullUpdate() - - # Force a full redraw of all elements without marking them as changed - def FullRedraw(self) -> None: - self.__elements.RedrawAll() - - # Returns the layout which this Context is referrencing - def GetLayout(self) -> Optional[Layout]: - return self.__layout - - def GetExtensionManager(self) -> ExtensionManager: - return self.__extensions - - def __GetLocationInfo(self, element: Element) -> LocationType: - return self.__db.location_manager.getLocationInfoNoVars( - cast(str, element.GetProperty('LocationString')) - ) - - # Return a set of all clockid's referred to - def GetVisibleClocks(self) -> Optional[Tuple[int, ...]]: - elements = self.GetElements() - if self.__number_elements != len(elements): - clock_set = set() - - for element in elements: - if element.NeedsDatabase(): - info = self.__GetLocationInfo(element) - if info[0] != self.__db.location_manager.INVALID_LOCATION_ID: # noqa: E501 - # only add if valid location - clock_set.add(info[2]) - clocks = tuple(clock_set) - self.__visible_clocks = clocks - self.__number_elements = len(elements) - - return self.__visible_clocks - - def GetLocationId(self, element: Element) -> int: - return self.__GetLocationInfo(element)[0] - - # Return a set of all locations referred to - def GetVisibleLocations(self) -> Set[int]: - locations = set() - for element in self.GetElements(): - if element.NeedsDatabase(): - locations.add(self.GetLocationId(element)) - return locations - - # Returns the All Objects - def GetElementPairs(self) -> List[Element_Value]: - return self.__elements.GetPairs() - - # Returns all pairs suitable for drawing - def GetDrawPairs( - self, - bounds: Optional[Tuple[int, int, int, int]] - ) -> List[Element_Value]: - return self.__elements.GetDrawPairs(bounds) - - def GetVisibilityTick(self) -> int: - return self.__elements.GetVisibilityTick() - - def GetElements(self) -> List[Element]: - return self.__elements.GetElements() - - def GetElementPair(self, e: Element) -> Optional[Element_Value]: - return self.__elements.GetPair(e) - - def UpdateElementExtents(self) -> None: - self.__elements.UpdateElementBounds() - self.__extents = self.InvalidExtents() - self.GetElementExtents() - - def GetElementExtents(self) -> Tuple[int, int, int, int]: - ''' - Returns the left,right,top,bottom extents of the layout based on what - elements it contains - @note Recalculates extents if necessary - @return (left,top,right,bottom) - ''' - if isinstance(self.__extents, self.InvalidExtents): - self.__extents = [0, 0, 1, 1] - els = self.GetElements() - for e in els: - (x, y) = cast(Tuple[int, int], e.GetProperty('position')) - (w, h) = cast(Tuple[int, int], e.GetProperty('dimensions')) - self.__extents[self.EXTENT_R] = \ - max(self.__extents[self.EXTENT_R], x + w) - self.__extents[self.EXTENT_B] = \ - max(self.__extents[self.EXTENT_B], y + h) - - return cast(Tuple[int, int, int, int], tuple(self.__extents)) - - # For testing purposes only - def __repr__(self) -> str: - return f'' - - # Jumps context to a specific tick. - # @param hc Hypercycle (tick) to jump to. This tick will be constrained - # to the endpoints of this database handle's file range - # @note Directly refreshes the associated Frame if not attached to a group. - # Otherwise, the this context and the associated frame will be refreshed - # through the group, when it invokes 'RefreshFrame' on all its contained - # layout - # contexts - # @todo rework this - # - # Performs new queries at the chosen tick and updates element data - def GoToHC(self, - hc: Optional[int] = None, - no_broadcast: bool = False) -> None: - # show busy cursor every call - assert self.__frame is not None - frame = self.__frame() - if frame: - frame.SetBusy(True) - - if hc is None: - hc = self.__hc - hc = self.__ClampHC(hc) - if self.__group is not None: - self.__group.MoveTo(hc, self, no_broadcast=no_broadcast) - else: - self.SetHC(hc, no_broadcast=no_broadcast) - self.RefreshFrame() - - # set cursor back - if frame: - frame.SetBusy(False) - - # Sets the current tick and updates. - # This does not notify groups and is an internal method - # @param hc New hypercycle (tick) - # @note Does not refresh. Refresh must be called separately (or use GoToHC - # which Refreshes or notififes a group which indirectly refreshes). - def SetHC(self, hc: int, no_broadcast: bool = False) -> None: - self.__hc = hc - if not no_broadcast: - self.__elements.HandleCycleChangedEvent() - self.Update() - - # Refresh this context (and its associated frame) - def RefreshFrame(self) -> None: - assert self.__frame, \ - 'A Layout_Context should always have a frame before attempting ' \ - 'a RefreshFrame call' - self.__elements.MetaUpdate(self.__hc) - frame = self.__frame() - if frame: - frame.Refresh() - - def GetHC(self) -> int: - ''' - Returns the current hypercycle (tick) for this layout context - ''' - return self.__hc - - def SetGroup(self, group: Group) -> None: - assert self.__group is None, \ - '(for now) SetGroup cannot be called on a LayoutContext after ' \ - 'it already has a group' - assert group is not None, \ - 'SetGroup parameter group must not be None' - logging.getLogger('LayoutContext').debug( - 'Context %s adding to group %s', - self, - group - ) - self.__group = group - self.__group.AddContext(self) - - def LeaveGroup(self) -> None: - assert self.__group is not None, \ - 'LeaveGroup cannot be called on a LayoutContext before it has ' \ - 'joined a group' - logging.getLogger('LayoutContext').debug('Context %s leaving group %s', - self, - self.__group) - self.__group.RemoveContext(self) - - def GetGroup(self) -> Optional[Group]: - return self.__group - - def SetFrame(self, frame: Layout_Frame) -> None: - assert self.__frame is None, \ - 'SetFrame cannot be called on a LayoutContext after it already ' \ - 'has a frame' - assert frame is not None, \ - 'SetFrame parameter frame must not be None' - logging.getLogger('LayoutContext').debug( - 'Context %s associated with frame %s', - self, - frame - ) - self.__frame = weakref.ref(frame) - - # Returns the frame associated with this context. If the associated frame - # was destroyed (or no Frame associated), returns None - def GetFrame(self) -> Optional[Layout_Frame]: - if self.__frame is None: - return None - return self.__frame() # May be None - - # Clamp the HC to the file extents - def __ClampHC(self, hc: int) -> int: - hc = max(hc, self.__qapi.getFileStart()) - # End is normally exclusive - hc = min(hc, self.__qapi.getFileInclusiveEnd()) - return hc - - # Returns a list of all Elements beneath the given point - # @param pt Point to test for collision with elements - # @param include_subelements Should subelements be searched (e.g. schedule - # line within a schedule) - # @param include_nondrawables Should selectable elements be returned even - # if they aren't drawable? Depth ordering might be lost when including non - # drawables - # Subelements are fake elements generated by elements on a collision - def DetectCollision( - self, - pt: Union[Tuple[int, int], wx.Point], - include_subelements: bool = False, - include_nondrawables: bool = False - ) -> List[Element_Value]: - mx, my = pt - res: List[Element_Value] = [] - # Search draw pairs instead of all element pairs because they are - # (1) visible - # (2) sorted by depth - - # Get bounds for quad-tree query - assert self.__frame is not None - frame = self.__frame() - if frame: - bounds = frame.GetCanvas().GetBounds() - else: - bounds = None - - # Query by draw pairs to get depth order correct - if include_nondrawables: - pairs = self.GetElementPairs() - else: - pairs = self.GetDrawPairs(bounds=bounds) - # After GetDrawPairs - vis_tick = self.__elements.GetVisibilityTick() - - exclude_offscreen = not include_nondrawables and bounds is not None - for e in pairs: - if exclude_offscreen and e.GetVisibilityTick() != vis_tick: - continue # Skip: this is off-screen - - element = e.GetElement() - x, y = cast(Tuple[int, int], element.GetProperty('position')) - w, h = cast(Tuple[int, int], element.GetProperty('dimensions')) - if x <= mx <= (x + w) and y <= my <= (y + h): - if include_subelements: - et = element.GetProperty('type') - if et == 'schedule': - element = cast('ScheduleElement', element) - sl = element.DetectCollision((mx, my)) - if sl and sl.GetProperty('type') == 'schedule_line': - sl = cast('ScheduleLineElement', sl) - # Hierarchical point containment test assumes that - # schedule objects contain schedule lines - mx, my = pt - c_x, c_y = cast(Tuple[int, int], - sl.GetProperty('position')) - loc_x = mx - c_x - loc_y = my - c_y - - # Go inside this scheule line - sub_object = sl.DetectCollision((loc_x, loc_y), e) - if sub_object: - res.append(sub_object) - elif et == 'schedule_line': - element = cast('ScheduleLineElement', element) - mx, my = pt - c_x, c_y = cast(Tuple[int, int], - element.GetProperty('position')) - loc_x = mx - c_x - loc_y = my - c_y - - # Go inside this element - sub_object = element.DetectCollision((loc_x, loc_y), e) - if sub_object: - res.append(sub_object) - else: - res.append(e) - else: - # just attach element - res.append(e) - - return res - - def GetLocationPeriod(self, location_string: str) -> int: - ''' - a function for getting the period of the clock at a certain time - ''' - db = self.dbhandle.database - clock = db.location_manager.getLocationInfo( - location_string, - self.GetLocationVariables() - )[2] - return db.clock_manager.getClockDomain(clock).tick_period - - def GetTransactionFields(self, - time: int, - location_string: str, - fields: List[str]) -> Dict[str, Any]: - ''' - performs random-access query at time and place and returns requested - attributes in dictionary - @return Dictionary of results {field:value} - ''' - # No results if time is outside of the currently loaded window. - # Cannot do a reasonably fast query and there is no way that the data - # is currently visible to the user anyway - dbapi = self.dbhandle.database.api - if time < dbapi.getWindowLeft() or time >= dbapi.getWindowRight(): - return {} - - results = {} - - def callback(t: int, tapi: TransactionDatabase) -> None: - assert t == time, f'bad tick {t}' - loc_mgr = self.dbhandle.database.location_manager - location = loc_mgr.getLocationInfo(location_string, - self.GetLocationVariables())[0] - trans_proxy = tapi.getTransactionProxy(location) - if trans_proxy: - for field in fields: - if field == 'time': - value = t - else: - fake_element = FakeElement() - fake_element.SetProperty('LocationString', - location_string) - value = content.ProcessContent( - field, - trans_proxy, - fake_element, - self.dbhandle, - t, - self.GetLocationVariables() - ) - results[field] = value - - self.dbhandle.query(time, time, callback, mod_tracking=False) - return results - - def SearchResultHash(self, start: int, location: int) -> int: - return hash(f'{start}:{location}') - - def AddSearchResult(self, search_entry: SearchResult) -> None: - self.__search_results.add( - self.SearchResultHash(search_entry['start'], - search_entry['location']) - ) - - def ClearSearchResults(self) -> None: - self.__previous_search_results.update(self.__search_results) - self.__search_results.clear() - - @overload - def IsSearchResult(self, start: Optional[int]) -> bool: ... - - @overload - def IsSearchResult(self, start: int, location: int) -> bool: ... - - def IsSearchResult(self, - start: Optional[int], - location: Optional[int] = None) -> bool: - if location is None: - # start is actually the hash - return start in self.__search_results - else: - assert start is not None - return self.IsSearchResult(self.SearchResultHash(start, location)) - - @overload - def WasSearchResult(self, start: Optional[int]) -> bool: ... - - @overload - def WasSearchResult(self, start: int, location: int) -> bool: ... - - def WasSearchResult(self, - start: Optional[int], - location: Optional[int] = None) -> bool: - if location is None: - # start is actually the hash - return start in self.__search_results - else: - assert start is not None - return self.WasSearchResult(self.SearchResultHash(start, location)) - - def HighlightUop(self, uid: Optional[Union[int, str]]) -> None: - ''' - Highlight the uop with the given annotation string - ''' - if isinstance(uid, str): - self.HighlightUop(highlighting_utils.GetUopUid(uid)) - elif uid is not None: - self.__highlighted_uops.add(uid) - - def UnhighlightUop(self, uid: Optional[Union[int, str]]) -> None: - ''' - Unhighlight the uop with the given annotation string - ''' - if isinstance(uid, str): - self.UnhighlightUop(highlighting_utils.GetUopUid(uid)) - elif uid is not None: - if uid in self.__highlighted_uops: - self.__highlighted_uops.remove(uid) - self.__previously_highlighted_uops.add(uid) - - # Check if a uop has been highlighted (by UID) - def IsUopUidHighlighted(self, uop_uid: Optional[int]) -> bool: - return uop_uid in self.__highlighted_uops - - # Check if a uop has been unhighlighted (by UID), but not yet redrawn - def WasUopUidHighlighted(self, uop_uid: Optional[int]) -> bool: - return uop_uid in self.__previously_highlighted_uops - - # Check if a uop has been highlighted (by annotation string) - def IsUopHighlighted(self, uid: Optional[Union[int, str]]) -> bool: - if isinstance(uid, str): - return self.IsUopHighlighted(highlighting_utils.GetUopUid(uid)) - return uid in self.__highlighted_uops - - # Check if a uop has been unhighlighted (by annotation string), but not yet - # redrawn - def WasUopHighlighted(self, uid: Optional[Union[int, str]]) -> bool: - if isinstance(uid, str): - return self.WasUopHighlighted(highlighting_utils.GetUopUid(uid)) - return uid in self.__previously_highlighted_uops - - # Redraw elements that have changed their highlighting state - def RedrawHighlightedElements(self) -> None: - self.__elements.RedrawHighlighted() - self.__previously_highlighted_uops.clear() - self.__previous_search_results.clear() diff --git a/helios/pipeViewer/pipe_view/model/layout_delta.py b/helios/pipeViewer/pipe_view/model/layout_delta.py deleted file mode 100644 index c47fd4b7ba..0000000000 --- a/helios/pipeViewer/pipe_view/model/layout_delta.py +++ /dev/null @@ -1,170 +0,0 @@ -# Selection Mgrs are in charge of drawing to the canvas a demonstration of -# the current selection, hence importing wx -from __future__ import annotations -import sys -from typing import List, Tuple, cast, TYPE_CHECKING - -from .element import Element, MultiElement - -if TYPE_CHECKING: - from .layout import Layout - - -# Represents a checkpoint between two layouts. -# The checkpoint can be applied or removed. The content of the checkpoint can -# effecively be thought of as a selection. Checkpoints can have a net null -# effect if they are meant simply to represent a change in selection -class Checkpoint: - - # Initialize the checkpoint. Before the checkpoint is used, after() should - # be called to capture the state AFTER the checkpoint - # @desc Description of the change - # @profile - def __init__(self, - layout: Layout, - before: List[Element], - desc: str) -> None: - assert layout is not None - self.__layout = layout - self.__description = desc - self.__before_pins = [el.GetPIN() for el in before] - self.__before = [ - self.__layout.CreateElement( - el, - initial_properties=el.GetProperties(), - element_type=cast(str, el.GetProperty('type')), - parent=el.GetParent() - ) - for el in before - ] - self.__before_follows_pins = self.__layout.DeterminePriorPins(before) - self.__after_pins: List[int] = [] - self.__after: List[Element] = [] - self.__after_follows_pins: List[List[int]] = [] - - # Sets checkpoint state afterward - def after(self, after: List[Element]) -> None: - self.__after_pins = [el.GetPIN() for el in after] - self.__after = [ - self.__layout.CreateElement( - el, - initial_properties=el.GetProperties(), - element_type=cast(str, el.GetProperty('type')), - parent=el.GetParent() - ) - for el in after - ] - self.__after_follows_pins = self.__layout.DeterminePriorPins(after) - - # Decsription of the change captured by this checkpoint - @property - def description(self) -> str: - return self.__description - - # Apply the checkpoint - # Returns a 2-tuple: (failures, created_elements). Number of failures is 0 - # on success, or a number of elements in the 'before' state that could not - # be found (likely indicating that the delta should not be applied here). - # Also counts PIN collisions when trying to add elements with PINs that - # were already in the layout. Checkpoint is still applied as much as - # possible since failures here are probably not recoverable. The second - # item in the result tuple is all of the elements modified after applying - # this delta. This should generally be used as the user's new selection - def apply(self) -> Tuple[int, List[Element]]: - errors = 0 - # Find all 'before' elements to make sure we're in a sane state - for pin in self.__before_pins: - el = self.__layout.FindByPIN(pin) - if el is not None: - self.__layout.RemoveElement(el) - else: - print('Error while applying layout delta. Element with pin ' - f'{pin} could not be found', - file=sys.stderr) - errors += 1 - - # Add all the 'after' elements - created_elements = [] - for el, pin, follows_pins in zip(self.__after, - self.__after_pins, - self.__after_follows_pins): - if self.__layout.FindByPIN(pin) is not None: - print('Error while applying layout delta. Element with pin ' - f'{pin} already exists.', - file=sys.stderr) - errors += 1 - else: - eltype = cast(str, el.GetProperty('type')) - parent = el.GetParent() - new_el = self.__layout.CreateAndAddElement( - el, - initial_properties=el.GetProperties(), - element_type=eltype, - parent=parent, - force_pin=pin, - follows_pins=follows_pins, - skip_list=parent is not None - ) - if parent is not None: - parent = cast(MultiElement, parent) - parent.AddChild(new_el) - parent.SetNeedsRedraw() - new_el.SetNeedsRedraw() - created_elements.append(new_el) - assert new_el.GetPIN() == pin - - return (errors, created_elements) - - # Remove the checkpoint (revert) - # Returns a 2-tuple: (failures, created_elements). Number of failures is 0 - # on success, or a number of elements in the 'after' state that could not - # be found (likely indicating that the delta should not be applied here). - # Also counts PIN collisions when trying to add elements with PINs that - # were already in the layout. Checkpoint is still removed as much as - # possible since failures here are probably not recoverable. The second - # item in the result tuple is all of the elements modified after applying - # this delta. This should generally be used as the user's new selection - def remove(self) -> Tuple[int, List[Element]]: - errors = 0 - # Find all 'after' elements to make sure we're in a sane state - for pin in self.__after_pins: - el = self.__layout.FindByPIN(pin) - if el is not None: - self.__layout.RemoveElement(el) - else: - print('Error while removing layout delta. Element with pin ' - f'{pin} could not be found', - file=sys.stderr) - errors += 1 - - # Add all the 'after' elements - created_elements = [] - for el, pin, follows_pins in zip(self.__before, - self.__before_pins, - self.__before_follows_pins): - if self.__layout.FindByPIN(pin) is not None: - print('Error while removing layout delta. Element with pin ' - f'{pin} already exists.', - file=sys.stderr) - errors += 1 - else: - eltype = cast(str, el.GetProperty('type')) - parent = el.GetParent() - new_el = self.__layout.CreateAndAddElement( - el, - initial_properties=el.GetProperties(), - element_type=eltype, - parent=parent, - force_pin=pin, - follows_pins=follows_pins, - skip_list=parent is not None - ) - if parent is not None: - parent = cast(MultiElement, parent) - parent.AddChild(new_el) - parent.SetNeedsRedraw() - new_el.SetNeedsRedraw() - created_elements.append(new_el) - assert new_el.GetPIN() == pin - - return (errors, created_elements) diff --git a/helios/pipeViewer/pipe_view/model/location_manager.py b/helios/pipeViewer/pipe_view/model/location_manager.py deleted file mode 100644 index 601f7954f0..0000000000 --- a/helios/pipeViewer/pipe_view/model/location_manager.py +++ /dev/null @@ -1,307 +0,0 @@ -# @package location_manager.py -# @brief Consumes argos location files through LocationManager class - -from __future__ import annotations -import functools -import re -import time -from typing import Dict, List, Optional, TextIO, Tuple - -# Expression for searching for variables in location strings -# @note value can be empty string -LOCATION_STRING_VARIABLE_RE = re.compile(r'{(\w+)\s*(?:=\s*(\w*))?}') - -LocationTree = Dict[str, 'LocationTree'] -LocationType = Tuple[int, str, int] -__LocationDict = Dict[str, LocationType] - - -# Consumes an Argos location file and provides a means of lookup up location -# IDs via location strings -# -# Also allows browsing of availble locations -class LocationManager: - - # Describes the location file - LOCATION_FILE_EXTENSION = 'location.dat' - - # Expeted version number from file. Otherwise the data cannot be consumed - VERSION_NUMBER = 1 - - # Integer refering to no clock when seen as a clock ID in this location - # file - NO_CLOCK = -1 - - # Integer refering to invalid location ID - INVALID_LOCATION_ID = -1 - - # Tuple indicating that a location string was not found in the map when a - # location was queried through getLocationInfo - LOC_NOT_FOUND = (INVALID_LOCATION_ID, '', NO_CLOCK) - - FILE_WAIT_TIMEOUT = 60 - - # Constructor - # @param prefix Argos transaction database prefix to open. - # LOCATION_FILE_EXTENSION will be appended to determine the actual - # filename to open - def __init__(self, prefix: str, update_enabled: bool) -> None: - # { Location Name : ( LocationID, name, ClockID ) } - self.__locs: __LocationDict = {} - self.__id_to_string = {} # { LocationID : Location Name } - self.__loc_tree: LocationTree = {} # Location tree - self.__regex_cache: Dict[str, str] = {} - try_count = self.FILE_WAIT_TIMEOUT - - index_file_exists = False - - filename = prefix + self.LOCATION_FILE_EXTENSION - while update_enabled and not index_file_exists: - try: - with open(filename) as f: - index_file_exists = True - except IOError as e: - if try_count == self.FILE_WAIT_TIMEOUT: - print(f"Index file{filename} doesn't exist yet.") - elif try_count == 0: - raise e - print("Retrying:", self.FILE_WAIT_TIMEOUT - try_count) - try_count -= 1 - time.sleep(1) - - with open(filename, 'r') as f: - # Find version information - while 1: - first = self.__findNextLine(f) - assert first is not None - if first == '': - continue - - try: - els = first.split(' \t#') - ver = int(els[0]) - except ValueError: - if update_enabled and (not first and try_count > 0): - print("Could not read version string. File may be " - "in-progress and not flushed yet. Retrying: " - f"{self.FILE_WAIT_TIMEOUT - try_count}") - try_count -= 1 - f.seek(0) - time.sleep(1) - continue - else: - raise ValueError('Found an unparsable (non-integer) ' - f'version number string: "{first}". ' - f'Expected "{self.VERSION_NUMBER}".') - - if ver != self.VERSION_NUMBER: - raise ValueError('Found incorrect version number: ' - f'"{ver}". Expected ' - f'"{self.VERSION_NUMBER}". This reader ' - 'may need to be updated') - break - - # Read subsequent location lines - # ,,\n - while 1: - ln = self.__findNextLine(f) - if ln == '': - continue - if ln is None: - break - - els = ln.split(',') - - try: - uid_str, name, clockid_str = els[:3] - uid = int(uid_str) - clockid = int(clockid_str) - except ValueError: - raise ValueError(f'Failed to parse line "{ln}"') - - self.__insertLocationInTree(name) - self.__locs[name] = (uid, name, clockid) - self.__id_to_string[uid] = name - - # Gets a tree of location strings represented as a dictionary with keys - # equal to objects between dots in location strings: Leaf nodes are empty - # dictionaries - # - # Example: - # @code - # { 'top' : { 'core0': { 'fetch': {}, - # 'alu0': {} }, - # 'core1': {}, - # }, - # } - # @endcode - @property - def location_tree(self) -> LocationTree: - return self.__loc_tree - - # Gets next line from file which is not a comment. - # Strips comments on the line and whitespace from each end - # @param f File to read next line from - # @return '' if line is empty or comment, None if EOF is reached - def __findNextLine(self, f: TextIO) -> Optional[str]: - ln = f.readline().strip() - if ln == '': - return None - - if ln.find('#') == 0: - return '' - - pos = ln.find('#') - if pos >= 0: - ln = ln[:pos] - - ln = ln.strip() - - return ln - - # Gets a tuple containing location information associated with a location - # string - # @param locname Location name string to lookup - # @param variables dict of variables that can be substituted into the - # locname - # @return 3-tuple (locationID, location name, clock id) if found. If not - # found, returns LOC_NOT_FOUND - def getLocationInfo(self, - locname: str, - variables: Dict[str, str], - loc_vars_changed: bool = False) -> LocationType: - if not isinstance(locname, str): - raise TypeError(f'locname must be a str, is a {type(locname)}') - - resolved_loc = self.replaceLocationVariables(locname, - variables, - loc_vars_changed) - try: - return self.__locs[resolved_loc] - except KeyError: - return self.LOC_NOT_FOUND - - # Gets a tuple containing location information associated with a location - # string - # @param locname Location name string to lookup - # @return 3-tuple (locationID, location name, clock id) if found. If not - # found, returns LOC_NOT_FOUND - def getLocationInfoNoVars(self, locname: str) -> LocationType: - if not isinstance(locname, str): - raise TypeError(f'locname must be a str, is a {type(locname)}') - try: - return self.__locs[locname] - except KeyError: - return self.LOC_NOT_FOUND - - # Gets a location string with variables in it replaced by the given - # variables dictionary. Caches results to improve performance for repeated - # calls. - # @param locname Location string - # @param variables dict of variables and values - # @param loc_vars_changed True if we need to refresh the entry in the - # regex cache - def replaceLocationVariables(self, - locname: str, - variables: Dict[str, str], - loc_vars_changed: bool = False) -> str: - if loc_vars_changed: - self.__regex_cache[locname] = LOCATION_STRING_VARIABLE_RE.sub( - functools.partial(self.__replaceVariable, variables), locname - ) - try: - return self.__regex_cache[locname] - except KeyError: - self.__regex_cache[locname] = LOCATION_STRING_VARIABLE_RE.sub( - functools.partial(self.__replaceVariable, variables), locname - ) - return self.__regex_cache[locname] - - # Find all location variables in location string - # Returns a list of tuples (variable name, value) representing each - # variable found - @staticmethod - def findLocationVariables(locname: str) -> List[Tuple[str, str]]: - if not isinstance(locname, str): - raise TypeError(f'locname must be a str, is a {type(locname)}') - - found = [] - matches = LOCATION_STRING_VARIABLE_RE.findall(locname) - for m in matches: - found.append((m[0], m[1])) - - return found - - # Gets location srings in no particular order - # @note This is slow because this list is generated on request - # @note Order of results is not necessarily consistent - # @return List of str instances containing names of all known locations - def getLocationStrings(self) -> List[str]: - return [x[1] for x in self.__locs.values()] - - # Gets location strings for the given locid - # @param locid Location ID to lookup a string for - # @return Location string (str) if found, None otherwise - def getLocationString(self, locid: int) -> Optional[str]: - return self.__id_to_string.get(locid, None) - - # Gets a sequence of potential location string completions given the - # input \a locname - # @param locname Location name string input - # @warning This is a very slow method since it iterates the entire list - # @todo Use an internal tree of location names (between '.'s) for - # generating completions more quickly - def getLocationStringCompletions(self, locname: str) -> List[str]: - results = [] - for k, v in self.__locs.items(): - _, n, _ = v - if locname == n[:len(locname)]: - results.append(n[len(locname):]) - - return results - - # Returns the maximum location ID known to this manager - def getMaxLocationID(self) -> int: - if len(self.__id_to_string) == 0: - return 0 - return max(self.__id_to_string.keys()) - - def __len__(self) -> int: - return len(self.__locs) - - def __str__(self) -> str: - return f'' - - def __repr__(self) -> str: - return self.__str__() - - def __insertLocationInTree(self, loc_name: str) -> None: - ''' - Parses a location name by '.' and builds a tree from its content - ''' - parent = self.__loc_tree - paths = loc_name.split('.') - for obj in paths: - if obj not in parent: - parent[obj] = {} - parent = parent[obj] - - # Handles replacing a word in a string with - # @param variables dict of variable names with values - # @param match Match regex result. Contains 2 groups: the variable name - # and default value (which is None if no default was supplied) - def __replaceVariable(self, - variables: Dict[str, str], - match: re.Match) -> str: - # Get the replacement form the variables dictionary - replacement = variables.get(match.group(1), None) - if replacement is None: - if match.group(1) is not None: - # Use the default value associated with with variable in the - # location string - replacement = match.group(2) - else: - return f'' - else: - pass - return replacement diff --git a/helios/pipeViewer/pipe_view/model/quad_tree.py b/helios/pipeViewer/pipe_view/model/quad_tree.py deleted file mode 100644 index 592773aea9..0000000000 --- a/helios/pipeViewer/pipe_view/model/quad_tree.py +++ /dev/null @@ -1,391 +0,0 @@ -from __future__ import annotations -from typing import Dict, List, Optional, Set, Tuple - -from .element_value import Element_Value - - -class QuadNode: - ''' - Node used by QuadTree - ''' - - def __init__(self, - bounds: Tuple[int, int, int, int], - contents: List[QuadNode], - elements: List[Element_Value], - parent: Optional[QuadNode] = None, - parent_index: Optional[int] = None) -> None: - self.bounds = bounds - self.contents = contents # list of QuadNodes - self.elements = elements - self.parent_index = parent_index - self.parent = parent - - __CalcChildBoundsRetType = Tuple[int, int, int, int, int, int] - - def __CalculateChildNodeBounds(self) -> __CalcChildBoundsRetType: - min_x, min_y, max_x, max_y = self.bounds - midpoint_x = int((max_x + min_x) / 2.0) - midpoint_y = int((max_y + min_y) / 2.0) - return (min_x, min_y, max_x, max_y, midpoint_x, midpoint_y) - - def BisectCreateNodes(self) -> None: - if not self.contents: - min_x, min_y, max_x, max_y = self.bounds - midpoint_x = int((max_x + min_x) / 2.0) - midpoint_y = int((max_y + min_y) / 2.0) - self.contents = [ - QuadNode((min_x, min_y, midpoint_x, midpoint_y), - [], - [], - self, - 0), - QuadNode((midpoint_x, min_y, max_x, midpoint_y), - [], - [], - self, - 1), - QuadNode((min_x, midpoint_y, midpoint_x, max_y), - [], - [], - self, - 2), - QuadNode((midpoint_x, midpoint_y, max_x, max_y), - [], - [], - self, - 3) - ] - - def UpdateBounds(self, new_bounds: Tuple[int, int, int, int]) -> None: - self.bounds = new_bounds - if self.contents: - (min_x, - min_y, - max_x, - max_y, - midpoint_x, - midpoint_y) = self.__CalculateChildNodeBounds() - self.contents[0].UpdateBounds( - (min_x, - min_y, - midpoint_x, - midpoint_y) - ) - self.contents[1].UpdateBounds( - (midpoint_x, - min_y, - max_x, - midpoint_y) - ) - self.contents[2].UpdateBounds( - (min_x, - midpoint_y, - midpoint_x, - max_y) - ) - self.contents[3].UpdateBounds( - (midpoint_x, - midpoint_y, - max_x, - max_y) - ) - - def SetVisibilityTickOnAllChildElements(self, vis_tick: int) -> None: - if self.elements: # leaves only - for x in self.elements: - x.SetVisibilityTick(vis_tick) - else: - for cnode in self.contents: - cnode.SetVisibilityTickOnAllChildElements(vis_tick) - - def GetAllChildElements(self) -> List[Element_Value]: - # contains duplicates - l: List[Element_Value] = [] - self.__GetAllChildren(self, l) - return l - - def __GetAllChildren(self, - node: QuadNode, - output_list: List[Element_Value]) -> None: - if node.elements and not node.contents: # leaves only - output_list.extend(node.elements) - else: - for cnode in node.contents: - self.__GetAllChildren(cnode, output_list) - - def GetAllBoxes( - self, - node: Optional[QuadNode] = None - ) -> List[Tuple[int, int, int, int]]: - if node is None: - node = self - if not node.contents: - bounds = [node.bounds] - else: - bounds = [] - for content in node.contents: - bounds.extend(self.GetAllBoxes(content)) - return bounds - - # Debug printout for this node and nodes below it. - def PrintRecursive(self, - node: Optional[QuadNode] = None, - level: int = 0) -> None: - if node is None: - node = self - space = ' ' * level - print('%sNode %s:' % (space, node)) - print('%s Bounds: %s' % (space, node.bounds)) - print('%s Elements: %s' % (space, str(node.elements))) - print('%s Contents:' % (space)) - for content in node.contents: - self.PrintRecursive(content, level + 1) - - -# A small QuadTree-based container for pairs. -class QuadTree: - # when more than this number of objects are dirty, the bsp rebuilds - FULL_REBUILD_THRESHOLD = 30 - WIDTH_LIMIT = 100 - - def __init__(self) -> None: - self.__objects: List[Element_Value] = [] - self.__tree: Optional[QuadNode] = None - - # Dictionary keyed by objects which stores - # all nodes object is in - # e.g. self.__lookup_table[myObj] = [QuadNode_instance1, QuadNode_instance2] # noqa: E501 - self.__lookup_table: Dict[Element_Value, List[QuadNode]] = {} - self.__dirty_objects: Set[Element_Value] = set() - self.Build() - - # Add function. - def AddObject(self, obj: Element_Value, push_update: bool = False) -> None: - self.__objects.append(obj) - if push_update: - self.AddSingleToTree(obj) - else: - self.__dirty_objects.add(obj) - - def AddObjects(self, objs: List[Element_Value]) -> None: - self.__objects.extend(objs) - self.__dirty_objects.update(objs) - - def RemoveObject(self, obj: Element_Value) -> None: - self.__dirty_objects.difference_update([obj]) - self.__objects.remove(obj) - self.RemoveSingleFromTree(obj) - - def RemoveSingleFromTree(self, obj: Element_Value) -> None: - lookup_entry = self.__lookup_table.get(obj) - - while lookup_entry: - node_owner = lookup_entry.pop() - node_owner.elements.remove(obj) - - def Update(self) -> None: - if len(self.__dirty_objects) >= self.FULL_REBUILD_THRESHOLD: - self.Build() - else: - for obj in self.__dirty_objects: - self.AddSingleToTree(obj, no_remove=True) - self.__dirty_objects.clear() - - def AddSingleToTree(self, - obj: Element_Value, - no_remove: bool = False) -> None: - lookup_entry = self.__lookup_table.get(obj) - if lookup_entry: - return # already there. - if not no_remove: - self.__dirty_objects.difference_update([obj]) - bounds = obj.GetElement().GetBounds() - - assert self.__tree is not None - if bounds[0] < self.__tree.bounds[0] or \ - bounds[1] < self.__tree.bounds[1] or \ - bounds[2] > self.__tree.bounds[2] or \ - bounds[3] > self.__tree.bounds[3]: - # our new object is outside bounds. Need to recalculate. - self.Build(no_clear=True) - else: - # limited recalc - self.Build(update=[obj]) # not a full rebuild - - def UpdateBounds(self) -> None: - assert self.__tree is not None - bounds = self.CalculateBounds() - self.__tree.UpdateBounds(bounds) - - def CalculateBounds(self) -> Tuple[int, int, int, int]: - lowest_x = 1000000 - highest_x = 0 - lowest_y = 1000000 - highest_y = 0 - for obj in self.__objects: - bounds = obj.GetElement().GetBounds() - if bounds[0] < lowest_x: - lowest_x = bounds[0] - elif bounds[2] > highest_x: - highest_x = bounds[2] - if bounds[1] < lowest_y: - lowest_y = bounds[1] - elif bounds[3] > highest_y: - highest_y = bounds[3] - return (lowest_x, lowest_y, highest_x, highest_y) - - # places objects in tree buckets - # pattern for indices used - # 0 1 - # 2 3 - # if an update list is specified, then only objects in there are updated - def Build(self, - update: Optional[List[Element_Value]] = None, - no_clear: bool = False) -> QuadNode: - if update is None: - update = [] - - if update: - assert self.__tree is not None - self.__tree.elements.extend(update) - else: - if not no_clear: - self.__dirty_objects.clear() # cleans all - min_x, min_y, max_x, max_y = self.CalculateBounds() - self.__tree = QuadNode((min_x, min_y, max_x, max_y), - contents=[], - elements=self.__objects[:]) - current_nodes = [self.__tree] - while current_nodes: - # filter the objects down through tree - node = current_nodes.pop(0) - if node.elements and \ - (node.bounds[2] - node.bounds[0]) / 2.0 >= self.WIDTH_LIMIT: - # will only create if needed - node.BisectCreateNodes() - while node.elements: - obj = node.elements.pop() - # accessing variable rather than dict (e.g. obj.pos) saves - # very little time - bounds = obj.GetElement().GetBounds() - midpoint_x = node.contents[0].bounds[2] - midpoint_y = node.contents[0].bounds[3] - if bounds[0] <= midpoint_x: - if bounds[1] <= midpoint_y: - node.contents[0].elements.append(obj) - if bounds[3] > midpoint_y: - node.contents[2].elements.append(obj) - if bounds[2] > midpoint_x: - if bounds[1] <= midpoint_y: - node.contents[1].elements.append(obj) - if bounds[3] > midpoint_y: - node.contents[3].elements.append(obj) - current_nodes.extend(node.contents) - elif node.elements: - # print route - if not update: - # go from leaves to root and build quick index map (path - # to element) - # could probably be combined with generator - for el in node.elements: - table_entry = self.__lookup_table.get(el) - if table_entry is not None: - if node not in table_entry: - table_entry.append(node) - # else: - # print 'overlap' - else: - self.__lookup_table[el] = [node] - else: - for el in node.elements: - if el in update: - table_entry = self.__lookup_table.get(el) - if table_entry is not None: - if node not in table_entry: - table_entry.append(node) - # else: - # print 'overlap' - else: - self.__lookup_table[el] = [node] - # self.__tree.PrintRecursive() - # print self.__lookup_table - return self.__tree - - def GetObjects( - self, - given_bounds: Tuple[int, int, int, int] - ) -> Set[Element_Value]: - s: Set[Element_Value] = set() - # if not built yet - if self.__tree is None: - return s - # recursive queryng of objects inside given_bounds - self.__GetObjects(self.__tree, given_bounds, s) - return s - - def __GetObjects( - self, - node: QuadNode, - given_bounds: Tuple[int, int, int, int], - output_set: Set[Element_Value] - ) -> None: # left, upper, right, lower - if not node.contents: # leaf - output_set.update(node.elements) - else: - for i in (0, 1, 2, 3): - cnode = node.contents[i] - # in bounds - x_intersects = ((given_bounds[0] <= cnode.bounds[2]) and - (given_bounds[2] >= cnode.bounds[0])) - y_intersects = ((given_bounds[1] <= cnode.bounds[3]) and - (given_bounds[3] >= cnode.bounds[1])) - if (x_intersects and y_intersects): - if (given_bounds[0] <= cnode.bounds[0] and - given_bounds[1] <= cnode.bounds[1] and - given_bounds[2] > cnode.bounds[2] and - given_bounds[3] > cnode.bounds[3]): - # inside--take all objects - child_elements = cnode.GetAllChildElements() - output_set.update(child_elements) - else: - self.__GetObjects(cnode, given_bounds, output_set) - - def SetVisibilityTick(self, - given_bounds: Tuple[int, int, int, int], - vis_tick: int) -> None: - if self.__tree: - self.__SetVisibilityTick(self.__tree, given_bounds, vis_tick) - - def __SetVisibilityTick(self, - node: QuadNode, - given_bounds: Tuple[int, int, int, int], - vis_tick: int) -> None: - if not node.contents: # leaf - for x in node.elements: - x.SetVisibilityTick(vis_tick) - else: - for i in (0, 1, 2, 3): - cnode = node.contents[i] - # in bounds - x_intersects = ((given_bounds[0] <= cnode.bounds[2]) and - (given_bounds[2] >= cnode.bounds[0])) - y_intersects = ((given_bounds[1] <= cnode.bounds[3]) and - (given_bounds[3] >= cnode.bounds[1])) - if (x_intersects and y_intersects): - if (given_bounds[0] <= cnode.bounds[0] and - given_bounds[1] <= cnode.bounds[1] and - given_bounds[2] > cnode.bounds[2] and - given_bounds[3] > cnode.bounds[3]): - # Update all objets because the node is completely - # contained within the given bounds - cnode.SetVisibilityTickOnAllChildElements(vis_tick) - else: - self.__SetVisibilityTick(cnode, given_bounds, vis_tick) - - def GetAllObjects(self) -> List[Element_Value]: - return self.__objects - - def RefreshObject(self, obj: Element_Value) -> None: - self.RemoveSingleFromTree(obj) - self.AddSingleToTree(obj) diff --git a/helios/pipeViewer/pipe_view/model/query_set.py b/helios/pipeViewer/pipe_view/model/query_set.py deleted file mode 100644 index d374e9fd57..0000000000 --- a/helios/pipeViewer/pipe_view/model/query_set.py +++ /dev/null @@ -1,598 +0,0 @@ -from __future__ import annotations -import copy -from logging import debug, info -import sys -import time -from typing import Dict, List, Optional, Tuple, cast, TYPE_CHECKING - -from . import content_options as content -from .schedule_element import ScheduleLineElement - -if TYPE_CHECKING: - from .element_value import Element_Value - from .database import TransactionDatabase - from .layout_context import Layout_Context - from .location_manager import LocationType - - TOffDict = Dict[int, Dict[int, List[Element_Value]]] - - -class ContinuedTransaction: - def __init__(self, - interval: Tuple[int, int], - processed_val: str, - last: bool) -> None: - self.interval = interval - self.processed_val = processed_val - self.last = last - - def unwrap(self) -> Tuple[Tuple[int, int], str, bool]: - return self.interval, self.processed_val, self.last - - -# Formerly known as Ordered_Dict -class QuerySet: - # For sorting elements with no clock - __DEFAULT_T_OFF = 0 - - # Get the Ordered Dict initialized. Note: in order to force-populate the - # Ordered Dict upon initialization, both optional parameters must be - # provided - def __init__(self, layout_context: Layout_Context): - self.__layout_context = layout_context - self.__handle = self.__layout_context.dbhandle - - # keeps track of the ranges we've already queried - self.old_hc = 0 - # stores elements that need ranges of data - self.__range_pairs: List[Element_Value] = [] - self.__continued_transactions: Dict[int, ContinuedTransaction] = {} - - # A count of the number of times this Layout Context has requested to - # move to a different HC. NOTE: not an index of the current HC, nor - # number of stabbing queries performed - self.__stab_index = 0 - # This will be a series of nested dictionaries, with t_offsets (in - # HC's) and loc_id's as their respective keys. The inmost values - # will be lists of Element Values - self.__t_off_sorted: TOffDict = {} - - # Adds a pair to the query set and stashes in correct location - # @profile - def AddPair(self, pair: Element_Value) -> None: - e = pair.GetElement() - # Recompute t_off in terms of plain HC's - lmgr = self.__handle.database.location_manager - loc_str = cast(str, e.GetProperty('LocationString')) - variables = self.__layout_context.GetLocationVariables() - loc, _, clock = lmgr.getLocationInfo(loc_str, variables) - t_off_property = cast(int, e.GetProperty('t_offset', - period=pair.GetClockPeriod())) - - # Warn about invalid locations for content types which DO require - # transactions - if loc == lmgr.INVALID_LOCATION_ID and \ - e.GetProperty('Content') not in content.NO_TRANSACTIONS_REQUIRED: - print(f'Warning: No collected location matching "{loc_str}" ' - f'(using variables:{variables})', - file=sys.stderr) - - if clock == lmgr.NO_CLOCK: - # Makes the assumption that there will always be something else at - # t_offset of 0. If not, then this could stand to be optimized - t_off = self.__DEFAULT_T_OFF - period = -1 - else: - period = self.__handle.database.clock_manager.getClockDomain(clock).tick_period # noqa: E501 - - t_off = period * t_off_property - pair.SetClockPeriod(period) - - if e.GetQueryFrame(period): - self.__range_pairs.append(pair) - else: - if self.GetID(pair) == -1 and \ - e.GetProperty('Content') not in content.NO_DATABASE_REQUIRED: - pair.SetMissingLocation() - else: - pair.SetVal('') - - if t_off in self.__t_off_sorted: - self.__t_off_sorted[t_off] = self.__AddAtLoc( - pair, - self.__t_off_sorted[t_off] - ) - else: - self.__t_off_sorted[t_off] = self.__AddAtLoc(pair) - - # Update this pair to indicate that it was added with this this t_off - # and location - # This will be recalled when deleting this pair - pair.SetLocationAndTimingInformation(t_off_property, - lmgr.getLocationString(loc)) - - # Helper method to AddPair() - # @profile - def __AddAtLoc( - self, - pair: Element_Value, - sub_dict: Optional[Dict[int, List[Element_Value]]] = None - ) -> Dict[int, List[Element_Value]]: - if sub_dict: - if self.GetID(pair) in sub_dict: - if pair not in sub_dict[self.GetID(pair)]: - sub_dict[self.GetID(pair)].append(pair) - else: - sub_dict[self.GetID(pair)] = [pair] - return sub_dict - else: - return {self.GetID(pair): [pair]} - - # Used for re-sorting an Element's location within t_off_sorted{}, - # before the Element's Properties have actually changed - def __ForceAddSingleQueryPair(self, - pair: Element_Value, - t_off_in: int, - id: int) -> None: - e = pair.GetElement() - - # Recompute t_off in terms of plain HC's - lmgr = self.__handle.database.location_manager - loc_str = cast(str, e.GetProperty('LocationString')) - loc, _, clock = lmgr.getLocationInfo( - loc_str, - self.__layout_context.GetLocationVariables() - ) - - if clock == lmgr.NO_CLOCK: - # Makes the assumption that there will always be something else at - # t_offset of 0. If not, then this could stand to be optimized - t_off = self.__DEFAULT_T_OFF - else: - period = self.__handle.database.clock_manager.getClockDomain(clock).tick_period # noqa: E501 - t_off = period * t_off_in - pair.SetClockPeriod(period) - - if id == -1 and \ - e.GetProperty('Content') not in content.NO_DATABASE_REQUIRED: - pair.SetMissingLocation() - else: - pair.SetVal('') - - if t_off in self.__t_off_sorted: - self.__t_off_sorted[t_off] = self.__ForceAddAtLoc( - pair, - id, - self.__t_off_sorted[t_off] - ) - else: - self.__t_off_sorted[t_off] = self.__ForceAddAtLoc(pair, id) - - # Update this pair to indicate that it was added with this this t_off - # and location - # This will be recalled when deleting this pair - pair.SetLocationAndTimingInformation(t_off_in, - lmgr.getLocationString(loc)) - - # Helper method to __ForceAddSingleQueryPair() - def __ForceAddAtLoc( - self, - pair: Element_Value, - id: int, - sub_dict: Optional[Dict[int, List[Element_Value]]] = None - ) -> Dict[int, List[Element_Value]]: - if sub_dict: - if id in sub_dict: - if pair not in sub_dict[id]: - sub_dict[id].append(pair) - else: - sub_dict[id] = [pair] - return sub_dict - else: - return {id: [pair]} - - # Removes the Element-Value associated with the provided Element from - # both draw_order and the t_off_sorted, without leaving lose ends. - # @param pair Element_Value pair. The element in this pair must not have - # had its location or t_offset changed since it was added, otherwise it - # will not be found in the expecected __t_offset_sorted bucket. - def DeletePair(self, pair: Element_Value) -> None: - # In the case of Resorting an Element in t_off_sorted, draw order - # delete range pair_entry if we have one - e = pair.GetElement() - - # Get the properties related to rendering/sorting at the time this pair - # was added - # get the fully-resolved (no variables) location string - prev_locstr = pair.GetDisplayLocationString() - prev_t_off = pair.GetDisplayTOffset() - - if e.GetQueryFrame(pair.GetClockPeriod()): - for r_pair in self.__range_pairs: - if r_pair == pair: - self.__range_pairs.remove(pair) - break - else: - # Recompute t_off in terms of plain HC's - lmgr = self.__handle.database.location_manager - if prev_locstr is not None: - loc, _, clock = lmgr.getLocationInfo(prev_locstr, {}) - else: - loc = lmgr.INVALID_LOCATION_ID - clock = lmgr.NO_CLOCK - - if clock == lmgr.NO_CLOCK: - # Makes the assumption that there will always be something else - # at t_offset of 0. If not, then this could stand to be - # optimized - t_off = self.__DEFAULT_T_OFF - else: - assert prev_t_off is not None - t_off = self.__handle.database.clock_manager.getClockDomain(clock).tick_period * prev_t_off # noqa: E501 - - # Note that we could ignore missing t_offs here, but then we might - # have stale links in another t_off bucket. This guarantees that - # the proper pair was removed by requiring it to be in the expected - # bucket - temp = self.__t_off_sorted[t_off].get(loc) - if not temp: - return - for p in temp: - if p == e: - temp.remove(p) - if len(self.__t_off_sorted[t_off][loc]) == 0: - del self.__t_off_sorted[t_off][loc] - if len(self.__t_off_sorted[t_off]) == 0: - del self.__t_off_sorted[t_off] - - def CheckLocationVariablesChanged(self) -> bool: - loc_vars_status = self.__layout_context.GetLocationVariablesChanged() - if loc_vars_status: - self.__layout_context.AckLocationVariablesChanged() - return loc_vars_status - - def __GetLocationInfo(self, pair: Element_Value) -> LocationType: - el = pair.GetElement() - loc_str = cast(str, el.GetProperty('LocationString')) - if el.LocationHasVars(): - return self.__handle.database.location_manager.getLocationInfo( - loc_str, - self.__layout_context.GetLocationVariables(), - self.CheckLocationVariablesChanged() - ) - - return self.__handle.database.location_manager.getLocationInfoNoVars( - loc_str - ) - - # Returns the internal ID which maps to the given Element's Location - # String, per the Location Manager - def GetID(self, pair: Element_Value) -> int: - return self.__GetLocationInfo(pair)[0] - - # Returns the clock ID which maps to the given' Element's location - # string, per the Location Manager - def GetClock(self, pair: Element_Value) -> int: - return self.__GetLocationInfo(pair)[2] - - # When an element has it's LocationString (therefore LocationID) or - # it's t_offset changed, it needs to be resorted in the - # dictionary. This method is called, and executes, BEFORE the new - # property is assigned to the Element - def ReSort(self, pair: Element_Value, t_off: int, id: int) -> None: - self.DeletePair(pair) - self.__ForceAddSingleQueryPair(pair, t_off, id) - - # Update the val of an Element Value when the Element's 'Content' - # property is changed - def ReValue(self, pair: Element_Value) -> None: - e = pair.GetElement() - if e.GetQueryFrame(pair.GetClockPeriod()): - pair.ClearTimedValues() - self.__layout_context.GoToHC(self.__layout_context.hc) - else: - if e.GetProperty('Content') in content.NO_TRANSACTIONS_REQUIRED: - # Recompute t_off in terms of plain HC's - clock = self.GetClock(pair) - t_off = cast(int, e.GetProperty('t_offset')) - if clock == self.__handle.database.location_manager.NO_CLOCK: - # Makes the assumption that there will always be something - # else at t_offset of 0. If not, then this could stand to - # be optimized - t_off = self.__DEFAULT_T_OFF - else: - t_off = self.__handle.database.clock_manager.getClockDomain(clock).tick_period * t_off # noqa: E501 - temp = self.__t_off_sorted[t_off][self.GetID(pair)] - for pair_tmp in temp: - if pair_tmp == e: - pair.SetVal( - content.ProcessContent( - cast(str, e.GetProperty('Content')), - None, - e, - self.__handle, - self.__layout_context.hc, - self.__layout_context.GetLocationVariables() - ), - self.__stab_index, - ) - return - else: - self.__layout_context.GoToHC(self.__layout_context.hc) - - # @profile - def Update(self) -> None: - ''' - This is where all Element Values get re-synchronized to the current hc - not sure if everything here is final, or best implemented - ''' - self.__stab_index = self.__stab_index + 1 - - # Cached variable lookups - no_trans = content.NO_TRANSACTIONS_REQUIRED - stab_index = self.__stab_index - handle = self.__handle - hc = self.__layout_context.hc - loc_vars = self.__layout_context.GetLocationVariables() - - ordered_ticks = [hc + toff for toff in self.__t_off_sorted] - # Clear all continued transactions so that we don't accidentally draw - # garbage - self.__continued_transactions.clear() - # add intermediate values to make sure Line-type elements have what - # they need - bottom_of_pair = 100000000000000000 - top_of_pair = -100000000000 - for pair in self.__range_pairs: - e = pair.GetElement() - assert isinstance(e, ScheduleLineElement) - # Always set time because it is used for drawing the schedule group - e.SetTime(hc) - period = pair.GetClockPeriod() - if period == -1: - # unset/invalid - continue - qframe = e.GetQueryFrame(period) - curr_time = qframe[0] + hc - end_time = qframe[1] + hc - curr_time = curr_time - curr_time % period - end_time = end_time - end_time % period - while curr_time <= end_time: - timed_val = pair.GetTimedVal(curr_time) - if not timed_val or not timed_val[0]: - ordered_ticks.append(curr_time) - if curr_time > top_of_pair: - top_of_pair = curr_time - if curr_time < bottom_of_pair: - bottom_of_pair = curr_time - - curr_time += period - if len(ordered_ticks) == 0: - return # Nothing to update - - ordered_ticks = sorted(set(ordered_ticks)) - - next_tick_idx = [0] - total_callbacks = [0] - total_useful_callbacks = [0] - total_updates = [0] - - # @profile - def callback(t: int, tapi: TransactionDatabase) -> None: - total_callbacks[0] += 1 - next_tick = next_tick_idx[0] - if len(ordered_ticks) == 0: - return - next_t = ordered_ticks[next_tick] - - if t < next_t: - # Ignore this t because there is no entry in ordered_ticks - return - next_tick_idx[0] += 1 - total_useful_callbacks[0] += 1 - - # get data for range_pairs - updated = 0 - GetID = self.GetID - for range_pair_idx, range_pair in enumerate(self.__range_pairs): - period = range_pair.GetClockPeriod() - if period == -1: - # unset/invalid - continue - e = range_pair.GetElement() - frame = e.GetQueryFrame(period) - assert frame is not None - tick_start = frame[0] + self.__layout_context.hc - tick_end = frame[1] + self.__layout_context.hc - if tick_start <= t <= tick_end: - timed_val = range_pair.GetTimedVal(t) - if not timed_val or not timed_val[0]: - updated += 1 - loc_id = GetID(range_pair) - content_type = cast(str, e.GetProperty('Content')) - - # Update element content based on transaction - # If there is no data for this tick, this will return - # None - trans_proxy = \ - self.__handle.api.getTransactionProxy(loc_id) - - if trans_proxy is not None and trans_proxy.isValid(): - if range_pair_idx in self.__continued_transactions: - old_interval, _, last = \ - self.__continued_transactions[range_pair_idx].unwrap() # noqa: E501 - if last and t >= old_interval[1]: - del self.__continued_transactions[range_pair_idx] # noqa: E501 - if range_pair_idx in self.__continued_transactions: - old_interval, old_processed_val, last = \ - self.__continued_transactions[range_pair_idx].unwrap() # noqa: E501 - old_left = old_interval[0] - new_interval = (old_left, - trans_proxy.getRight()) - # Fix for ARGOS-158/ARGOS-164 - # There's a corner case where a heartbeat - # occurs in the middle of a clock period. We - # would ordinarily skip over it, and - # consequently miss the last part of a - # continued transaction. If a continued - # transaction ends before the next clock period - # begins, we add it to the ordered_ticks list - # so that we can catch the next part of it. - if new_interval[1] < new_interval[0] + period: - ordered_ticks.insert(next_tick_idx[0], - new_interval[1]) - self.__range_pairs[range_pair_idx].SetTimedVal( - old_left, - (old_processed_val, new_interval) - ) - if not trans_proxy.isContinued(): - self.__continued_transactions[range_pair_idx].interval = new_interval # noqa: E501 - self.__continued_transactions[range_pair_idx].last = True # noqa: E501 - - else: - processed_val = content.ProcessContent( - content_type, - trans_proxy, - e, - handle, - hc, - loc_vars - ) - interval = (trans_proxy.getLeft(), - trans_proxy.getRight()) - if trans_proxy.isContinued(): - self.__continued_transactions[range_pair_idx] = ContinuedTransaction(interval, copy.copy(processed_val), False) # noqa: E501 - # Fix for ARGOS-158/ARGOS-164 - # There's a corner case where a heartbeat - # occurs in the middle of a clock period. - # We would ordinarily skip over it, and - # consequently miss the last part of a - # continued transaction. If a continued - # transaction ends before the next clock - # period begins, we add it to the - # ordered_ticks list so that we can catch - # the next part of it. - if interval[1] < interval[0] + period: - ordered_ticks.insert(next_tick_idx[0], - interval[1]) - else: - range_pair.SetTimedVal(interval[0], - (processed_val, - interval)) - if trans_proxy.getLeft() != t: - if t % period == 0: - original_start = \ - trans_proxy.getLeft() - range_pair.SetTimedVal( - t, - (original_start, (t, t)) - ) # placeholder - if not range_pair.GetTimedVal(original_start): # noqa: E501 - info('Unable to make full query.') # noqa: E501 - - else: - if t % period == 0: - range_pair.SetTimedVal( - t, - (None, (t, t)) - ) # placeholder - - # Query at this time and update all elements for which a - # transaction exists. - if t - hc in self.__t_off_sorted: - for loc_id, els in self.__t_off_sorted[t - hc].items(): - for pair in els: - e = pair.GetElement() - - content_type = cast(str, e.GetProperty('Content')) - if content_type in no_trans: - # Update this element, which is not dependent on a - # transaction - pair.SetVal(content.ProcessContent(content_type, - None, - e, - handle, - hc, - loc_vars), - stab_index) - else: - # Update element content based on transaction - # If there is no data for this tick, this will - # return None - trans_proxy = \ - self.__handle.api.getTransactionProxy(loc_id) - if loc_id == -1: - pair.SetMissingLocation() - elif trans_proxy is not None and trans_proxy.isValid(): # noqa: E501 - pair.SetVal( - content.ProcessContent(content_type, - trans_proxy, - e, - handle, - hc, - loc_vars), - stab_index - ) - else: - # There is no transaction here. It might be a - # fake query response or a genuine empty - # transaction. - # If there was previously no transaction at - # this location, assume this is still the case. - # If an element changes locations and then - # points to a location that is valid but has no - # transaction, it is the responsibility of - # AddElement-related methods to clear the 'no - # location' value so that it doesn't persist - if pair.GetVal() is not content.OverrideState('no loc'): # noqa: E501 - pair.SetVal( - content.OverrideState('no trans') - ) - - updated += 1 - total_updates[0] += updated - - debug('Querying from %s to %s', ordered_ticks[0], ordered_ticks[-1]) - t_start = time.monotonic() - try: - self.__handle.query(ordered_ticks[0], - ordered_ticks[-1], - callback, - True) - debug("Done with db query") - except Exception as ex: - debug('Exception while querying!: %s', ex) - raise - finally: - debug( - '%ss: Query+Update for %s elements. %s callbacks (%s useful)', - time.monotonic() - t_start, - total_updates[0], - total_callbacks[0], - total_useful_callbacks[0] - ) - debug(' %s', self.__handle.api) - node_states = self.__handle.api.getNodeStates().split('\n') - for ns in node_states: - debug(' %s', ns) - - debug('Done') - - # For debug purposes - def __repr__(self) -> str: - return self.__str__() - - def __str__(self) -> str: - return '' - - def GetElementDump(self) -> str: - res = '' - for t_off in self.__t_off_sorted: - res += str(t_off) + '\t' - for loc in self.__t_off_sorted[t_off]: - res += str(loc) + '\t' - for e in self.__t_off_sorted[t_off][loc]: - res += repr(e) + ', ' - res += '\n\t' - res += '\n' - return res diff --git a/helios/pipeViewer/pipe_view/model/rpc_element.py b/helios/pipeViewer/pipe_view/model/rpc_element.py deleted file mode 100644 index 9f34a14efa..0000000000 --- a/helios/pipeViewer/pipe_view/model/rpc_element.py +++ /dev/null @@ -1,217 +0,0 @@ -from __future__ import annotations -from .element import Element -from . import element_propsvalid as valid - -import wx -import math -import re -import sys -from typing import Any, Callable, List, Optional, Tuple, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from .element_value import Element_Value - from .extension_manager import ExtensionManager - from ..gui.layout_canvas import Layout_Canvas - from .element import PropertyValue, ValidatedPropertyDict - - -# An element that acts as a graph node -class RPCElement(Element): - _RPC_PROPERTIES: ValidatedPropertyDict = { - 'id': ('', valid.validateString), - # See content_options.py - 'Content': ('annotation', valid.validateContent), - 'auto_color_basis': ('', valid.validateString), - # needs better validator eventually - 'color_basis_type': ('string_key', valid.validateString), - 'meta_properties': ([], valid.validateList), - 'annotation_basis': ('', valid.validateString), - # needs better validator eventually - 'anno_basis_type': ('meta_property', valid.validateString), - } - - __CONTENT_OPTIONS = [ - 'annotation', - 'auto_color_annotation', - 'auto_color_anno_notext', - 'auto_color_anno_nomunge' - ] - - _ALL_PROPERTIES = Element._ALL_PROPERTIES.copy() - - # The RPCElement class shouldn't have a caption field, so we remove it if - # it exists - if 'caption' in _ALL_PROPERTIES: - _ALL_PROPERTIES.pop('caption') - _ALL_PROPERTIES.update(_RPC_PROPERTIES) - - # Name of the property to use as a metadata key - _METADATA_KEY_PROPERTY = 'id' - - # Additional metadata properties that should be associated with this - # element type - _AUX_METADATA_PROPERTIES = list(Element._AUX_METADATA_PROPERTIES) - - ANNO_BASIS_TYPES = ['meta_property', 'python_exp', 'python_func'] - - __EXPR_NAMESPACE = {'re': re, 'math': math} - - @staticmethod - def GetType() -> str: - return 'rpc' - - @staticmethod - def IsDrawable() -> bool: - return True - - @staticmethod - def UsesMetadata() -> bool: - return True - - @staticmethod - def GetDrawRoutine() -> Callable: - return RPCElement.DrawRoutine - - def __init__(self, *args: Any, **kwargs: Any) -> None: - Element.__init__(self, *args, **kwargs) - self.__extensions = ExtensionManager() - - # Update the meta_properties property for this instance - def UpdateMetaProperties(self) -> None: - _AUX_METADATA_PROPERTIES = list(Element._AUX_METADATA_PROPERTIES) - _AUX_METADATA_PROPERTIES.extend( - cast(List[str], self.GetProperty('meta_properties')) - ) - - def SetProperty(self, key: str, val: PropertyValue) -> None: - if key == 'Content' and val not in self.__CONTENT_OPTIONS: - raise ValueError( - f'Content type {val} not allowed for RPC elements' - ) - - Element.SetProperty(self, key, val) - - if key == 'meta_properties': - self.UpdateMetaProperties() - - # Return a listing of the valid options for the 'Content' property - def GetContentOptions(self) -> List[str]: - return self.__CONTENT_OPTIONS - - # Return the annotation for this element - def GetAnnotation(self, pair: Element_Value) -> Optional[str]: - meta_entry = pair.GetMetaEntries() - - # Get the annotation basis and basis type - anno_basis_type = self.GetProperty('anno_basis_type') - annotation_basis = cast(str, self.GetProperty('annotation_basis')) - - # Just use the value of a metadata property - if anno_basis_type == 'meta_property': - if not meta_entry: - return None - return meta_entry.get(annotation_basis) - - # Use the result of a Python expression - elif anno_basis_type == 'python_exp': - try: - return eval(annotation_basis, - {'meta_properties': meta_entry}, - self.__EXPR_NAMESPACE) - - except Exception: - print(f'Error: expression "{annotation_basis}" raised ' - f'exception on input "{meta_entry}":') - print(sys.exc_info()) - return None - - # Use the result of a Python function - elif anno_basis_type == 'python_func': - func = self.__extensions.GetFunction(annotation_basis) - if func: - try: - return func(meta_entry) - except Exception: - print(f'Error: function "{annotation_basis}" raised ' - f'exception on input "{meta_entry}":') - print(sys.exc_info()) - return None - else: - print( - f'Error: function "{annotation_basis}" can not be loaded.' - ) - return None - - # Invalid basis type - else: - return None - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int) -> None: - (c_x, c_y) = cast(Tuple[int, int], self.GetProperty('position')) - (c_w, c_h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - xoff, yoff = canvas.GetRenderOffsets() - (c_x, c_y) = (c_x - xoff, c_y - yoff) - - auto_color = (cast(str, self.GetProperty('color_basis_type')), - cast(str, self.GetProperty('auto_color_basis'))) - - annotation = self.GetAnnotation(pair) - - # annotation == None => Something went wrong in generating the - # annotation, so we set it to ! - if not annotation: - annotation = '!' - - # Set the element value to the generated annotation - pair.SetVal(annotation) - - content_type = cast(str, self.GetProperty('Content')) - - # Generate the color for the element - record = canvas.GetTransactionColor(annotation, - content_type, - auto_color[0], - auto_color[1]) - if record is not None: - string_to_display, brush, _, _, _, _ = record - else: - string_to_display, brush, _, _ = \ - canvas.AddColoredTransaction(annotation, - content_type, - auto_color[0], - auto_color[1], - tick, - self) - - dc.SetBrush(brush) - - # Parameters to easily shift the text within a cell. - c_y_adj = 0 - c_x_adj = 1 - - # Draw the element - dc.DrawRectangle(c_x, c_y, c_w, c_h) - if content_type != 'auto_color_anno_notext': - c_str = string_to_display - (c_char_width, c_char_height) = dc.GetTextExtent(c_str) - # Truncate text if possible - if c_char_width != 0: - c_num_chars = int(1 + (c_w / c_char_width)) - c_content_str_len = len(c_str) - if c_num_chars < c_content_str_len: - dc.DrawText(c_str[:c_num_chars], c_x+c_x_adj, c_y+c_y_adj) - - elif c_content_str_len > 0: - dc.DrawText(c_str, c_x+c_x_adj, c_y+c_y_adj) - else: - dc.DrawText(c_str, c_x+c_x_adj, c_y+c_y_adj) - - self.UnsetNeedsRedraw() - - -RPCElement._ALL_PROPERTIES['type'] = (RPCElement.GetType(), - valid.validateString) diff --git a/helios/pipeViewer/pipe_view/model/schedule_element.py b/helios/pipeViewer/pipe_view/model/schedule_element.py deleted file mode 100644 index 469dc870fd..0000000000 --- a/helios/pipeViewer/pipe_view/model/schedule_element.py +++ /dev/null @@ -1,930 +0,0 @@ -from __future__ import annotations -import copy -from typing import (Any, - Callable, - List, - Optional, - Tuple, - Union, - cast, - TYPE_CHECKING) -import wx -from .element import (Element, - FakeElement, - LocationallyKeyedElement, - MultiElement, - PropertyValue, - ValidatedPropertyDict) -from . import element_propsvalid as valid - -if TYPE_CHECKING: - from .element_value import Element_Value, FakeElementValue - -if TYPE_CHECKING: - from .clock_manager import ClockManager - from ..gui.layout_canvas import Layout_Canvas - -# Global module members for commonly used brushes/pens so that they only need -# to be created once -# TODO: Maybe encapsulate these in some kind of singleton class? - -_WHITE_BRUSH: Optional[wx.Brush] = None -_BLACK_PEN: Optional[wx.Pen] = None - - -# Get the white brush -def GetWhiteBrush() -> wx.Brush: - global _WHITE_BRUSH - if _WHITE_BRUSH is None: - _WHITE_BRUSH = wx.Brush((255, 255, 255)) - return _WHITE_BRUSH - - -# Get the black pen -def GetBlackPen() -> wx.Pen: - global _BLACK_PEN - if _BLACK_PEN is None: - _BLACK_PEN = wx.Pen(wx.BLACK, 1) - return _BLACK_PEN - - -# validates schedule draw style. -# Should be in valid, but I don't want circular dependencies. -def decodeScheduleDraw(name: str, raw: str) -> str: - if raw in ScheduleLineElement.DRAW_LOOKUP: - return raw - raise TypeError( - f'Parameter {name} must be a valid schedule line draw style' - ) - - -class ScheduleLineElement(LocationallyKeyedElement): - # draw line depending on Argos global settings - DRAW_DEFAULT = 0 - # minimalistic schedule line drawing. No tick marks. Fastest drawing. - DRAW_CLEAN = 1 - # shows number of local clock cycles taken up by rendering n-1 dots in - # transaction box - DRAW_DOTS = 2 - # draws repeating auto_id number over transaction for number of cycles it - # has no boxes. NEOCLASSICAL - DRAW_FAST_CLASSIC = 3 - # slowest option - # draws boxes the size of local ticks regardless of if there are - # transactions. Also draws repeating text seen in FAST_CLASSIC - DRAW_CLASSIC = 4 - - SHORT_FORMAT_TYPES = ['single_char', 'multi_char'] - - # time is relative to current clock. - _ALL_PROPERTIES = copy.copy(LocationallyKeyedElement._ALL_PROPERTIES) - # time scale in ticks per pixel - _ALL_PROPERTIES.update({ - 'time_scale': (30.0, valid.validateTimeScale), - 'line_style': ('default', decodeScheduleDraw), - 'short_format': ('single_char', valid.validateString) - }) - - DRAW_LOOKUP = { - 'default': DRAW_DEFAULT, - 'clean': DRAW_CLEAN, - 'dots': DRAW_DOTS, - 'fast_classic': DRAW_FAST_CLASSIC, - 'classic': DRAW_CLASSIC - } - - def __init__(self, *args: Any, **kwargs: Any) -> None: - LocationallyKeyedElement.__init__(self, *args, **kwargs) - self.__buffer = None - self.__hc = 0 - self.__line_style = \ - self.DRAW_LOOKUP[cast(str, self.GetProperty('line_style'))] - - @staticmethod - def GetType() -> str: - return 'schedule_line' - - @staticmethod - def GetElementProperties() -> ValidatedPropertyDict: - return ScheduleLineElement._ALL_PROPERTIES - - @staticmethod - def GetReadOnlyProperties() -> List[str]: - # we use pixel_offset instead - return ['t_offset', 'time_scale'] - - @staticmethod - def GetDrawRoutine() -> Callable: - return ScheduleLineElement.DrawRoutine - - # override to add inheritance stuff - # period is local period for line element - # only used when there is a parent schedule giving a t_offset - def GetProperty(self, - key: str, - period: Optional[int] = 1) -> PropertyValue: - # lock horizontal scale to container - if key[0].isupper(): - return self._properties[key] - - parent = self._parent - if key == 'dimensions': - if parent: - x_dim, _ = cast(Tuple[int, int], parent.GetProperty(key)) - _, y_dim = cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, - key)) - return x_dim, y_dim - elif key == 'position': - if parent: - x_pos, _ = cast(Tuple[int, int], parent.GetProperty(key)) - _, y_pos = cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, - key)) - return x_pos, y_pos - elif key == 'time_scale': - if parent: - return parent.GetProperty(key) - else: - prop = cast(float, self._properties[key]) - scale = cast(Union[Tuple[float, float], Tuple[int, int]], - self.GetProperty('scale_factor'))[0] - return prop / scale - elif key == 't_offset': - if parent: - # assume parent is schedule (for now) - assert period is not None - offset = cast(int, parent.GetProperty('pixel_offset')) - time_scale = cast(float, parent.GetProperty('time_scale')) - return -offset * time_scale / period - return LocationallyKeyedElement.GetProperty(self, key) - - def SetProperty(self, key: str, val: PropertyValue) -> None: - LocationallyKeyedElement.SetProperty(self, key, val) - if key == 'line_style': - assert isinstance(val, str) - self.__line_style = self.DRAW_LOOKUP[val] - - def GetXDim(self) -> int: - if self._parent: - return self._parent.GetXDim() - else: - return cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, - 'dimensions'))[0] - - def GetYDim(self) -> int: - return cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, - 'dimensions'))[1] - - def GetXPos(self) -> int: - if self._parent: - return self._parent.GetXPos() - else: - return cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, - 'position'))[0] - - def GetYPos(self) -> int: - return cast(Tuple[int, int], - LocationallyKeyedElement.GetProperty(self, 'position'))[1] - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int, - time_range: Optional[Tuple[int, int]] = None, - render_box: Optional[Tuple[int, int, int, int]] = None, - fixed_offset: Optional[Tuple[int, int]] = None) -> None: - - # -Set up draw style- - line_style = self.__line_style - dc.SetPen(self._pen) - # if default, set values based on canvas settings. - if line_style == self.DRAW_DEFAULT: - line_style = canvas.GetScheduleLineStyle() - # set flags to pass to Cython renderer - renderer_flags = 0 - if line_style == self.DRAW_DOTS: - renderer_flags = 1 - elif line_style == self.DRAW_CLASSIC: - renderer_flags = 2 - - (c_x, c_y) = cast(Tuple[int, int], self.GetProperty('position')) - (c_w, c_h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - xoff, yoff = canvas.GetRenderOffsets() - if fixed_offset is None: - (c_x, c_y) = (c_x - xoff, c_y - yoff) - else: - c_x = c_x - fixed_offset[0] - c_y = c_y - fixed_offset[1] - - period = pair.GetClockPeriod() - t_scale = cast(float, self.GetProperty('time_scale')) - t_offset = cast(int, self.GetProperty('t_offset', period=period)) - - dc.SetBrush(GetWhiteBrush()) - - if render_box: - if render_box[2] == 0 or render_box[3] == 0: - return # we're done - - dc.SetClippingRegion(*[int(r) for r in render_box]) - dc.DrawRectangle(int(render_box[0] - 1), - int(c_y), - int(render_box[2] + 2), - int(c_h)) - else: - dc.SetClippingRegion(int(c_x), int(c_y), int(c_w), int(c_h)) - dc.DrawRectangle(int(c_x), int(c_y), int(c_w), int(c_h)) - - if period == -1: - # we can't render more with an invalid period - dc.DestroyClippingRegion() - return - - # if not manually specified render everything - if not time_range: - frame_range = self.GetQueryFrame(period) - time_range = (frame_range[0] + self.__hc, - frame_range[1] + self.__hc) - - r = pair.GetTimeRange(time_range) - - # unified clip box - if render_box: - clip = render_box[0] - 1, render_box[2] + 2 - else: - clip = c_x - 1, c_w + 2 - - # width of period in pixels - local_period_width = int(period / t_scale) - - # latest full value rendered - latest_solid_value = None - # TODO convert this to an enum instead, take an enum, return an enum - content_type = cast(str, self.GetProperty('Content')) - auto_color = (cast(str, self.GetProperty('color_basis_type')), - cast(str, self.GetProperty('auto_color_basis'))) - - # Draw vertical ticks in background - if line_style in (self.DRAW_CLASSIC, self.DRAW_FAST_CLASSIC): - line_end_y = c_y + c_h - - # Draw verical lines at clock boundaries to delimit cycles. - # These vertical lines can be too close together, so make sure they - # are at least 1px apart or don't draw them - end_x = clip[0] + clip[1] - dc.DrawLine(int(clip[0]), int(c_y), int(end_x), int(c_y)) - - current_pixel = \ - (time_range[0] - time_range[0] % period - self.__hc - t_offset * period) / t_scale # noqa: E501 - width = period / t_scale - - while current_pixel <= end_x: - start = int(current_pixel) - dc.DrawLine(int(c_x + start), - int(c_y), - int(c_x + start), - int(line_end_y)) - current_pixel += width - - # Walk through intervals - for val, interval in r: - time_width = interval[1] - interval[0] - - # handle phantom elements (width=0) - if time_width == 0: - # This is the code that handles drawing out-of-frame - # elements that extend into our draw area (long elements) - # IMPORTANT function, however, slows down rendering. - # optimizing this (if possible) would make things feel faster. - - # we have something we have no info on - if not latest_solid_value and val is not None: - # backtrack - # phantom's value is main object's start HC - assert isinstance(val, int) - if val < interval[0]: - # Element outside of view may not still be in cache, - # especially if they are long. - # This could cause trouble. - # Solution: trace the phantoms in the query stage and - # query what they point to. - # Drawback: performance. - qv = pair.GetTimedVal(val) - if qv: - val, interval = qv - latest_solid_value = (val, interval) - else: - continue - else: - continue - else: - continue - else: - latest_solid_value = (val, interval) - - start_time = interval[0] - end_time = interval[1] - - time_width = end_time - start_time - width = int(time_width / t_scale) - start = \ - int(((start_time - self.__hc) - (t_offset) * period) / t_scale) - rect = (c_x + start, c_y, width + 1, c_h) - - NOT_MISSING_LOC = False - short_format = cast(str, self.GetProperty('short_format')) - canvas.GetRenderer().drawInfoRectangle( - interval[0], - self, - dc, - canvas, - rect, - val, - NOT_MISSING_LOC, - content_type, - auto_color, - clip, - schedule_settings=(local_period_width, renderer_flags), - short_format=short_format - ) - - dc.DestroyClippingRegion() - self.UnsetNeedsRedraw() - - def GetTime(self) -> int: - return self.__hc - - def SetTime(self, hc: int) -> None: - self.__hc = hc - - # Called at query time and indicates what should be updated - def GetQueryFrame(self, period: int) -> Tuple[int, int]: - parent = self._parent - time_scale = cast(float, - parent.GetProperty('time_scale') - if parent is not None - else self.GetProperty('time_scale')) - - # Optimized version for parent case - if parent: - # assume parent is schedule (for now) - offs = -cast(int, parent.GetProperty('pixel_offset')) * time_scale - return (int(offs - period), - int((offs + period) + self.GetXDim() * time_scale)) - else: - # in ticks - offs = cast(int, self.GetProperty('t_offset', period=period)) - return (int((offs - 1) * period), - int((offs + 1) * period + self.GetXDim() * time_scale)) - - # Generates elements with addresses based on the x coordinate. - # passes these (fake) elements to the hover preview, which then - # queries all the needed data fresh. - # accepts point in local coord - def DetectCollision(self, - pt: Union[Tuple[int, int], wx.Point], - pair: Element_Value) -> FakeElementValue: - from .element_value import FakeElementValue - - mx, my = pt - - period = pair.GetClockPeriod() - t_scale = cast(float, self.GetProperty('time_scale')) - offs = cast(int, self.GetProperty('t_offset', period=period)) * period - offs += int(t_scale * pt[0]) - - location = cast(str, self.GetProperty('LocationString')) - t_offset = offs // period - - fake_element = FakeElement() - fake_element.SetProperty('LocationString', location) - fake_element.SetProperty('caption', '') - fake_element.SetProperty('t_offset', t_offset) - fake_element.SetProperty('Content', self.GetProperty('Content')) - fake_element.SetProperty('color_basis_type', - self.GetProperty('color_basis_type')) - fake_element.SetProperty('auto_color_basis', - self.GetProperty('auto_color_basis')) - - # calculate out coordinates of current transaction - pos = cast(Tuple[int, int], self.GetProperty('position')) - fake_x = pt[0] + pos[0] - int((offs % period) / t_scale) - fake_y = pos[1] - fake_w = int(period / t_scale) - fake_h = self.GetYDim() - fake_element.SetProperty('position', (fake_x, fake_y)) - fake_element.SetProperty('dimensions', (fake_w, fake_h)) - - fake_element_val = FakeElementValue(fake_element, - f'{location}@{t_offset}') - fake_element_val.SetClockPeriod(pair.GetClockPeriod()) - return fake_element_val - - -ScheduleLineElement._ALL_PROPERTIES['type'] = (ScheduleLineElement.GetType(), - valid.validateString) - - -# Element shows start times every few cycles. -class ScheduleLineRulerElement(ScheduleLineElement): - _ALL_PROPERTIES = ScheduleLineElement._ALL_PROPERTIES.copy() - - if 'short_format' in _ALL_PROPERTIES: - _ALL_PROPERTIES.pop('short_format') - - # special setting not available to user - DRAW_RULER = 10 - - @staticmethod - def GetType() -> str: - return 'schedule_line_ruler' - - @staticmethod - def GetDrawRoutine() -> Callable: - return ScheduleLineRulerElement.DrawRoutine - - def __init__(self, *args: Any, **kwargs: Any) -> None: - ScheduleLineElement.__init__(self, *args, **kwargs) - self.__step = 5 - - # very brief rendering function generating time-line - # I tried just showing transactions and their start time, - # however it was buggy and added too many special cases and hacks - # this is quick and simple in comparison - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int, - time_range: Optional[Tuple[int, int]] = None, - render_box: Optional[Tuple[int, int, int, int]] = None, - fixed_offset: Optional[Tuple[int, int]] = None) -> None: - - # render box is disregarded so shift is - # over-written when acting as a child of a container - dc.SetPen(self._pen) - - (c_x, c_y) = cast(Tuple[int, int], self.GetProperty('position')) - (c_w, c_h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - xoff, yoff = canvas.GetRenderOffsets() - if not fixed_offset: - (c_x, c_y) = (c_x - xoff, c_y - yoff) - else: - c_x = c_x - fixed_offset[0] - c_y = c_y - fixed_offset[1] - - t_scale = cast(float, self.GetProperty('time_scale')) - t_offset = cast(int, self.GetProperty('t_offset', - period=pair.GetClockPeriod())) - - period = pair.GetClockPeriod() - - dc.SetBrush(GetWhiteBrush()) - - dc.SetClippingRegion(int(c_x), int(c_y), int(c_w), int(c_h)) - dc.DrawRectangle(int(c_x), int(c_y), int(c_w), int(c_h)) - - if period == -1: - dc.DestroyClippingRegion() - return - - clip = c_x - 1, c_w + 2 - - # figure out step-size based on scale: - step = self.__step * (int((t_scale + 150) / 100)) - # Set practical limit. No zero-sized steps. - if step == 0: - step = 1 - - full_interval = period * step - current_time = t_offset * period - current_time -= current_time % (period * step) - width = int(full_interval / t_scale) - end_time = c_w * t_scale + full_interval + current_time - while current_time < end_time: - start = int((current_time - t_offset * period) / t_scale) - rect = (c_x + start, c_y, width + 1, c_h) - val = 'C=1 %i' % (current_time / period) - NOT_MISSING_LOC = False - canvas.GetRenderer().drawInfoRectangle( - tick, - self, - dc, - canvas, - rect, - val, - NOT_MISSING_LOC, - 'caption', - ('', ''), - clip, - schedule_settings=(full_interval, self.DRAW_RULER) - ) - current_time += full_interval - - dc.DestroyClippingRegion() - self.UnsetNeedsRedraw() - -# Container class for Schedule Lines. -# Gives schedule elements the following properties: -# Clock, Width, Time Offset -# Also Controls drawing of schedule lines. - - -class ScheduleElement(MultiElement): - _ALL_PROPERTIES = MultiElement._ALL_PROPERTIES.copy() - _ALL_PROPERTIES.update({'time_scale': (30.0, valid.validateTimeScale)}) - _ALL_PROPERTIES.update({'pixel_offset': (0, valid.validateOffset)}) - _ALL_PROPERTIES.update({'pixels_per_cycle': (0, valid.validateTimeScale)}) - _ALL_PROPERTIES.update({'cycle_offset': (0, valid.validateOffset)}) - _ALL_PROPERTIES.update({'clock': ('', valid.validateString)}) - - def __init__(self, *args: Any, **kwargs: Any) -> None: - MultiElement.__init__(self, *args, **kwargs) - self.__buffer: Optional[wx.Bitmap] = None - self.__temp_buffer: Optional[wx.Bitmap] = None - self.__dc = wx.MemoryDC() # store our own DC - self.__temp_dc = wx.MemoryDC() - self.__graphics_dc: Optional[wx.GCDC] = None - - self.__old_dimensions: Optional[Tuple[int, int]] = None - self.__last_hc: Optional[int] = None - self.__old_period = 1 - self.__remainder_dp = 0.0 - # Check that the clock name (if any) is valid. - clock_name = cast(str, self.GetProperty('clock')) - if clock_name: - if self.__FindClockOrWarn(clock_name) is not None: - # If it is, go ahead and refresh the scale and offset - # parameters - self.__RefreshProperty('pixels_per_cycle') - self.__RefreshProperty('cycle_offset') - - @staticmethod - def GetType() -> str: - return 'schedule' - - @staticmethod - def IsDrawable() -> bool: - return True - - @staticmethod - def IsSelectable() -> bool: - return False - - @staticmethod - def GetElementProperties() -> ValidatedPropertyDict: - return ScheduleElement._ALL_PROPERTIES - - @staticmethod - def GetReadOnlyProperties() -> List[str]: - # we use pixel_offset instead - return ['t_offset', 'time_scale', 'pixel_offset'] - - @staticmethod - def GetDrawRoutine() -> Callable: - return ScheduleElement.DrawRoutine - - def __FindClockOrWarn( - self, - clock_name: str - ) -> Optional[ClockManager.ClockDomain]: - clock_domain = None - assert self._layout is not None - if self._layout.HasContext(): - for context in self._layout.lay_cons: - clock_manager = context.dbhandle.database.clock_manager - if clock_manager.doesClockNameExist(clock_name): - clock_domain = \ - clock_manager.getClockDomainByName(clock_name) - break - if clock_domain is None: - print(f'Warning: Could not find clock named {clock_name}. ' - 'Falling back to time_scale value.') - return clock_domain - - def __RefreshProperty(self, key: str) -> None: - self.SetProperty(key, self.GetProperty(key)) - - def GetProperty(self, - key: str, - period: Optional[int] = None) -> PropertyValue: - if key == 'pixel_offset' or key == 'pixels_per_cycle': - scale_factor = \ - cast(Union[Tuple[float, float], Tuple[int, int]], - MultiElement.GetProperty(self, 'scale_factor'))[0] - return round( - cast(int, MultiElement.GetProperty(self, key)) * scale_factor - ) - elif key == 'time_scale': - scale_factor = \ - cast(Union[Tuple[float, float], Tuple[int, int]], - MultiElement.GetProperty(self, 'scale_factor'))[0] - return cast(float, - MultiElement.GetProperty(self, key)) / scale_factor - return MultiElement.GetProperty(self, key) - - def SetProperty(self, key: str, val: PropertyValue) -> None: - MultiElement.SetProperty(self, key, val) - if key == 'clock': - if val: - self.__RefreshProperty('pixels_per_cycle') - self.__RefreshProperty('cycle_offset') - elif key == 'pixels_per_cycle': - # Need to add the ability for an element to access the clock - # manager... - clock_name = cast(str, self.GetProperty('clock')) - clock_domain = self.__FindClockOrWarn(clock_name) - val = cast(float, val) - if clock_domain is not None and val != 0: - time_scale = clock_domain.tick_period / val - self.SetProperty('time_scale', time_scale) - self.__RefreshProperty('cycle_offset') - elif key == 'cycle_offset': - # Need to add the ability for an element to access the clock - # manager... - clock_name = cast(str, self.GetProperty('clock')) - clock_domain = self.__FindClockOrWarn(clock_name) - if clock_domain is not None: - val = cast(float, val) - time_scale = cast(float, self.GetProperty('time_scale')) - pixel_offset = int(clock_domain.tick_period * val / time_scale) - self.SetProperty('pixel_offset', pixel_offset) - - def __ReinitializeBuffer(self, - canvas: Layout_Canvas, - width: int, - height: int) -> None: - self.__buffer = wx.Bitmap(canvas.MAX_ZOOM * width, - canvas.MAX_ZOOM * height) - self.__temp_buffer = wx.Bitmap(canvas.MAX_ZOOM * width, - canvas.MAX_ZOOM * height) - self.__SwapBuffers(canvas) - - def __SwapBuffers(self, canvas: Layout_Canvas) -> None: - assert self.__buffer is not None - assert self.__temp_buffer is not None - self.__buffer, self.__temp_buffer = self.__temp_buffer, self.__buffer - self.__dc.SelectObject(self.__buffer) - self.__temp_dc.SelectObject(self.__temp_buffer) - self.__graphics_dc = wx.GCDC(self.__dc) - self.__graphics_dc.SetFont(self.__dc.GetFont()) - self.__graphics_dc.SetLogicalScale(canvas.MAX_ZOOM, canvas.MAX_ZOOM) - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int) -> None: - children = self.GetChildren() - if not children: - # our work is done here - return - - (c_x, c_y) = cast(Tuple[int, int], self.GetProperty('position')) - (c_w, c_h) = cast(Tuple[int, int], self.GetProperty('dimensions')) - absolute_x = c_x - xoff, yoff = canvas.GetRenderOffsets() - (c_x, c_y) = (c_x - xoff, c_y - yoff) - - window_max = dc.GetSize() - if window_max[0] < c_x or window_max[1] < c_y: - return # all out of bounds - - # some kind of huge number - highest_y = 100000000000 - lowest_y = 0 - - pairs = [] - largest_period = 0 - - # @todo This is imporper. HasChanged refers to layout state, not - # drawing state. Reading HasChanged to determine if an update should - # be done is not appropriate. Writing MarkAsUnchanged() will break - # other frames using the same layout AND break "modified layout" - # functionality - - full_update = self.HasChanged() or self.NeedsRedraw() - # --Pre-Render Cycle-- - # * Poll children for changes - # * Populate our pairs list (could be moved outside render function) - # * Find largest period - # * Find bounds of schedule lines - for child in children: - # poll for changes - if child.HasChanged() or child.NeedsRedraw(): - full_update = True - child._MarkAsUnchanged() - child.UnsetNeedsRedraw() - - ycpos = child.GetYPos() - lowest_y_tmp = ycpos + child.GetYDim() - - # set highest and lowest y values for element bounds - if lowest_y_tmp > lowest_y: - lowest_y = lowest_y_tmp - if ycpos < highest_y: - highest_y = ycpos - - # collect pairs from elements - assert isinstance(child, LocationallyKeyedElement) - new_pair = canvas.context.GetElementPair(child) - assert new_pair is not None - pair = new_pair - pairs.append(pair) - possible_period = pair.GetClockPeriod() - if possible_period > largest_period: - largest_period = possible_period - first_child = cast(ScheduleLineElement, children[0]) - hc = first_child.GetTime() - # ticks per pixel - t_scale = cast(float, first_child.GetProperty('time_scale')) - tick_offset = cast(int, first_child.GetProperty('t_offset')) - frame_range = (int(tick_offset), - int(tick_offset + self.GetXDim() * t_scale)) - - # Determine update type - if self.__old_period != largest_period: - full_update = True - self.__old_period = largest_period - if self.__last_hc is None: - full_update = True - self.__last_hc = hc - - # Find deltas - d_t = hc - self.__last_hc - d_p = -d_t / t_scale + self.__remainder_dp - i_d_p = int(d_p) - - # convert pixel to time space: - # if d_t is wider than window, do full update - if d_t > (frame_range[1] - frame_range[0]): - start_range = frame_range[0] + hc - end_range = frame_range[1] + hc - full_update = True - else: - # update smaller than window, just fill what is needed - if d_t > 0: - # forward motion. need high range - start_range = frame_range[1] + self.__last_hc - largest_period - end_range = frame_range[1] + hc + largest_period - elif d_t < 0: - # back, low range needed - start_range = frame_range[0] + hc - largest_period - end_range = frame_range[0] + largest_period + self.__last_hc - else: - start_range = hc - end_range = hc - - sched_height = lowest_y - highest_y - # Execute the set update type - if self.__buffer is None: - time_range = None # draw full frame - clip_region = None - self.__dc.SetFont(dc.GetFont()) - self.__ReinitializeBuffer(canvas, c_w, sched_height) - self.__dc.Clear() - elif full_update: - time_range = None - clip_region = None - if self.__old_dimensions != (c_w, sched_height): - self.__dc.SetFont(dc.GetFont()) - # we need to make a new buffer - self.__ReinitializeBuffer(canvas, c_w, sched_height) - self.__dc.Clear() - else: - # make box - box_size = (c_w, sched_height) - - clip_region = None - if i_d_p < 0: - # moving forward (to left) need to fill in forward - clip_region = (c_w + i_d_p, 0, -i_d_p, box_size[1]) - else: - # moving backward (to right), need to fill in back - clip_region = (0, 0, i_d_p, box_size[1]) - - time_range = (start_range, end_range) - if i_d_p < 0: - sub_x = -canvas.MAX_ZOOM * i_d_p - sub_width = self.__buffer.GetWidth() - sub_x - dest_x = 0 - else: - sub_x = 0 - dest_x = canvas.MAX_ZOOM * i_d_p - sub_width = self.__buffer.GetWidth() - dest_x - - assert self.__graphics_dc is not None - self.__graphics_dc.SetLogicalScale(1, 1) - self.__temp_dc.Blit(int(dest_x), - 0, - int(sub_width), - int(self.__buffer.GetHeight()), - self.__dc, int(sub_x), - 0) - self.__SwapBuffers(canvas) - self.__graphics_dc.SetLogicalScale(canvas.MAX_ZOOM, - canvas.MAX_ZOOM) - - assert self.__buffer is not None - - # --Render Loop-- - for child_idx, child in enumerate(children): - child = cast(ScheduleLineElement, child) - assert self.__graphics_dc is not None - child.DrawRoutine(pairs[child_idx], - self.__graphics_dc, - canvas, - tick, - time_range, - clip_region, - fixed_offset=(absolute_x, highest_y)) - - # Calculate the blit destination location, width, and height - update_box = canvas.GetScaledUpdateRegion() - update_top_left = update_box.GetTopLeft() - update_bottom_right = update_box.GetBottomRight() - sched_x = c_x - sched_y = highest_y - yoff - sched_x_end = sched_x + c_w - sched_y_end = sched_y + sched_height - blit_x = max(sched_x, update_top_left[0]) - blit_y = max(sched_y, update_top_left[1]) - blit_x_end = min(update_bottom_right[0], sched_x_end) - blit_y_end = min(update_bottom_right[1], sched_y_end) - blit_width = max(0, blit_x_end - blit_x) - blit_height = max(0, blit_y_end - blit_y) - - # Calculate offsets, width, and height within our buffer - blit_x_offset = canvas.MAX_ZOOM * (blit_x - sched_x) - blit_y_offset = canvas.MAX_ZOOM * (blit_y - sched_y) - blit_src_width = canvas.MAX_ZOOM * blit_width - blit_src_height = canvas.MAX_ZOOM * blit_height - - assert blit_src_width <= self.__buffer.GetWidth() - assert blit_src_height <= self.__buffer.GetHeight() - - dc.StretchBlit(int(blit_x), - int(blit_y), - int(blit_width), - int(blit_height), - self.__dc, - int(blit_x_offset), - int(blit_y_offset), - int(blit_src_width), - int(blit_src_height)) - - # draw vertical line at offset 0 - dc.SetPen(GetBlackPen()) - zero_x_coord = c_x - tick_offset / t_scale - dc.DrawLine(int(zero_x_coord), - int(highest_y - yoff), - int(zero_x_coord), - int(lowest_y - yoff)) - - if i_d_p != 0: - self.__remainder_dp = d_p - i_d_p - # only update if we've progressed the pixels. This cuts down on - # rounding error. - self.__last_hc = hc - self.__old_dimensions = (c_w, sched_height) - - self._MarkAsUnchanged() - self.UnsetNeedsRedraw() - - # Detect collision with children (they are likely not drawn) - # @param pt Point to test - # @return First child which includes pt - def DetectCollision( - self, - pt: Union[Tuple[int, int], wx.Point] - ) -> Optional[Element]: - mx, my = pt - - for child in self.GetChildren(): - x, y = cast(Tuple[int, int], child.GetProperty('position')) - w, h = cast(Tuple[int, int], child.GetProperty('dimensions')) - - if x <= mx <= (x + w) and y <= my <= (y + h): - return child - - return None # No collision detected - - def AddChild(self, child: Element) -> None: - if not isinstance(child, ScheduleLineElement): - raise Exception( - 'Children of ScheduleElement must be ScheduleLineElements or ' - 'descendants.' - ) - - MultiElement.AddChild(self, child) - # grab render control - child.EnableDraw(False) - - -ScheduleElement._ALL_PROPERTIES['type'] = (ScheduleElement.GetType(), - valid.validateString) diff --git a/helios/pipeViewer/pipe_view/model/search_handle.py b/helios/pipeViewer/pipe_view/model/search_handle.py deleted file mode 100644 index 5336465ee7..0000000000 --- a/helios/pipeViewer/pipe_view/model/search_handle.py +++ /dev/null @@ -1,186 +0,0 @@ -from __future__ import annotations -import os -import time -import sys -import subprocess -from logging import error -from typing import Callable, List, Optional, Tuple, TYPE_CHECKING -import shutil - -if TYPE_CHECKING: - from .layout_context import Layout_Context - -__SEARCH_PROGRAM_ENV_VAR_NAME = 'TRANSACTIONSEARCH_PROGRAM' -TRANSACTION_SEARCH_PROGRAM = os.environ.get(__SEARCH_PROGRAM_ENV_VAR_NAME, - shutil.which("transactionsearch")) -print("INFO: looking for ", TRANSACTION_SEARCH_PROGRAM) - -can_search = False -if TRANSACTION_SEARCH_PROGRAM and os.path.isfile(TRANSACTION_SEARCH_PROGRAM): - can_search = True -else: - # keep looking if not explicitly stated - # try to figure out based on transactiondb path - __MODULE_ENV_VAR_NAME = 'TRANSACTIONDB_MODULE_DIR' - env_var = os.environ.get(__MODULE_ENV_VAR_NAME) - if env_var is None: - added_path = os.path.join( - os.path.dirname(__file__), - "../../../../release/helios/pipeViewer/transactionsearch" - ) - added_path = os.path.abspath(added_path) - can_search = True - if not os.path.isdir(added_path): - can_search = False - transaction_module_path = added_path - else: - transaction_module_path = os.environ.get(__MODULE_ENV_VAR_NAME, - os.getcwd()) - - build_folder_name = \ - transaction_module_path.strip(os.path.sep).split(os.path.sep)[-1] - TRANSACTION_SEARCH_PROGRAM = os.path.join(transaction_module_path, - 'transactionsearch') - TRANSACTION_SEARCH_PROGRAM = os.path.normpath(TRANSACTION_SEARCH_PROGRAM) - if os.path.isfile(TRANSACTION_SEARCH_PROGRAM): - can_search = True - -if can_search: - print('transaction search initialized: you will be able to use search.') - print('Using:', TRANSACTION_SEARCH_PROGRAM) -else: - print('transaction search could not be found.' - 'Try setting TRANSACTIONSEARCH_PROGRAM ' - 'to path of your transactionsearch executable', file=sys.stderr) - - -# Gets data for use by search dialog. -class SearchHandle: - - def __init__(self, context: Layout_Context) -> None: - # other access to db. only used for resolving location id's the strings - self.__db = context.dbhandle.database - - # Does search, first argument query_type takes 'string' or 'regex' - # @param progress_callback This is NOT wx.ProgressDialog's update function - # Takes 3 arguments (% complete, result count, result info string) - # Callback returns a 2-tuple: (continue, skip) - # @return Results list. Each entry is a tuple of matches - # (start, end, loc ID, annotation) - # TODO does this really need to launch a subprocess? can't it just make an - # API call to transaction db? - def Search(self, - query_type: str, - query_string: str, - start_tick: Optional[int] = None, - end_tick: Optional[int] = None, - locations: List[int] = [], - progress_callback: Optional[Callable] = None, - invert: bool = False) -> List[Tuple[int, int, int, str]]: - assert isinstance(query_string, str) - if query_type not in ('string', 'regex'): - raise Exception( - f'query type must be string or regex, got {query_type}' - ) - if not can_search: - raise Exception( - 'There were problems with the initialization of search. ' - 'You cannot search.' - ) - results: List[Tuple[int, int, int, str]] = [] - - assert TRANSACTION_SEARCH_PROGRAM is not None - - arglist = [TRANSACTION_SEARCH_PROGRAM, - self.__db.filename, - query_type, - query_string, - "1" if invert else "0"] - - # Start tick arg - if start_tick is not None: - arglist.append(str(start_tick)) - else: - arglist.append(str(0)) - - # End tick arg - if end_tick is not None: - arglist.append(str(end_tick)) - else: - arglist.append(str(-1)) - - # Location filter arg - if locations: - arglist.append(','.join([str(loc) for loc in locations])) - else: - arglist.append('') - - # print 'Search Arglist = ', arglist - - process = subprocess.Popen(arglist, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - process.poll() - if process.returncode is not None and process.returncode != 0: - assert process.stderr is not None - stderr = process.stderr.read().decode("utf-8") - raise IOError(f'Search subprocess failed: {stderr}') - - cont = True - assert process.stdout is not None - while cont: # Loop until process terminates - line = process.stdout.readline() - while line: - line_type = line[0] - line_content = line[1:].strip() - if line_type == b'p'[0]: # process - if progress_callback: - percent = min(99.99, float(line_content) * 100) - num_results = len(results) - cont, skip = progress_callback( - percent, - num_results, - f'{num_results} Potential Matches' - ) - if cont is False: - break # early exit when cont is False - elif line_type == b'r'[0]: # result - annotation_start = line_content.find(b':') - try: - start_and_loc = line_content[0:annotation_start] - time_range, loc_id = start_and_loc.split(b'@') - start, end = time_range.split(b',') - annotation = line_content[annotation_start + 1:] - except Exception: - line_str = line_content.decode('utf-8') - error('Failed to parse search output line "%s"', - line_str) - raise - results.append((int(start), - int(end), - int(loc_id), - annotation.decode('utf-8'))) - # update line - line = process.stdout.readline() - - # Only terminate after the remainder of stdout has been processed - # AND the return code from the previous iteration's poll was set. - # This guarantees that all data was processed - if process.returncode is not None: - break - - time.sleep(0.001) - process.poll() - - # Check for failures after stdin has been exhausted. - # If waiting for process completion above, there is no need to check - # for a 'None' returncode here. - if process.returncode != 0 and process.returncode is not None: - assert process.stderr is not None - stderr = process.stderr.read().decode("utf-8") - print('Search subprocess returned non-zero error code: ' - f'{process.returncode}: {stderr}', - file=sys.stderr) - - return results diff --git a/helios/pipeViewer/pipe_view/model/settings.py b/helios/pipeViewer/pipe_view/model/settings.py deleted file mode 100644 index 6360eeb1cc..0000000000 --- a/helios/pipeViewer/pipe_view/model/settings.py +++ /dev/null @@ -1,119 +0,0 @@ -from __future__ import annotations -import atexit -import json -import os -import sys -from typing import Any, Dict -from ..gui.autocoloring import BrushRepository -from ..gui.font_utils import GetDefaultFontSize, GetDefaultControlFontSize - - -# Stores settings in a JSON file in the user's config directory so that they -# can be persisted across runs -class ArgosSettings: - __DEFAULT_CONFIG = { - 'layout_font_size': GetDefaultFontSize(), - 'hover_font_size': GetDefaultFontSize(), - 'playback_font_size': GetDefaultControlFontSize(), - 'palette': 'default', - 'palette_shuffle': 'default' - } - - __ALLOWED_VALUES = { - 'palette': BrushRepository.get_supported_palettes(), - 'palette_shuffle': BrushRepository.get_supported_shuffle_modes() - } - - def __init__(self) -> None: - self.__dirty = False - - argos_config_dir = 'argos' - # %APPDATA% = Windows user config directory - # $XDG_CONFIG_HOME = Unix-like (and sometimes OS X) user config - # directory - config_dir = os.environ.get('APPDATA') or \ - os.environ.get('XDG_CONFIG_HOME') - - if config_dir is None: - if sys.platform == 'darwin': - # Apple-recommended location for program settings - config_dir = os.path.expanduser('~/Library/Preferences') - argos_config_dir = 'sparcians.argos' - else: - # Default for Unix-like systems that don't have XDG_* variables - # defined - config_dir = os.path.join(os.environ['HOME'], '.config') - - full_config_dir = os.path.join(config_dir, argos_config_dir) - self.__config_json_path = os.path.join(full_config_dir, 'config.json') - - if not os.path.exists(full_config_dir): - os.makedirs(full_config_dir) - - self.__config = ArgosSettings.__DEFAULT_CONFIG.copy() - # Grab a reference to open() so we don't get any errors when at exit - self.__open = open - - if not os.path.exists(self.__config_json_path): - self.__dirty = True - self.save() - else: - with self.__open(self.__config_json_path, 'r') as f: - self.__config.update(json.load(f)) - - for k, v in self.__config.items(): - self.__validate_setting(k, v) - - # Register atexit cleanup function so that we don't try to call save() - # after open() has been deleted - def cleanup() -> None: - self.save() - - atexit.register(cleanup) - - def __del__(self) -> None: - self.save() - - def save(self) -> None: - if self.__dirty: - with self.__open(self.__config_json_path, 'w') as f: - json.dump(self.__config, f) - self.__dirty = False - - def update(self, new_settings: Dict[str, Any]) -> None: - if not isinstance(new_settings, dict): - raise TypeError('update() must be called with a dict argument') - - for k, v in new_settings.items(): - self[k] = v - - def __validate_setting(self, key: str, value: Any) -> None: - if key not in ArgosSettings.__DEFAULT_CONFIG: - raise KeyError(f'{key} is not a recognized settings field') - - allowed_values = ArgosSettings.__ALLOWED_VALUES.get(key) - if allowed_values and value not in allowed_values: - raise ValueError( - f'{value} is not an allowed value for setting {key}. ' - f'Allowed values are: {allowed_values}' - ) - - def __getitem__(self, key: str) -> Any: - return self.__config[key] - - def __getattr__(self, key: str) -> Any: - try: - return self.__getitem__(key) - except KeyError: - raise AttributeError - - def __setitem__(self, key: str, value: Any) -> None: - self.__validate_setting(key, value) - self.__config[key] = value - self.__dirty = True - - def __setattr__(self, key: str, value: Any) -> None: - try: - self.__setitem__(key, value) - except KeyError: - object.__setattr__(self, key, value) diff --git a/helios/pipeViewer/pipe_view/model/speedo_element.py b/helios/pipeViewer/pipe_view/model/speedo_element.py deleted file mode 100644 index 53c243362d..0000000000 --- a/helios/pipeViewer/pipe_view/model/speedo_element.py +++ /dev/null @@ -1,341 +0,0 @@ -from __future__ import annotations -from .widget_element import WidgetElement -from . import element_propsvalid as valid - -import wx -import math - -import wx.lib.agw.speedmeter as SM - -from typing import (Any, - Callable, - List, - Optional, - Tuple, - Union, - TYPE_CHECKING, - cast) - -if TYPE_CHECKING: - from .element import (PropertyValue, - ValidatedPropertyDict) - from .element_value import Element_Value - from ..gui.layout_canvas import Layout_Canvas - - -class SpeedoWidget(wx.Control): - _DEFAULT_TEXT_FORMAT = '{f}' - - def __init__(self, parent: wx.Panel, *args: Any, **kwargs: Any) -> None: - wx.Control.__init__(self, parent, *args, **kwargs) - self.__parent = parent - self.__panel = wx.Panel(self.__parent) - self.__speedo_panel = wx.Panel(self.__panel) - self.__speedo_panel.SetPosition((0, 0)) - self.__speedo = SM.SpeedMeter(self.__speedo_panel, - agwStyle=(SM.SM_ROTATE_TEXT | - SM.SM_DRAW_HAND | - SM.SM_DRAW_MIDDLE_TEXT | - SM.SM_DRAW_SECONDARY_TICKS)) - self.__speed_text_panel = wx.Panel(self.__panel) - self.__speed_text = wx.StaticText(self.__speed_text_panel, - style=wx.ALIGN_CENTER) - self.__speed_text_format = SpeedoWidget._DEFAULT_TEXT_FORMAT - self.__speed_text.Raise() - - def Bind(self, # type: ignore[override] - event: wx.PyEventBinder, - func: Callable) -> None: - wx.Control.Bind(self, event, func) - self.__speedo.Bind(event, func) - self.__speed_text.Bind(event, func) - - def SetPosition(self, pos: Union[Tuple[int, int], wx.Point]) -> None: - self.__panel.SetPosition(pos) - - def SetSize( # type: ignore[override] - self, - size: Union[Tuple[int, int], wx.Size] - ) -> None: - self.__panel.SetSize(size) - self.__speedo_panel.SetSize(size) - self.__speedo.SetSize(size) - - def SetBackgroundColor( - self, - color: Union[Tuple[int, int, int], wx.Colour] - ) -> None: - self.__panel.SetBackgroundColour(color) - self.__speedo_panel.SetBackgroundColour(color) - self.__speed_text_panel.SetBackgroundColour(color) - self.__speed_text.SetBackgroundColour(color) - self.__speedo.SetSpeedBackground(color) - - def Update(self) -> None: - self.__speedo_panel.Fit() - self.__speed_text_panel.Fit() - self.__panel.Fit() - self.__panel.Update() - - def SetValue(self, val: float) -> None: - self.__speed_text.SetLabel(self.__speed_text_format.format(val)) - self.__speedo.SetSpeedValue(val) - - def SetAngleRange(self, start: float, end: float) -> None: - self.__speedo.SetAngleRange(start, end) - - def SetIntervals(self, intervals: List[int]) -> None: - self.__speedo.SetIntervals(intervals) - - def SetTicks(self, ticks: List[str]) -> None: - self.__speedo.SetTicks(ticks) - - def SetTicksColor(self, - color: Union[Tuple[int, int, int], wx.Colour]) -> None: - self.__speedo.SetTicksColour(color) - - def SetNumberOfSecondaryTicks(self, num: int) -> None: - self.__speedo.SetNumberOfSecondaryTicks(num) - - def SetTicksFont(self, font: wx.Font) -> None: - self.__speedo.SetTicksFont(font) - - def SetMiddleText(self, text: str) -> None: - self.__speedo.SetMiddleText(text) - - def SetMiddleTextColor( - self, - color: Union[Tuple[int, int, int], wx.Colour] - ) -> None: - self.__speedo.SetMiddleTextColour(color) - - def SetMiddleTextFont(self, font: wx.Font) -> None: - self.__speedo.SetMiddleTextFont(font) - - def SetHandColor(self, - color: Union[Tuple[int, int, int], wx.Colour]) -> None: - self.__speedo.SetHandColour(color) - - def DrawExternalArc(self, draw: bool) -> None: - self.__speedo.DrawExternalArc(draw) - - def ShowTextValue(self, show: bool) -> None: - self.__speed_text_panel.Show(show) - - def SetTextValueFormat(self, fmt: str) -> None: - self.__speed_text_format = fmt - - def SetTextValueFont(self, font: wx.Font) -> None: - self.__speed_text.SetFont(font) - - def SetTextValueColor( - self, - color: Union[Tuple[int, int, int], wx.Colour] - ) -> None: - self.__speed_text.SetForegroundColour(color) - - def SetTextValuePosition(self, - pos: Union[Tuple[int, int], wx.Point]) -> None: - self.__speed_text_panel.SetPosition(pos) - - -class SpeedoElement(WidgetElement): - _SPEEDO_PROPERTIES: ValidatedPropertyDict = { - 'min_val': (0, valid.validateOffset), - 'max_val': (100, valid.validateOffset), - 'num_intervals': (20, valid.validateOffset), - 'num_secondary_ticks': (5, valid.validateOffset), - 'center_text': ('', valid.validateString), - 'start_angle': (-math.pi/6, valid.validateTimeScale), - 'end_angle': (7*math.pi/6, valid.validateTimeScale), - 'tick_color': ((0, 0, 0), valid.validateColor), - 'bg_color': ((255, 255, 255), valid.validateColor), - 'error_color': ((255, 0, 0), valid.validateColor), - 'text_color': ((0, 0, 0), valid.validateColor), - 'hand_color': ((255, 0, 0), valid.validateColor), - 'tick_font_size': (12, valid.validateOffset), - 'center_font_size': (12, valid.validateOffset), - 'draw_external_arc': (False, valid.validateBool), - 'speedo_delay': (0, valid.validateOffset), - 'show_text_value': (False, valid.validateBool), - 'text_value_format': ('{:03.2f}', valid.validateString), - 'text_value_font_size': (12, valid.validateOffset), - 'text_value_color': ((0, 0, 0), valid.validateColor), - 'text_value_position': ((50, 90), valid.validatePos), - } - - _ALL_PROPERTIES = WidgetElement._ALL_PROPERTIES.copy() - _ALL_PROPERTIES.update(_SPEEDO_PROPERTIES) - - _DEFAULT_VALUE = 0 - _DEFAULT_DIMENSIONS = (100, 100) - - @staticmethod - def GetType() -> str: - return 'speedo' - - @staticmethod - def GetDrawRoutine() -> Callable: - return SpeedoElement.DrawRoutine - - def __init__(self, *args: Any, **kwargs: Any) -> None: - WidgetElement.__init__(self, self.__InitSpeedo, *args, **kwargs) - self.__start_tick: Optional[int] = None - self.__tick_delay: Optional[int] = None - - def __GetWidget(self) -> SpeedoWidget: - widget = self._GetWidget() - assert widget is not None - return cast(SpeedoWidget, widget) - - def __SetSpeedoBackgroundColor(self, color: Tuple[int, int, int]) -> None: - widget = self.__GetWidget() - widget.SetBackgroundColor(color) - - def __UpdateSpeedoBackgroundColor(self) -> None: - self.__SetSpeedoBackgroundColor( - cast(Tuple[int, int, int], self.GetProperty('bg_color')) - ) - - def __SetSpeedoError(self, error: bool) -> None: - if error: - self.__SetSpeedoBackgroundColor( - cast(Tuple[int, int, int], self.GetProperty('error_color')) - ) - else: - self.__UpdateSpeedoBackgroundColor() - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int) -> None: - WidgetElement.DrawRoutine(self, pair, dc, canvas, tick) - - if self.__start_tick is None: - self.__start_tick = canvas.context.dbhandle.api.getFileStart() - if self.__tick_delay is None: - self.__tick_delay = cast(int, self.GetProperty('speedo_delay')) - self.__tick_delay *= pair.GetClockPeriod() - - offset_tick = tick - self.__start_tick - if offset_tick < self.__tick_delay: - value_scale = float(offset_tick) / self.__tick_delay - else: - value_scale = 1.0 - - val = None - try: - self.__SetSpeedoError(False) - val = float(pair.GetVal()) - except ValueError: - self.__SetSpeedoError(True) - val = SpeedoElement._DEFAULT_VALUE - val = value_scale * val - widget = self.__GetWidget() - widget.SetValue(val) - widget.Update() - - def SetProperty(self, key: str, val: PropertyValue) -> None: - WidgetElement.SetProperty(self, key, val) - if key == 'speedo_delay': - self.__tick_delay = None - if self._GetWidget() is not None: - self.__UpdateSpeedoProperties() - - def __UpdateSpeedoProperties(self) -> None: - widget = self.__GetWidget() - widget.SetAngleRange(cast(float, self.GetProperty('start_angle')), - cast(float, self.GetProperty('end_angle'))) - - # Create the speedometer intervals - min_val = cast(int, self.GetProperty('min_val')) - max_val = cast(int, self.GetProperty('max_val')) - num_intervals = cast(int, self.GetProperty('num_intervals')) - interval_size = (max_val - min_val) // num_intervals - intervals = list(range(min_val, - max_val + interval_size, - interval_size)) - widget.SetIntervals(intervals) - - self.__UpdateSpeedoBackgroundColor() - - # Create ticks - ticks = [f'{interval}' for interval in intervals] - widget.SetTicks(ticks) - widget.SetTicksColor( - cast(Tuple[int, int, int], self.GetProperty('tick_color')) - ) - - # Secondary ticks - widget.SetNumberOfSecondaryTicks( - cast(int, self.GetProperty('num_secondary_ticks')) - ) - - # Tick font - widget.SetTicksFont( - wx.Font(cast(int, self.GetProperty('tick_font_size')), - wx.SWISS, - wx.NORMAL, - wx.NORMAL) - ) - - # Center text - widget.SetMiddleText(cast(str, self.GetProperty('center_text'))) - widget.SetMiddleTextColor( - cast(Tuple[int, int, int], self.GetProperty('text_color')) - ) - widget.SetMiddleTextFont( - wx.Font(cast(int, self.GetProperty('center_font_size')), - wx.SWISS, - wx.NORMAL, - wx.BOLD) - ) - - # Hand color - widget.SetHandColor( - cast(Tuple[int, int, int], self.GetProperty('hand_color')) - ) - - # External arc - widget.DrawExternalArc( - cast(bool, self.GetProperty("draw_external_arc")) - ) - - widget.ShowTextValue(cast(bool, self.GetProperty("show_text_value"))) - widget.SetTextValueFormat( - cast(str, self.GetProperty("text_value_format")) - ) - widget.SetTextValueColor( - cast(Tuple[int, int, int], self.GetProperty('text_value_color')) - ) - widget.SetTextValuePosition( - cast(Tuple[int, int], self.GetProperty('text_value_position')) - ) - widget.SetTextValueFont( - wx.Font(cast(int, self.GetProperty('text_value_font_size')), - wx.SWISS, - wx.NORMAL, - wx.BOLD) - ) - - # Set value - widget.SetValue(SpeedoElement._DEFAULT_VALUE) - - canvas = self._GetCanvas() - assert canvas is not None - size = canvas.GetSize() - widget.SetSize(size) - widget.SetPosition((0, 0)) - - def __InitSpeedo(self) -> None: - canvas = self._GetCanvas() - assert canvas is not None - self._SetWidget(SpeedoWidget(canvas)) - self.__UpdateSpeedoProperties() - - -SpeedoElement._ALL_PROPERTIES['dimensions'] = \ - (SpeedoElement._DEFAULT_DIMENSIONS, valid.validateDim) -SpeedoElement._ALL_PROPERTIES['type'] = (SpeedoElement.GetType(), - valid.validateString) diff --git a/helios/pipeViewer/pipe_view/model/utilities.py b/helios/pipeViewer/pipe_view/model/utilities.py deleted file mode 100644 index 67afe4996b..0000000000 --- a/helios/pipeViewer/pipe_view/model/utilities.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations -import logging -import sys -from typing import Any, Dict, Optional, Union, cast -import yaml - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - -class Settings: - ''' - Will convert a nested dict into an object to make it easier to access - various members - ''' - - def __init__(self, value: Union[str, Dict[str, Any]]) -> None: - ''' - take a filename, open it as a yaml file and convert that into an object - with an arbitrary hierarchy - ''' - if isinstance(value, str): - value = yaml.safe_load(open(value)) - - assert isinstance(value, dict) - - for a, b in value.items(): - if isinstance(b, (list, tuple)): - setattr(self, - a, - [Settings(x) if isinstance(x, dict) else x for x in b]) - else: - setattr(self, a, Settings(b) if isinstance(b, dict) else b) - - -class LogFormatter(logging.Formatter): - - baseFormats = {logging.DEBUG: "DEBUG: {funcName}:{lineno}: {message}", - logging.WARNING: "WARN: {message}", - logging.ERROR: "ERROR: {funcName}: {message}", - logging.INFO: "{message}", - 'DEFAULT': "{levelname}: {message}"} - - def __init__(self, - prefixValue: Optional[str] = None, - isSmoke: bool = True, - autoWidth: bool = False) -> None: - self.reset(prefixValue, isSmoke, autoWidth) - - def reset(self, - prefixValue: Optional[str] = None, - isSmoke: bool = True, - autoWidth: bool = False) -> None: - - self.formats: Dict[Union[int, str], logging.StrFormatStyle] = {} - - if autoWidth: - width = len(prefixValue) if prefixValue else 1 - else: - width = 16 - - for k, v in LogFormatter.baseFormats.items(): - k = cast(Union[int, str], k) - # TODO find some more elegant way to get this result - # if k != logging.INFO and prefixValue: - if prefixValue and (isSmoke or k != logging.INFO): - self.formats[k] = logging.StrFormatStyle( - f'[{bcolors.OKGREEN if sys.stdout.isatty() else ""}' - f'{prefixValue if prefixValue else "":{width}}' - f'{bcolors.ENDC if sys.stdout.isatty() else ""}]{v}' - ) - else: - self.formats[k] = logging.StrFormatStyle(v) - - def format(self, record: logging.LogRecord) -> str: - # This doesn't play nicely with the base definition of - # logging.Formatter - self._style = self.formats.get( - record.levelno, - LogFormatter.baseFormats['DEFAULT'] # type: ignore - ) - return logging.Formatter.format(self, record) diff --git a/helios/pipeViewer/pipe_view/model/widget_element.py b/helios/pipeViewer/pipe_view/model/widget_element.py deleted file mode 100644 index 549d1681a7..0000000000 --- a/helios/pipeViewer/pipe_view/model/widget_element.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations -from .element import LocationallyKeyedElement -from . import element_propsvalid as valid - -import wx -from typing import Any, Callable, Optional, Tuple, Union, cast, TYPE_CHECKING - -if TYPE_CHECKING: - from .element_value import Element_Value - from .element import PropertyValue - from ..gui.layout_canvas import Layout_Canvas - - -class WidgetElement(LocationallyKeyedElement): - _ALL_PROPERTIES = LocationallyKeyedElement._ALL_PROPERTIES.copy() - - @staticmethod - def GetType() -> str: - return 'widget' - - @staticmethod - def IsDrawable() -> bool: - return True - - @staticmethod - def GetDrawRoutine() -> Callable: - return WidgetElement.DrawRoutine - - def __init__(self, - widget_init: Callable, - *args: Any, - **kwargs: Any) -> None: - self.__canvas: Optional[wx.Panel] = None - self.__parent: Optional[Layout_Canvas] = None - self.__widget: Optional[wx.Window] = None - self.__widget_init = widget_init - LocationallyKeyedElement.__init__(self, *args, **kwargs) - - def SetProperty(self, key: str, val: PropertyValue) -> None: - LocationallyKeyedElement.SetProperty(self, key, val) - if self.__canvas is None: - return - val = self.GetProperty(key) - if key == 'position': - val = cast(Tuple[int, int], val) - self.__SetCanvasPosition(val) - self.__canvas.Update() - elif key == 'dimensions': - val = cast(Tuple[int, int], val) - self.__SetCanvasSize(val) - self.__canvas.Update() - - def __SetCanvasPosition(self, - val: Union[Tuple[int, int], wx.Point]) -> None: - if self.__canvas is not None: - assert self.__parent is not None - self.__canvas.SetPosition(self.__parent.CalcScrolledPosition(val)) - - def __SetCanvasSize(self, val: Union[Tuple[int, int], wx.Size]) -> None: - if self.__canvas is not None: - self.__canvas.SetClientSize(val) - self.__canvas.SetSize(val) - - def _GetCanvas(self) -> Optional[wx.Panel]: - return self.__canvas - - def __ShowCanvas(self) -> None: - assert self.__canvas is not None - self.__canvas.Show(True) - - def __HideCanvas(self) -> None: - assert self.__canvas is not None - self.__canvas.Show(False) - - def DrawRoutine(self, - pair: Element_Value, - dc: wx.DC, - canvas: Layout_Canvas, - tick: int) -> None: - if self.__canvas is None: - self.__CreateCanvas(canvas) - if self.__widget is None: - self.__InitializeWidget() - assert self.__canvas is not None - self.__canvas.Update() - - def EnableDraw(self, enable: bool) -> None: - LocationallyKeyedElement.EnableDraw(self, enable) - if not self.ShouldDraw(): - self.__HideCanvas() - return - else: - self.__ShowCanvas() - - def __SkipEvent(self, event: wx.Event) -> None: - assert self.__parent is not None - wx.PostEvent(self.__parent, event) - event.Skip() - - def __OnFocus(self, event: wx.FocusEvent) -> None: - assert self.__parent is not None - self.__parent.SetFocus() - event.Skip() - - def _SetWidget(self, widget: wx.Window) -> None: - self.__widget = widget - - def _GetWidget(self) -> Optional[wx.Window]: - return self.__widget - - def __InitializeWidget(self) -> None: - assert self.__widget_init is not None - self.__widget_init() - assert self.__widget is not None - self.__widget.Bind(wx.EVT_LEFT_DOWN, self.__SkipEvent) - self.__widget.Bind(wx.EVT_KEY_DOWN, self.__SkipEvent) - self.__widget.Bind(wx.EVT_KEY_UP, self.__SkipEvent) - self.__widget.Bind(wx.EVT_SET_FOCUS, self.__OnFocus) - - def __CreateCanvas(self, parent: Layout_Canvas) -> None: - assert self.__canvas is None - self.__parent = parent - self.__canvas = wx.Panel(parent) - self.__SetCanvasSize(wx.Size(self.GetXDim(), self.GetYDim())) - self.__SetCanvasPosition(wx.Point(self.GetXPos(), self.GetYPos())) - - -WidgetElement._ALL_PROPERTIES['type'] = \ - (WidgetElement.GetType(), valid.validateString) diff --git a/helios/pipeViewer/pipe_view/model/workspace.py b/helios/pipeViewer/pipe_view/model/workspace.py deleted file mode 100644 index ed2e7be7d9..0000000000 --- a/helios/pipeViewer/pipe_view/model/workspace.py +++ /dev/null @@ -1,349 +0,0 @@ -# @package workspace.py -# @brief Contains the workspace class which is central to an argos session - -from __future__ import annotations -import os -import sys -import weakref -import logging -import wx -from typing import Any, Dict, List, Optional, Tuple, Union, TYPE_CHECKING - -from . import group -from .layout import Layout -from .layout_context import Layout_Context -from ..gui.layout_frame import Layout_Frame -from ..gui import autocoloring - -if TYPE_CHECKING: - from .database import Database - from .settings import ArgosSettings - - -# Responsible for managing Databases, LayoutContexts & and -# Groups, the building blocks for the model-side of the application. -class Workspace: - - WINDOW_CASCADE_STEP_X = 20 - WINDOW_CASCADE_STEP_Y = 20 - - # Constructor - def __init__(self, settings: ArgosSettings) -> None: - self.__groups = [group.Group()] # Synchronization groups - # Weak references to frames - self.__frames: List[weakref.ReferenceType[Layout_Frame]] = [] - self.__settings = settings - - # Figure out a starting window pos - pos = (0, 0) - self.__primary_window_size: Optional[Tuple[int, int]] = None - for i in range(wx.Display.GetCount()): - d = wx.Display(i) - if d.IsPrimary(): - self.__primary_window_size = d.Geometry.Size.Get() - pos = d.Geometry.Position.Get() - break - - self.__primary_window_coords = pos - self.__last_window_pos: Optional[Tuple[int, int]] = None - - # User resource dirs - self.__user_resource_dirs: List[str] = [] - - # Builtin resource dir - this_script_dir = os.path.dirname(os.path.join(os.getcwd(), __file__)) - self.__builtin_resource_dir = os.path.join( - os.path.dirname(this_script_dir), - 'resources' - ) - - self.SetPalette(self.__settings.palette) - self.SetColorShuffleState(self.__settings.palette_shuffle) - - def __str__(self) -> str: - return '' - - def __repr__(self) -> str: - return self.__str__() - - # Opens a new layout frame - # @param lf Layout filename. If '' or None, opens blank layout - # @param db Database to link this layout frame to through a layout context - # @param start_tick Startup tick. If None, default is used - # @param loc_vars Location variable values. May be None or a dictionary - # @param norefresh (kwarg only, default False) If True, do not show or - # refresh the frame/layout. This is used if the client wants to resize or - # change current time before rendering the layoutframe - # @return frame created - def OpenLayoutFrame(self, - lf: Optional[str], - db: Database, - start_tick: Optional[int], - update_enabled: bool, - title_prefix: str, - title_override: str, - title_suffix: str, - loc_vars: Optional[Dict[str, str]], - **kwargs: Any) -> Layout_Frame: - norefresh = False - for k, v in kwargs.items(): - if k == 'norefresh': - norefresh = True - else: - raise KeyError(f'Unknown kwargs {k}') - - if lf is None: - lf = '' - # If '', no layout will be loaded (as the user intended) - layout = Layout(lf, workspace=self) - - # Jump to startup tick on construction - lc = Layout_Context(layout, db, start_tick, loc_vars) - - frame = Layout_Frame(self, - lc, - update_enabled, - title_prefix, - title_override, - title_suffix) - - if not norefresh: - frame.Show(True) - # Updates meta-data so that everything is good on the first - # rendering - lc.RefreshFrame() - - return frame - - def AddFrame(self, frame: Layout_Frame) -> None: - self.__frames.append(weakref.ref(frame, self.__RemoveFrame)) - - # Set the palette for this workspace - def SetPalette(self, palette: str) -> None: - self.__settings.palette = palette - # The autocoloring module is global, so we can set the mode once and - # then update all of the canvas brushes - autocoloring.SetPalettes(palette) - for f in self.__frames: - frame = f() - if frame is not None: - frame.GetCanvas().RefreshBrushes() - - # Set the color shuffle state for this workspace - def SetColorShuffleState(self, state: str) -> None: - self.__settings.palette_shuffle = state - # The autocoloring module is global, so we can set the mode once and - # then update all of the canvas brushes - autocoloring.SetShuffleModes(state) - for f in self.__frames: - frame = f() - if frame is not None: - frame.GetCanvas().RefreshBrushes() - - # Get the palette for this workspace - def GetPalette(self) -> str: - return self.__settings.palette - - # Get the color shuffle state for this workspace - def GetColorShuffleState(self) -> str: - return self.__settings.palette_shuffle - - def AddGroup(self) -> None: - # not in first cut - return - - def GetGroup(self) -> None: - return - - def GetGroups(self) -> List[group.Group]: - return self.__groups[:] - - def GetDefaultGroup(self) -> group.Group: - assert len(self.__groups) > 0, \ - 'Workspace should never have fewer than 1 sync group' - return self.__groups[0] - - def NewLayoutContext(self) -> None: - # doit - return - - def OpenDatabase(self) -> None: - return - - def CloseDatabase(self) -> None: - return - - def AddUserResourceDir(self, dirname: str) -> None: - self.__user_resource_dirs.append( - os.path.join(os.getcwd(), - os.path.expanduser(dirname)) - ) - - def GetUserResourceDirs(self) -> List[str]: - return self.__user_resource_dirs[:] - - def GetBuiltinResourceDirs(self) -> List[str]: - return [self.__builtin_resource_dir] - - # Get all resources in the proper resolution order - def GetResourceResolutionList(self) -> List[str]: - return self.GetBuiltinResourceDirs() + self.GetUserResourceDirs() - - # Resolve a resource by its filename in all known resource directories in - # the proper resolution order - # @param filename Filename to find in resource dirs. If absolute path, - # does not check resource dirs - # @param ignore Single string or a sequence of stringsd. Should some - # resolved resource paths be skipped? If this was invoked and the file - # path it returns was not a valid file or could not be opened, then this - # method can be re-invoked with that path (or a sequence of paths) in - # order to ignore them. This is done by comparing canonical paths - # @return Absolute path of a resource specified by filename in some - # resource dir to exist. A string will be returned only if the filed it - # points to exists (it maybe not be the correct resource though). - # Otherwise returns None to indicate a failed resolution. - def LocateResource( - self, - filename: str, - ignore: Union[str, List[str]] = [], - try_first: Optional[Union[str, Tuple[str, ...], List[str]]] = None - ) -> Optional[str]: - if isinstance(ignore, str): - ignore = [ignore] - - if os.path.isabs(filename): - for igp in ignore: - if os.path.realpath(igp) == filename: - return None # Caller ignored their own input absolute path - if os.path.exists(filename): - return filename - return None - - rds: List[str] = [] - if isinstance(try_first, (list, tuple)): - rds.extend(try_first) - elif try_first is not None: - rds.append(try_first) - rds.extend(self.GetResourceResolutionList()) - - for d in rds: - fullpath = os.path.join(d, filename) - - # Ignore it? - skip = False - for igp in ignore: - if os.path.realpath(igp) == filename: - skip = True # Caller ignored this path - if skip: - continue # Skip this iteration - - if os.path.exists(fullpath): - return fullpath - - return None - - # Exit all of argos by closing all frames - def Exit(self) -> None: - for idx, f in enumerate(self.__frames): - frame = f() # weakref deref - if frame is not None: - try: - if not frame._PromptBeforeClose(): - return # About exit - except wx.PyDeadObjectError: - print(f'Failed to access workspace frame idx {idx}: ' - f'{frame}. It was already destroyed', - file=sys.stderr) - - # Now that all layouts had a chance to save, discard, or cancel, force - # All close to avoid prompting again - while len(self.__frames) > 0: - framewr = self.__frames[0] - frame = framewr() # weakref deref - if frame: - try: - frame._HandleClose(force=True) - except wx.PyDeadObjectError: - print( - f'Failed to access {frame}. It was already destroyed', - file=sys.stderr - ) - - # Should have been removed by _HandleClose. If it wasn't (due to - # exception or other bug) remove it anyway - if framewr in self.__frames: - self.__frames.remove(framewr) - - # Returns a 2-tuple window position based on the existing windows - def GetNextNewFramePosition( - self, - size: Union[Tuple[int, int], wx.Size] - ) -> Tuple[int, int]: - lg = logging.getLogger('Workspace') - if self.__last_window_pos is not None: - # Selet position based on previous position, with an x and y offset - lg.debug('Getting New Frame Position based on last pos') - pos = self.__last_window_pos - pos = (pos[0] + self.WINDOW_CASCADE_STEP_X, - pos[1] + self.WINDOW_CASCADE_STEP_Y) - self.__last_window_pos = pos - return pos - elif self.__primary_window_size is not None: - # Determine window pos so it is centered on screen - lg.debug('Getting New Frame Position based on screen geometry') - pos = (int(self.__primary_window_coords[0] + - (self.__primary_window_size[0]/2 - size[0]/2)), - int(self.__primary_window_coords[1] + - (self.__primary_window_size[1]/2 - size[1]/2))) - # \todo Ensure that this is actually ON the screen. Should also - # adjust if the window is largely off the screen - self.__last_window_pos = pos - return pos - else: - # Just pick something since we don't have geometry - lg.debug('Getting New Frame Position using wx.DefaultPosition') - return wx.DefaultPosition.Get() - - # Remove a frame given its weak pointer - def __RemoveFrame(self, - frame: weakref.ReferenceType[Layout_Frame]) -> None: - assert isinstance(frame, weakref.ref) - assert frame in self.__frames - self.__frames.remove(frame) - - # Remove a frame given its reference. This is meant to be called by Frames - # themselves, but could be used for inter-workspace frame management - # @return True if the frame was removed, False if not - def RemoveFrame(self, frame: Layout_Frame) -> bool: - for fwr in self.__frames: - if fwr() == frame: - self.__RemoveFrame(fwr) - return True - - return False - - # Get settings object - def GetSettings(self) -> ArgosSettings: - return self.__settings - - # Updates settings and propagates them to the rest of the model if - # necessary - def UpdateSettings(self, new_settings: Dict[str, Any]) -> None: - self.__settings.update(new_settings) - update_funcs = [] - if 'layout_font_size' in new_settings: - update_funcs.append(lambda f: f().GetCanvas().UpdateFontSize()) - - if 'playback_font_size' in new_settings: - update_funcs.append( - lambda f: f().GetPlaybackPanel().UpdateFontSize() - ) - - if update_funcs: - for f in self.__frames: - for fn in update_funcs: - fn(f) - - # Cleanup resources at shutdown - def Cleanup(self) -> None: - self.__settings.save() diff --git a/helios/pipeViewer/pipe_view/resources/Actions-transform-move-icon.png b/helios/pipeViewer/pipe_view/resources/Actions-transform-move-icon.png deleted file mode 100644 index 7e3a63b156433e95ddef8bd7bbcf113897f4b294..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1544 zcmV+j2KV`iP)+d?sjupv!`S zlBg(;fv|_0OnmUrb)t(?6mjk$6I7rSO51(6(8<)rg2G9@|Ms5$zvq1ax#yh!mZ4V> zhIkl7q^I_wn;80&9EM@4I2?|%Ua$AmX>~V7lmNqF!spDHb70}Zg_M|>_#;%a^*X(N zLS}j?Pl~8(GI%Q_=Ex~ID8a@JyVg0$Boc%uY7tEtB zwaTDEB9hqLl6-w*_|Am9Te z0s$VL9>SucQzRCPsidTYVq@3o9{&CC9Oe!HM*)*c(i8$hK>f@!cEG{K?=21t2??R- z=;(G;r%DRlch5Ms2E;(q;0Z8fgE12bhRM~q<`0zXx;!)(8NlXq3oa ztyZ&kHlspRCJ{iXh2gSi<*Jp|zCJz_{!zqb+`s#bgO^F*G#BUc!+D1&Dd`ictf;(= zYKcUJeEYuy;CLb~*$xg40sHptC3kl>(jvp%-Dc>2yLX>WtaJZ{n4wV^;{kWzCnV!; z?9g~9iGZTdw{HuO$yX4cFQAIkm6VXMQQy+ilFL4awQWxc^c^hg&|cFL5;mB}$HyDW zjBPSPGa(&6eu846*F3;I+hi{GKl63Kg8B28N|#buc{!Dqlvbhq{6UY;zAnF+ej!H- zj*wojHS~+o=3+SEQ*v@L<>u!834gJO#&IKoa`f}Soa`L3n_x?i+aD3gkeNV80)ZI` zaXoqRB*|p*YOJ0GOtKk2?w9(@b=0fvC6Q1>o*v$Gx9P4Mo3zf;oX=NRS7S~?s;*80 zw7tC~9&JGqYa6<7;Ua~Hhf`x?W3-Wg5(<~}v^46~YpA{B2?-E3GdO2Sm>~-J`A0|* zrC?7ze_*QJ1iPyZ4OdCs(}hN)quDe4sqxlr56nH0jFU7J7Z>v;PjRG9RR?vk1h8g| z#aJ3^X+?SY`Lt=%CQWN=Yp9U`TPmBat*v9XY}q0RT^39oo$Uypoou{0z)-fhc7Eo|4icp2H-2>h4yPg?J3zy4gtU z*T+*`UELY1S&KZl{@*HKXbQ%CNWQt-wq=lmqdoFKM;U7hvDp&%VlqRd3%x~ zGUj(^-v%_#t?Uy$J>U5j*uF8oBJSgLv}60O?zXnJJ}A^A~M2FE|-&3 zD%G?+Xo-O?_XSZh$+)S)=D}w|k#K5I(EChQRu-BJpK5Aqi0vIp<A)N7n(D#p8!LFPEJ$HO1~;**e1i}at%th z53s$WyzDeZMJjGW>4He8O*1)IVW=o0V{6a~*#~GceXXCz;}ee&kcX$I4&^!*3HO8P z<_1q-qNSz9wfg#c5tq*~v|EXl4SQBlK|#SmIFA^dxuFP}j!Y&?$|L=R+QlKnMpr)wI;UFYSG}Gx~{7Q@$YT^|kjTbZX!pw`g#Aw_M z7j3!Fg&8qU{9w*#Dj{5YVP>M6OW2l8H#KBaHfE`m0fUxd(+}G7zR%-@oTe=W3c)M+ zCnwK2?|E~c-}683$2q{JHnpk$yM%Sp*Vo5H1ZKv65sTY;dU^x^*en1kC4>+lqIE|A zVCIssRNn#s!!R~H$x3kvfRwV}VNRzrcb$J&7OvKo-)@>F;_>*(4oE2jLI|JLNeBVA z+nwuJs_NyO$d6iUjE;^L7Z_MF2mv4%4Eh21EOLj#frf?#2q9KXvfMqT6hfg85{X3q zAPjVMb@@vI06>3#zn__N5O%wLRVPY;5D`M55R%DcF1gk^;Pd(XC8uF22Y5HU^+R8OE=Hr!e2uRsr16ZbNfe* zmjz%!p z*Df9$?JWxc00svKmki8&C$gMDkdDCH@+vB16)r~4B10JvX)qC(7)*q_tG3~u>icly zWFVc+WLt+1{W&4a0q}agej!A_Fbqg3p_D=_7R%kga{Fi8S$!8gRZRuS-+A~Wyu0V) zMF_%pJcPN-Ej+*Tg-U4Z1pry@uI2T5{Zh(+loC=(n5KznG>Vy-8C#5W0O0JKU|%ChuMo004A!bmSlm!+=r>M1+`_#x_?g!ii8p<(9giIF6%F zod6RdZ6@J#R6?6vw`Kqf!Z3^+gp?9mGl&U9s-W^>(V`QYG_)eIE&u>%Z*MPn9S+ku zc4O`jxGOwhVjMcRA8!r4lI#D*>3w+phnK)aunPy`sW`Oaxpf2NK^(T(Y=>hy`sPsh zd$iQF1q5&i3P z2V%hQ_j>_&G+|Agb6$A^e!pKQ*MuKh0)arFKwK0>n9b&glRzL4@Dh$VkDc ziJ}O*-L8{sRcZoN#fobu9OoQHqw&7v`J-62flw%_Im8Vm+BG&Jl>yDr?qMAf%A_ULguC!K`T+6-`D zJ-v!wSFU2_*7UBN**$yBJ%6$2`+(o?FEEjR@Z=lVKpf+yZ}3*jd$5@sF_-)uD{>4f z%c0Kj2#(gcV1fx>|NI5t*t-4x^|OBr6@4M``F!`7a5|m3359kM6hFtCt!|jAthoO7 zRb(j(A_h(bP8^&FP1YlLyt);ijSprrS;aNmzZ5U(9O&%q^b*maMk~uQVzHRs7ja<; z4of4P*5(57PoBSkPha@zjtN3^dl{SA+c=ZCjuh|rjqc$_krHt zUfl!{!Ln12#m$>=n4I9m(SOa252xSNf8RHC1|Lkm15N~^AYxnI22=TFSvR7-zCOJ% z4{x?%Ix>M{wNDl-qr1sL9L}0%%*Pi%Dt-fiL*?8FCnqPv9UUEln6tFcVg{|!6R=wy z_$$2%aE>d>WB74tR3Amj;(%ms#N}~cHkZx4zWqxoS(X6+n3|djcbs}zsFQ8&R}y1* z=I{wL*SA2AVBAzf2qBP=IQOtPXEw}GmkDuN1Syo+jnJ^JcGBB>zjJ&T_vW^=p#_`| zdS3eF%h#O$`F-b{?;ha4ZpwNGdV70W2!WE45(pt6gy{79{lTpgAl5rTL||qRk+BZ+ z_4VC90fuvCHrD_G{r&yHe~W-gVk<-xMZoEF!r^dah!_|c2;MgW0O0X>OcAU5n-WoU z4gf$+O%3dJyLErVB8nnl2v8KoBw^VP3=R$k|8WA@Bn%NoP82I*Lj=q&rYNQeYl#jG z4F$KHfVB*>Nx0o^*zI=f*@8tBOu*`JO+Q~HVTdpib9i_-xTOTLqt8dea&7C1D4IY% z3I>U65zHJI85uc}??1Le2^eeZZ!Xi2#+3aB%sE4da}2uF0o~U6{6JHw)N%Tv|N4!h z1P~DvMakI|D3{`2U--6?fBmJcJC~> z3i34~W`>j!6R|7Uv27>3<+T`(U4}|3h$On7_z>?le*`816CskA#j?7Hmz!QKgQU$9 zkWwNNiD1S#fu}q#;FtKM>6}-u4S*?tnJ{~M3iX~nkQ(nVI0teOAR^4q&qGyJ+}W{& zhg^GbE;?#PZna}i-^;nTFC4;(ybGtJ3{vCjf~&yDdP9Vi(sV9_fUXmW2}F8EDI&=~)~ZI}9euh@4ih!6BST{FQ*z zdAaBu&@>Ix)6+;Mlh$_InNZ^IT$($NhKk0FhbyC-5&^{Ux@r-+F$L0bt*AysL`WnO z#%usZQ6Q!4?C9tSKG)ai`fVw6`_`RkRa50Y{2aRqFlR*0#HcRwz-6n#=-3%mYVxh3 zKS-=J4CerF!5>Pd)A~!FecwZ@2|VsOh)Tx;5D-upaM>!btEv&cZM*R4xz9PZq6QXD z-JC0EGHP#cXJaa|PDfo`U29&s<`WIikW@zZdLOIY|L~J=Z?6F}V=1wO$?zn;zIrjO zrnQcRlhOAJ9?ja?+SZI_-QC@JpV)oJ>nd2oQK|8ZQsa85Q>m03A)UXGIv-m&8J)rZ bX0g8keRz($=E$Y*00000NkvXXu0mjf2x|Y4 diff --git a/helios/pipeViewer/pipe_view/resources/add-space-up.png b/helios/pipeViewer/pipe_view/resources/add-space-up.png deleted file mode 100644 index db92e5985e6d78c6e5444bf53c636b5d39eacf97..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1100 zcmV-S1he~zP)B`|2)FCIZEbC#T?dH8Vj*VUXl_AtbaaGvZ3MAcEJQ>>B7zVCvMj^laKL;|M8Uqk zzTE+cMx&NF5uvQC3_hO^s;WYkWgA3qZ|}At?%A$6GY8F9o92?VK0&wJja)8=d_HfE z;>Fh1R%vKxC{Z#%Boev9-0$~W=b0G*;BvW;&1P*7EiEn5$jC@y#{j0ed7~+cVkI|? z%?bb>j|Z7d#s<;c+$@cbjwb#yKqL~eJ!*=g6ir@OB}uX%G)=Q0nwpxVv9Ynlwjaa( z{{B0X`~Ci+$v4(KYe@1Wrq?Esmaan=0!q#WCFf3DE1zpmv`;T=31FHR&RkX1Et8vd z34h^SSp*GL58;WL=MZq!0Sqi`2Ms_WCe4;D)c~O9HI2;0CnQu2aGh^uyiqe}Lo!E*pkIPuxUX{@A|Ph5EOj~*L9I2^VmR}|&0 z$<5a~Yxv4}8m~2W!Rc_}!s<9Q(m*7@L||et5$<#C!~O0?d@%H$kh6|$UMw*g~w9G4AjP21YC6(UijKFfB&gZ007++U0@>2rDx!lJvjR4F?ryJ zSPy_#3IS|udnS`9w$>N@?5g)?98+J#ueYXb$*t}Ytsl&oxp^56tA`vH5V6$}FsfHjB6Nn63@{J-Q zNDK%P5C%y$fX>cNX{Vctmt&8OUAghwOR6(~x%`YZ^TOnGdkl)B9O-Nt!r&Vv?T89x z81(6Pv)^EUX2Nx3^MSetplZq4Ke!rO7iB?->K?Hyod11>Lq~L zmYj)EQ=!5mS7UJEtR{4E{D14f`S#>L`Mh!Ti;KO)x&u$D&%&$R3kMvK81TrIXsAAj zT6Y8bhQH*soEDt#xi-7wJ>h10=HvY@kdOu+4;=O$dEjaIs{CMPtfW^k_2(4MT}tG& zy#CJonbb!mZA!P&p%3dTS;yN#=N6&!gTf%E5Ijc)e=7_=HGd{`W$zD7q5cKKd}1OU SwF2}20000iHcP|IE%$ZL#^=u-3oN_5 zGA=}Sk<+w>+}l=5OWZVDHM$%EMKq>*9oQf2mEL{SNq^C_^quSpkzsdimWrHuYP2Ov z_kszph^uFd!Tj@I_q?wBrhC3SdfHOsV`+Z^pPv=}wDVKeR;fFA<{XD7=(leFSmAT+ z_14+xv5W378Tfwo{`|Jg+U~prGebkxR<7%>O$85Sm?+IYd+h9Lvp_w@56^ze$1V-( zOrK|d&Yhv5YA@fu_v%g(eC;W_@9wI+qCQEkA#+w|I8ce_+{!|gmLrNRjsool5>eMP(}Q;J3tlgcjojL`gk?jFp77cK@6P)uzimWC1`9t|D<03ZY_gS7u;f3Yd<};|rzwmym5K4oN=m~K5fS#y!`{O5HEZ6@kg8HLi;sV~h%}3p6z!BE zIXH#`x`z#4R$1DvsNT}2~=ujeSJ=PR&rM7eSLxZ zR19*a+36C$S`aEIs$FNX;F6&lH6@!R0jA2~aqo8$uGqt%@ zGbeDZ!Qf$>8lTFe_MOCS5N2u->%cCVZNLU2y}H+#Scm({;{&$h=TrNWUxIFCPlx^e zKODO7xOh(Y-6C};T#K^=d1NUdML5j!OllZt7+AubrNF3QHrXgxO-Us=9$>aqk(g=C zK^+}L&VMLFT-qE>Qo4kR->E6>2}kl7y4|f-QO($Ai~HU1k*E5JIIj0RZ2$HC)hXhz zc71)5iyE<$BFwte7FNJHS`;r)c3}AyC6`1~P$0v^GF_ku2*YCnW}vGL0_1p=L2u=? zfabtAy+pt;w8`w=)nv5tXl*bFPXY6ILphdc7>=$&u~b8~A-Bld_!6eOxXDw8yGUNU z*#KY7eS2}k)As20>|dB8D*u=N;SWxtvrWMUPt@G+-%TcEim9UT*(g$B8DSb+$U$5I z#KtZxL@O=o{v$Wqk9Y8l)@o8VHUJv%YuDHPCLEcsiPRx5Cwu^U@v84+ui~h#Uvl9m zHU2(%G_!H#?J%0Ithp6V$|-fecAHIFlc42_GTukA0=G}az7wbGXB}6ZtVQqz@sZY3 zQNtSRIG+AOGg9uwXL6w=rKLC;cx7E>xFnDxj02AV3L%$r>k{wuSasRTl>;p1i}(dI*6n)tb&%}n0ip|U{IuuBvR=c}}PxD6n~3p&<;e`&%4Y=B!`4wK<< z0ktzwff-t_!Jo3!Nyllm4ZVi<0Li6wcFDQ2;pFwg^#B`_Dt&&9%|8 z_j$lbX3))V(;!(bAOCe|ZfDij-?9%3f!<0r%gxr#l3`&kuPT+vG1DO-fykVrevWTq{b@9Sgl$?JY#uYEc*W zdyMkLIja}zCn>K`2gxd#UvxSt@o&YvaiYEo`gO{P5=Lv~iCwh>lIw&k{IWZP^A@i{ zS)OdsUo(OjD(AJLEThA2@>(QU{1G+MIy{=E5^hfqiBk8Y{n^f&o`1Fvy_d4trrBk~ zNXpvjelD4z4V$sK1Vg$)fjl)I`5SK`Q5*8=EiMb0tsH;4T+3+Mz{TLXEJmxeG03?(JgLy6AV$oGSXHR^J7f00HQ^K^|4g94SMruOWAUy z=eKz8)Aeo7?JPR+>6Vwv;&XY`BOV|HR@2+p%w}M-MIJT)-mn=j4jyqI&?j20%Z3Ms zM&)N%v0TU~n@km8XuKv(s4oiOC|n>G9z9zvHyG@t9qDW6z#DqRCTN=xF`6TY9Yv)D zay#cM2ria~Rw!_(DAmXY&vL|tVIh{|VkuHo>890$=VcB1Tc~BVm{~A}n9ODq1oB0A zhKqQ*0Yr4wryK6uN6t_$goqOX-KZRKMmb0)r0ca#-4s(a#!}7%x*_xTU`zvc(jQ?s zE_xq%^+{uxMDa#>HLOKh`C(3=Y-2r=fu`npCf?p}(O7*+MC8g?!dipo0?%|a--6Y7 zxzq|g3X)06(c}cR!ieNp&~+!P7%jWc$$)PuLY_!^7gqAc^#2*0q&%HV;W-w7e?NcR1x5CSW;~+d^-v-Mh-TM zj>7R=F4YeAeFrY!@SGzcea3-|{7$&j<=Z!@wv7Ri0>|z{>=h}PC}l!fx0fmaaYNYQ zSc7QTJeVe-(KI{-_-Wj_xU(QL7q2mjjzfBDv09J&2 zv+U{q*0Wb_IfI@GoUhH>i(5!LVRVqFhu6>0aFM|1-X*A)e7?-;QMeNCk|khq$mm2) zJ5s9)jTgzQTO#hEC0A?-1d1gST3bF$n+=^=gQAGVyq?;?2^3@Vnlgj`^{BEWdvesY zNP<;r|BBJ)4dg)$ZN`EvB}W7@1-mui>Q3DcyAdQF&JNHx+0*Gb@JwfE`~K1Ak88f{ zaz1ovMNfOKXzA}6BatkI+GOOu>PE-Xz6zQ$`_1BF9E@)7jwj^(P&nq^}XQy?E34^*2HVWK0 z02X_KM`)+%{RMHaop0uh5uz_r%42}r{@~%%wD<0t@KvdP?gyf7_)Am&Pkx(2o3CQm z$K``A;HG{9L&tfL%%D!h59zzZcunUqknsH1!Qc4#y(cRsffB);XImGwx5mFh(MW^aIo>FxE~`@?=os_qN>-Q7$P{wZsUOuR>o$%R}z z46H3Kg~#RduKjDoMBbIjUh|5y3!GNU^OJWq*xdhbQkmC&9jbHvac2FEBI(XH2q>hr zGfm{-B|&!Y$~oP|teLU+uIGQt8)cUu~!SxK;2p2r08DEZdQNQja z2)_E_QZ-Sk2j$l)mDL-#gvpubshwYmpA;P-uq)snIHo$vR?K|$zIzGeX~7!@vK#)o z3}2|ATS0Bzs?2?x9>|&!7Tr6xt>*Xw9Hits>e}&oHJ0Q6bv^9J4oaq4uXp;5z17(A z#)GSY=pzXCs|<6PmfU5!T-M)mx?yQVQhHub;HvvNbI)n@H2-I)PuUFI+1zi_`F^t#ii|I7IhokH znTp?zu*pOY@@5hQxJABVNughPZiq{rbU)u@Ieh2&VtTU;FTP-hV?~-J8C5v?@pMhN zx5-H_5MEhz7BnJWcQ@^M3v<>Kx1LH)y`r_n7>?mhY`rvoMyjhhWwaboeb;&3xt-hV zbLRew8G3z~(0jk(<=o~uVbg3-A_?buX5)O0{(5#RspDn}Rr%qT2w%_{!g`DgC}#sT zP^kLCZoHm(?_JQB6bpGxI{-N8wnnOX9A-W!VN4mbn$7g1`+Yu~b3*4fS631-x7iEG<-AI={VQi(DcZSsY+&I($YmYM&sCR8ElBYJXl0xWd~GtR#{xkhoh9{8`3T50-yK& z^l5yP*X{xBA_Bzww)m}R*?X!*OAOi6ZU`P{uRsy76^35BB?&+*)pYL>zO(;`e0J~E zDQE8;;$@@~ZA3>Qnxc(wqJGPpe0$6%r=J;sdcPHDXlI1RruSuoPj|ZMr7E#xmxZBJ zWQ~v{w~j?PwC3H?5@o#J#<<;~J%*1jt(`6Ij)DLVI2mZOY?I^v^ZrGhD5}t)8fBZR zl>c;@Tj#T@cYFWtXg)RCUF8RpRtCq|$yVpi7Q^3*+OeOpl-J!zLQenPipeOMO*i^GFd6Ek3H{q+6mbRoGm_eHV>AC5x1KUXHzt+|7b$=s$fAQ3PF{(u_ zI$i$kdj^l5jmP`S)ti8Qd{bn_ZP4GU){O_{W4}fi;@1CfXu21uA7BZz@Uw36fM^E# zco=;?r{0qGAeYh9T)!-#$1xvumj$jk5IJ9lGM>Nn@TTj7m*soxT)Y5mJU-ADSxgm- zW~KpTLnn1h<__E7r4Llb_i#=B%i&E_f`-&kor(Wi_(2`S5}@A2qFkj!yEGZ1VMRbG zWX^!0H`*a^Mbb9e;`mlT)fN`30I#I=~qdV>IyopwMviYEnkEys?mhF zg2jK~;KTH#ug20%9-g2LR<7roCG(wL3n|V4Tj{kuH{ir}*9x7l`>Xo4PH}h@scuEi z%YUjvrkw+Km%Scu(t~aXp1jS~x*X=Z68z1o7D^_pyu9=PC$thB=i9T>-KZOS zn>2^>I90IwlipC%(x-k=3DLIm?LXIso9-*_P*3=L+7U@Q)IR@@-L#S*0krQO$=~% z;L+SX^tl+X!^mciRJORDsjO{)y_0nLr!wfKN zP#tEOy{?FcLYc)?Io{5M3{a>rq6G9Sh2q7p@!>+4xt-=c)8~9EsZYcA$FpfnH}_R_qL0 zzK4BDtWBW7Lea^rB_pS}IANM+$^p&G{=s?EZ9xUchY5XLScf@HEUGiXBl*81-wy?J zCK3=@(w~AEMw2~$$aHtytVKC)35&kJ%&O~hJf41dk#@cBeBOEf{r7iQOHOwCcP*&x zIep-dKb14xFEd`5%%1GFZhSuf^%tbgLtDnD%Y-)U`POecMAB=s6S>j2(zL;=MuQ;~ z)q3)SvqkFSdV1I3<+DF&B92`rI}7?R?G*D+#3#N>3IXT7x0WxcZlxnJB!UAXBet%v zSDaSULDAF#76e5<@ol4cX9(14YlIx20vG+#$eX>F8WK){re4IsoAOFqxBYcz^jESu zWY?c9t#dni+RvzQncZFPLK#nOr(ZZ4b23PeRKuu|!J+&elslK|ex@B@Ds*}1=lau!Yc)_qN#KjK5;2)$$;Fh*lNEeXy- z91~*x3awkZ{-$lUE^z2Bb>eNZTe8*7-g3YF6XHAlI-3GDM6ZJ4V7dB8bH|Q+{j9rr z6Pw-p#wdvDvr5?`be^U;qX<*OV>^S}n{-V|Z)SbDTE-7EkBP&e(jj{vlwd z|9bZEY_;*zDY!N#%&=5b%6PrmD-JSLj6r*@ag=V{baNsI^J-#;kjE!1pY?cMAw(oy zJ$~6PQUm>%D^ymD8KhB7Vh6q6`!Yea=w2uEsMhQ^lJ~zzoEtGuK}+`J+Xs_7N0zBX zk%~p9kCDXK9){%zPZ#Kk8=KZjdAP%Epgyg+T^{|;ge8O6Q_gC7_uXKLs$YV*6a3_{ z_DLnrzzOwek#6#cm}E~A0(M}q>{w&10zt zdCP?)RKVPQhi2b!5^PKLtD4efE~t4yEJ+>P5cb00K@fa+DBXN`)Xvr688xnFvuX`XIT-^3nx^h=0?GwN3qn|5lqiTww3UX1+Gub-?Q54>66AU+u3-P`N14_& zNIt8rqGP8m0ez+URDj8tKPo{kpcJp5gdD;*-XYAC0Ksf0ddp^I35=fwimLE_Er{XkY4`OfmkP%3-Htym3IEs|QMfBzPC9nw1TJsRH z3C#ZX``0t1B#@)zKQvjQWP76cZ(qj`en{Jetu+wAF9-eHEYo>b;0@ z%LS665cKa8kvAz~u@Hv``fjYM{Ht}<6zK90!d%#XIHjARrChwIW`-0X2B_hZ*y$W7 zq}Oe!;V34v&M}eCH}AOzi-jbUTk6!m$g=mqkwJ@>3&EGsa{u|+m=xT!f>|*3p$o9lUC0FA=F&t zAA8bCI57Kl?KeZ}qd-7=Ja+dVF?d|0DuB>W$iH80tKuY3oRqRm1chinpUcZk1kLLZ zMq!{i%Ah@u77b$yW>Lh&SSkc9eS06UsGw2Z6Bw|SZ4xw=Nt;$J{t+5f_2re}hUr@4 z2NKMTFC;?tZwlyilD`WgS_O=!-s}?pA%JWOu?z3LUnv{%cN%^i;&*GGCw)Bn0qU>W z8Y@DkLFd$aM`2>jm2x?Kpsv2lTtzu$%)u45Ph=AY9(<;hbzAj^^RXhYvR0%z%TU%# z0`X99yQ^3x;CXRyRX&2 zHIt$muLU_H4~WgL=)&$tI^h29gQfIFqk6cXt}Ld3NE@A(5_)=q2s9F%H(z3`ZP*fy zMR=u87ejV64|})S%Xl+3FQh#+WR;;01IBd$S4-HD^ToQPFEUG6k|!$4O8Q^R7geFX zKKEp)ah$QpW!&yff&9JIV3-|+mq~DhN6jR+3a6YqR8W*@M7n4%5R$NlBSW;cXLFux zrHVCb8VO^m*hlKl0#_%dCHw{oQ)6MccYdHZ828r`7?BU-V-#ERzT-yh9`q;FyN&y^fkkY60V2evGeKZX*6{zmgSG+SyK$@5gnSLGybrtY$_Hz25D;@ zJ1%ojPN`xVsG#Qo~W9S2u5 z76nWPy#znTDwfpeXHq;xR~Bf$lP&qbsQ;oOTk;?LM<@TUSpJVr{$IQNU!DBFcKQEx z%KyJz*4N@LC;tzs3$(Sk|AYVBr9k@^{>-B^cH%xB?+sby|JZo2f(EEo&g{$o2M=ZP AOaK4? diff --git a/helios/pipeViewer/pipe_view/resources/resize_nesw.png b/helios/pipeViewer/pipe_view/resources/resize_nesw.png deleted file mode 100644 index 39f650eae92e6ca8a12cfc1e59137d86bac6da7d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmeAS@N?(olHy`uVBq!ia0vp@Ak4uAB#T}@sR2@KN#5=*KpF^sI`6IrQk(@Ik;M!Q ze1}1p@p%4<6rdn`iKnkC`+ZgxK@K6mU3yhOp@p6E_tf zX8NMRHN!G3foYa=63?mK({$}twZsU?PYe3-{^onjO@|oX^sf|UnJ`O}S;u6eiY{-+ zcH6LnFDr}HnYKx_&pZ6r`1GDLIWkq{uC9-Mzn-Ec_dM?!L&TTHC1MP-Ph0=|=z3v~ z-_LuNNzYc>xNC$IOD|d#8hmfVy}3{3E`AhHlo?sT|AYC9YW$yU*6sRk$I25zeD9|N P9n0Y9>gTe~DWM4fd52{b diff --git a/helios/pipeViewer/pipe_view/resources/resize_nwse.png b/helios/pipeViewer/pipe_view/resources/resize_nwse.png deleted file mode 100644 index 49897d5a817cae305fa5f4c0d45a371194dc61b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 272 zcmeAS@N?(olHy`uVBq!ia0vp@Ak4uAB#T}@sR2@KN#5=*KpF^sI`6IrQk(@Ik;M!Q ze1}1p@p%4<6rdn`iKnkC`+ZgxK@NEX?x5p9p}C$ejv*T7dnaw=J!Bx@`rci#$xKo; z=#l}Ws%av#+JdELrPer|*644#Va0BJV^ZqfpZ|BiuNQ0h8#(F8<1*j0g9rEzo>`+A zsxO$d+}6oF?v=+Ro(1#Mc%MfXZJ)!pWhNiz#~&5$o#(UN88*rNu*hk6_|L0=E8%|z z!~4c?hMK##cd2;R)kF)=p0xI_$!e4Gw`V$I_G+x37VswIv$*}V&B-1?GxOtt4rTCk L^>bP0l+XkKXI*2F diff --git a/helios/pipeViewer/pipe_view/resources/shape-align-bottom.png b/helios/pipeViewer/pipe_view/resources/shape-align-bottom.png deleted file mode 100644 index 2168d0debb2a4d2e3932e06c372080b465112dfd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 499 zcmV4u@H9v~MRz3Lp%ah5zyty| zgmhwPN<95n`7cqS-Iv?+xcv#Dbas#n^Fb053gVb1qOuxV%dw|cQ==&t!=3T z01>0>rRXuQos>HO0zen%rxZfB;g zRtZ`Z<`aktEFzREoP&Q=2xKN}{ALKOxXa2|L%^os%y&reQUaYpWX7o<^9gt%SWCbQ zfub-Zskav_&HbXdDPj~g#9tna|5YlvY)qC}-yM|~svA*xobsdh4yErzWwq?q3|F{E)2#}ww z$?Or*K4+@4D0cim86H;Oy@^s94`@Aa<7!HTcKLG!UC4roop%tMq))ZK5 zL5zT%a-OV83G%D=3xLgERN!kswXC||xptZSl?v3!8m~4^psau;z}2$G_EYUDDqs#g zz3rU}elQ0rW7_9H7N9;RN#O40;(vS#W}EFBoXS7)&kw8z00000NkvXXu0mjf+DpSG diff --git a/helios/pipeViewer/pipe_view/resources/shape-align-right.png b/helios/pipeViewer/pipe_view/resources/shape-align-right.png deleted file mode 100644 index 6d11c2afee7c0058fc399968b23f96b81ab2fc17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 469 zcmV;`0V@89P)`m1ThdT2Ny18E3SpPhaN>iyhPtX5N{%Q4fnlq;Yv|= zgD~L2uIFM>>7-KWWV(?SoWL+4^?9#e6aMLts+m;q{5s-0M>v;dT0f-22 zKQ|{?Bjjzt+U5B&QoWi0yCkF%YG8_Nl-NcqAW4pwix? z`zjl=xdnDt;N!DP&bHO-P#K`Nk0y|S>S`6RZJR*g;%xaV@5bMG3vM1>k@OG2HH)FO z`k1@xmHiel7r+Jq>SMABbdJF3$x)PmGytiDnRU3MA@iq~O@TnSb(Vhs@e#@b1$Tm0 zg!))1FiSvMU=i@QSR|{KHm*J{06u?Ff$t90vg&>hwM+6>Do`h@U9C-^tbixL)v{Xa zxpoy55QpdYub_e*?t#jfVhf63xjj%BlO=F_wVLG{VfMFRKlbAXLzzLHV?>+m00000 LNkvXXu0mjfF+t1Q diff --git a/helios/pipeViewer/pipe_view/resources/shape-align-top.png b/helios/pipeViewer/pipe_view/resources/shape-align-top.png deleted file mode 100644 index e56cf056a8b96fa14eba852b3b1a8c2a7e97eef2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 516 zcmV+f0{i`mP)TB4#B113m>Lz6Ob}=#Iocbm9{jm_WdW zkSc~=@)+*Yb8P3h6xxOQ6gii==jUfT3Gm;gNt5|}&bDol#%i^KeD}sCHjBlgkpK>M zAM;*vY4pO~!;+Bz2@IOhYnSJ<1^{mEUl=11A`t=+r55ws>oemEaB6^Z5FiQm3z<&# z0+bwjtt5vAK=0&uc3`581l~UbnAsB~ae8uu5W0xlUWX9z^!5dgAg~N1)E*%6eVif?vfX{-qryFt3L`4cZhcwR7uWdgC&Ihv;C>GYvvPXcg zWgrz=`ha9a)#m0WB}5s*Nz?oaU}N}x1qcHjkKXnRbCs_0A%asef_sxU{Aw- zynaM=8WJ%IW(LZ%n7KEXl1!n$st9XkE2m+N zn5$QTiLC+nZ3;{c)C1_8K0YtaO*n)dq(HZu*MegV?ByHr%zK~z|U?Ug@o(?A%;pVxnqG-*?{Sn`*E55feCcLo*~Iz5yX-us*13fBEXF&d|$EbMz#96x`5p57Qh6v>eii31=LnNE&w1mNW3 z?~GtZZ;lG6B764kEc>Sb#b}%gBnJU{1WH0l5XcYSoUR>3V*sn304fIPMiMeuJBmgC zR(&^~?cO7x5~@VM8$y-Y{_E4VqF4!F)rSIX_XD$A?{g(V0_jc%fe5I`4~phdtOT&? zH#Y_q=66s$#x79$s}asD6Rys>U*)E7m1&@D^SPuf|AK}GF=(P zQh-|Zo&{$>?O|jLki?)9qTdNYW%lUs+m%r)0Mw~R#lkWJ)v>!3L4+v5{Ndr}+9(zP z>eSB3c5K8UdT{`%W)EI`T^fbhi()iR z;|O1rNYq={T+{1z^ZNMimH@z$;~!M72g*tmmHA)OlDFSJy>q_Q!2DV74jgZB--aK7 q>w~#_=jIa;x%S_7TWz)V|JENAxv%#q)FfU20000|=aneq*k$K^81NYpV_dML2 z4`8ftt?(H=n;1ejq>ZUm~ez*VB& zl5{6Z3PM0KI2f5gx@!$!@`OUUq|j_J6M>NoRBM3?Q65RVY$;40S5N?$;Ac?;26auu z*xALO!NH*oWa&B{nJ?J$ATMT4>0~wkdXdvBnZ?S{j zvUl!~2F6l@CLX`~xM^`h#x|m+f4|GXNwn)Mj)*R zSI+q@J>iJZuLlu6Zio2sD}oWinBEYPWQ-#h0%_e}-*PGNNZ1uiq*-_q7;DNIwWW__Pz^XCvxEq96ik-Fm+1Lf{E#S1gUwrz~g%hyYtV zF@YC%=JF+w*3;`VzC>_D;5R~pFW*Dd8xcAt=TneHi_7m^19~LvvZb-NZy+Sr?m5K~ zNNeTxB|p+7#|ZpJ*vY&Ha{Vl?-kjN8y*oEgV%`XLMTV~_No? zgR0;n?)9SqQ0rV>9*p4cU*DKpKDq=dxJdu(nFgXL4AXl=hFi^!eAl*|6 z5SG1?S>q2y{AN6u-Sxhww_1s4!GA|0FI81BGn4lakvnHSd7dXOtHP_i#BLnBHWq<^ zG)<#FwonwsV*&sSA{GpGU~T~~&lWJc0|1DbX}do%L-r-Y^;iDyHtS>(0k@lQC5Ix_ zVg!>3_;_C_;+p*X=S-SrvK+9;^)Uz<*KkY$0CkE81kB-p&W*_gNI`27*OAZW0N^;_ z<$!!xmgOkTsO`Wl2XM!Mal2yEemx!t$g<4r-=AM6zTE4#?#R5P3&xy6)P1HzjJ`zn zn0o{sq?B4EH8xsnKHdSeHyC0+``7TE0}MZ!E(d~nuMoKo1X0G3d#?0$Kq5a>zpaf7 tV$&siZI2@ed$}g83GBY_tzED{@CPhH+l^H$_5lC@002ovPDHLkV1i~wrh)(f diff --git a/helios/pipeViewer/pipe_view/resources/shape-move-forward.png b/helios/pipeViewer/pipe_view/resources/shape-move-forward.png deleted file mode 100644 index b25dd61117e0e8288be11b6b71cce31d26a8914c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 412 zcmV;N0b~A&P)Nw@&~>3?cvoUV0Gm(!=XZd|zLlo0Ncq88Ev6 zu49e{-3HA0=FAE(i=K{;!A5ZkVL*5M{&k^(_znSkk~nTnaCeV#0=z8ZT<|g~W=>7u z9{AnDEGOVswl<(AR!snGD!d-Q+bdF#DkeP@UJf!%6V}wOt?&CrNI^aRu$ICW5{OoR z4V9~J*L54@!!X1J*o9n=AAn7PZ!!UU5#16{PF=Bpu=?wYQhE)u1nldmBoNFC7ZEe3 zYd{-ZqM{~XPX2NJ_BM(Hj#7&IDpCO9Wb3`o0+Ez&v4Gs4%RJ8jKyBNm+_L5ic(`@p zy#yQ*zP}=XnK#dWE})vgIF5EKePmMe)dEUeZq07H8T$cbQMM}tjMp>(0000`q3w3NX#l1o*Cv5@?dDnlek{SS0r>M0%<8%Z0II4gSa^Or6SFub;_eFuF)sfB z6zzL?tfKwFlt7&JPI%y4F5>}+%s$I(0G+dk``gwztEdAO+K2-Nx^I2{q?9sd6-Xo@ zy@N|f(Z^-sZf}->{eB;`RK{JWqrNc4fKadg^|??0K#yKuoeLStFHy_91!n?Suh-G+ ze0u!=M;S;{9wZ+w$!r3!7SasZlt%%l7_b)Ppmvy}T<3@cGJye!)JOTaN-}-G1fXdy zraaET@pwcm*=-5>KH#H#TqQD(j|>A-C|8nom9$!|0{`-Y%2t79z}uLtJ|G=9Q3dVg z@6A_ud890(Al1IH|C|Jx(nuLE4%K!iX07*qo IM6N<$f)0G#3;+NC diff --git a/helios/pipeViewer/pipe_view/resources/sub-space-down.png b/helios/pipeViewer/pipe_view/resources/sub-space-down.png deleted file mode 100644 index be9199878140386e861c1475e21b545e7f019676..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1001 zcmV-?z4L5+wzZH8cr9O=;3z3hDuI;|P_gR3$PwgbNZ9XHE?S$?G5v zaLExV*APWf6b?ld?I|ihqC%DUQ40b`mSqweTt@-NcACu3$Dvtk9j{$Cg%f$w=$oAx z&3p6ao7r992uC=={~hF4W@l%`S_=`ue-RJ1&CJXY07QHN%nTxewf4{v07N7-hK;Qg z;5g2|PSPqi2w>*A$>Q;N?KxO{3m?=LY*$L5R4TQ0fSJ=ol=24=5fX_+ZD81{n>i7T z#u#KWnS%>VHzXkd%+1Yt08+m4SS$wDbwNZeNt*R(t&z=UQ7)H*Ntm9Vp7ufo0KokG zyeA?x5z%P$flh=8vDPA+&7xAN)X0r7=~OD^g&xB&5&lH&IuW!zm&>7At=5rGO-*?R zxj0O~#KeT>I8K_G;W!SIQYaRSwM=hSLi0JvGoQD-h{CY2 zu+ZQ{Z*Ol?C#0ltV}Vn(A; zxUSof6UG>f+*m=^Q!aW&hQM0|&{_txdk?nr80_hD5CC_-z6s0p^5FX)e%Y1)09agH zY)H(Y6C#4AZ(K*>SQmQEoJ922H-J)r))1utr2(a2`-fmpzktH^?=bMe$4r25I|9bX z$2}rSJB|ZphSnOJo13-%KQ}hey}gZ|k)fvKPztnq8+0d+?o-bqKl#otO_Q*h2xd+* zGnl!Sn1w>sm*U*u863}X-?j~#p$z;-7T3UjLq=*PwYlsLE z#k=Ty@htd{p8>5wmp%#ITl4m-fYK0SAxh(+90-aC9#LjSv|0r@47#^-f91`f_MNe& zHZF3QkBG6cF>iT!86uJbfZd3bSH-QtSiBolF2lYzQD-~$w?p=~2gX9QI-CPR5zIUp zi9{wNrLBuw-~R;Ff2v{68^}AmVU2z?2Yifp!Yj9laHjFaLu132uYPv)FA&5( X+|em74AqAk00000NkvXXu0mjf0EflK diff --git a/helios/pipeViewer/pipe_view/resources/sub-space-left.png b/helios/pipeViewer/pipe_view/resources/sub-space-left.png deleted file mode 100644 index 4f477d2785bc2657eec0defbd9274386dd796d32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1011 zcmV`oQ&|}P&TV^b!J^2$B}E4Vil{MajLsI9YFwC@Sxq97kU$IzVlb%^LoZP` z8kWR0A=joMWnp4QH|UI-7{!^4?~KG?K~dCHc@*&6^B)&-udTF5Ep?;+s*ASY<9{CD zM-R4?f7a8}V-Qi8h#-Uj5uvfMaX}1aDZ!cBQFP1l?WjmMcI_67U4EUO(PUS7Un+g0BC2c?3;7u4aa2!MH9u*Rn_F&GDZ;7}N=3LZCx7JUzxrkO?}b?|&i zV)MW?6bCev)K)`DjDd|j0h^rx6|Dem+yM#T_4NUe!s;qdoc?>!2T6B#_dF88V9+5! zM5zAnHdZZPhLTO|;JJST;2f|lNX`Ln0i1(YSA#Zg$Jk%Du;%0$g#mI=bD+My-XNk5 zTWKViERrlzJaG&F>okr5Q_ z+ydprL%_1YLgzB(Ry6GaxCJQ)l3U-l4V#;r=Q*KFOn|u*u$k#k%CiNZbB2WTJ@>cV z11&8rMrUUy?EUc8<%0eu0oCgVo16r-{hnq!=F%bO(gQ(|7SHb@YHe+G3UgqkhQz;j z!B^MJ%lSO;B0or2jstn!grQK#5JGe$R4=UYNyym`5c)6K!8wPu_cy%!^9sPN{Q%&~ z_bTr1@4s4CSI7SMuh_lj=1ur_{R()Cp^QFD2UoqY4jjV!yALt*>h<&W=VOPL^kvlE z-fpa#91rh%_5`InwxYPIGJR1I;pL?(5YyA68)6qL3i32-Yilz|i12~IM=0g98Qy{H h;D!sA`tUPS;v4P)h!#3TatmKFCE6BZwBQ3tB|cD%=IpGB;DXm>H2)?E@EW zOiaXC1Y#}1q@pKTp+y=^bHfKBmD5dUJg*ir_j2czb8V_c9T?`^!{L7a@Ao~JArHxrU)}9oQhZz0jrAWDMP=~TtqX1(7=TMA+#N%*%`T$v55AAR( z=0~57Du5C|6-A&>BQibzr0eh*2G%#%!#(~m!#EgEJu?RKegb4xhp%x5^otkX7F`8} zN>B&^UDr`JtD~Yu1N}N*(758MIlX@e71dh+&YFt~K_LQCO2lF@B$G+hE`CL2iyw66 z4Zt}_;9A~~O>yohz&MnUP>egz0aJt!!ZJ=NMR_6t;-Vml6AS4uLaGV2-F-%QO!7vPayA6An zzMc3o_8P3VF(>9Z#w&f0f?EU}9mnx_Jj1vUA{+<=LQU6ij`6qwm0W?)wihV7e7(ed#gTzil2odh+=&%kN&Xcv%x9&jH)I&7v%{A`vf}cK%#h2p{#`o!6HwK1^ zx*4^%w=45jWWODCc6R2Ra#JUd9a4lo+SIxS>-=?@j!22Q+anOm%d<^`Lp4QrLeq1z z3IK$=y1Mcnv5k@8#|ohDR`|r{CogvA-UG?0rdzio_&;pp2QxH}TbAuXGXMYp07*qo IM6N<$f+NkXM*si- diff --git a/helios/pipeViewer/pipe_view/resources/sub-space-up.png b/helios/pipeViewer/pipe_view/resources/sub-space-up.png deleted file mode 100644 index 4da10091e2bbbb5ffddb3702991c6e390d14f0e3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1002 zcmV+cNsOi7i-dPGa(YJiE|yIA=+0mee~R2v5#O z&hP&`|MPwX*pL0#j~XJGOb#WJ$)Q@Fp;i0Csi~=`Qffp9F%pl*qt!l#)c_wRA}FP> zlYfXrBGEk;kVqt=MD$;{C8EE-Ke}fYBoc|JQfgQ!1t}$5E*CUSgL$l!8Xg-P+ZzG# zc-(?3rQmkE(cIjOU@!=m%jFO;Iyzdhh`LJQL^N!++JrNot>R*etv$T;$b*BIayS> zuInW$FO{W~@I6W+Bnqe}!dn1oiEEL|R|elJz=Yd94+ev~R&Hh@Li@@C1hN_U4(afP zf?)1Ch)o0Uo;-&taA=GjJ#;zxw4{KEiHRrx3tnmE=J!vRZX*zE#)0E)(AL+$UOooP zZ$bH+piXpv0KC4l1f{VrTd(|h3jkbpxV>__=XOXbFcG1Bc?tf5O=vvU3fKKbfO9|y z5a(c((xJUw+Z7JZ zq1~Q^ulFK0mLDE>SRkcDnFZ#(`A8-M|H&}4zteyaU_)0+4lg;_4R8UH3d98*0?Jia zNv_Xtfz@eXTe)q|%S|qPr!z`!XS;xbfdN+SrsCVd{#*Rb`WcV65iFB|`aa@lJ9g}# z9eW@ZNbuTrMD`Foe`)>hGI-0!MV=SI{f$skSRMEv0>amSxhpm^pnL&RKU^NpImG!s ztp0iv;Nt5KCIA40$hW+@F$)=LgFJSs81C^x^nHhodk-Kp#;f+96Vp}i3CqRf-t%7& z(|$kLaT0-}t&VG?6joC=A#=HZ!c)_snl`1o1p#d3n{$6MhEIwY9(EyoBbD6$Lt~GB Y0m{HOUD*lzk^lez07*qoM6N<$f-$zgI{*Lx diff --git a/helios/pipeViewer/pipe_view/transactiondb/.gitignore b/helios/pipeViewer/pipe_view/transactiondb/.gitignore deleted file mode 100644 index 2f7d835b9a..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/transactiondb.cpp diff --git a/helios/pipeViewer/pipe_view/transactiondb/__init__.pyi b/helios/pipeViewer/pipe_view/transactiondb/__init__.pyi deleted file mode 100644 index 916e1f10fa..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/__init__.pyi +++ /dev/null @@ -1,67 +0,0 @@ -# Stub definitions for the transactiondb library -# Cython isn't very good at generating these automatically yet - -from typing import Callable, ClassVar, List, Optional, Union - -class Transaction: - ANNOTATION_TYPE_STR: ClassVar[str] = ... - CONTINUE_FLAG: ClassVar[int] = ... - FLAGS_MASK_TYPE: ClassVar[int] = ... - INSTRUCTION_TYPE_STR: ClassVar[str] = ... - MEMORY_OP_TYPE_STR: ClassVar[str] = ... - PAIR_TYPE_STR: ClassVar[str] = ... - def __init__(self, trans_ptr: int, is_proxy: bool) -> None: ... - def contains(self, hc: int) -> bool: ... - def containsInterval(self, l: int, r: int) -> bool: ... - def getAnnotation(self) -> Optional[str]: ... - def getAnnotationLength(self) -> Optional[int]: ... - def getComponentID(self) -> Optional[int]: ... - def getDisplayID(self) -> Optional[int]: ... - def getFlags(self) -> Optional[int]: ... - def getLeft(self) -> int: ... - def getLocationID(self) -> Optional[int]: ... - def getOpcode(self) -> Optional[int]: ... - def getPairID(self) -> Optional[int]: ... - def getParentTransactionID(self) -> Optional[int]: ... - def getRealAddress(self) -> Optional[int]: ... - def getRight(self) -> int: ... - def getTransactionID(self) -> Optional[int]: ... - def getType(self) -> int: ... - def getTypeString(self) -> str: ... - def getVirtualAddress(self) -> Optional[int]: ... - def isContinued(self) -> bool: ... - def isProxy(self) -> bool: ... - def isValid(self) -> bool: ... - def makeRealCopy(self) -> Transaction: ... - -class TransactionDatabase: - OBJECT_DESTROYED_ERROR: ClassVar[str] = ... - def __init__(self, filename: str, num_locs: int, update_enabled: bool) -> None: ... - def ackUpdate(self) -> None: ... - def clearCurrentTickContent(self) -> None: ... - def disableUpdate(self) -> None: ... - def enableUpdate(self) -> None: ... - def forceUpdate(self) -> None: ... - def getChunkSize(self) -> int: ... - def getDisplayID(self) -> Optional[int]: ... - def getFileEnd(self) -> int: ... - def getFileInclusiveEnd(self) -> int: ... - def getFileStart(self) -> int: ... - def getFileVersion(self) -> int: ... - def getLocationMap(self) -> List[Union[int, str]]: ... - def getNodeDump(self, node_idx: int, loc_start: int = 0, loc_end: int = 0, tick_entry_limit: int = 0) -> str: ... - def getNodeLength(self) -> int: ... - def getNodeStates(self) -> str: ... - def getNumCachedAnnotations(self) -> int: ... - def getPairID(self, loc: int) -> Optional[int]: ... - def getSizeInBytes(self) -> int: ... - def getTransactionAnnotation(self, loc: int) -> Optional[str]: ... - def getTransactionID(self, loc: int) -> Optional[int]: ... - def getTransactionProxy(self, loc: int) -> Transaction: ... - def getVerbose(self) -> bool: ... - def getWindowLeft(self) -> int: ... - def getWindowRight(self) -> int: ... - def isUpdateReady(self) -> bool: ... - def query(self, start_inc: int, end_inc: int, callback: Callable, modify_tracking: bool = True) -> None: ... - def setVerbose(self, verbose: bool) -> None: ... - def unload(self) -> None: ... diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/PipelineDataCallback.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/PipelineDataCallback.hpp deleted file mode 100644 index 0aabb9b666..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/PipelineDataCallback.hpp +++ /dev/null @@ -1,33 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/utils/SpartaException.hpp" - -namespace sparta::pipeViewer { - /** - * \class PipelineDataCallback - * \brief An abstract class that recieves transactions as they are - * read from disk. - */ - class PipelineDataCallback - { - public: - virtual ~PipelineDataCallback() {} - - virtual void foundTransactionRecord(const transaction_t*) - { - throw sparta::SpartaException("Read transaction with unknown transaction type"); - } - virtual void foundInstRecord(const instruction_t*) = 0; - virtual void foundMemRecord(const memoryoperation_t*) = 0; - virtual void foundAnnotationRecord(const annotation_t*) = 0; - //! Add a virtual method for the case we find a Pair Transaction Record. - // This method is called back in TransactionDatabaseInterval to build the - // TransactionInterval to be used by pipeViewer. - virtual void foundPairRecord(const pair_t*) = 0; - }; -}//NAMESPACE:sparta::pipeViewer diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/Reader.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/Reader.hpp deleted file mode 100644 index 5cebad0484..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/Reader.hpp +++ /dev/null @@ -1,1216 +0,0 @@ -// -*- C++ -*- -/** - * \file Reader - * - * \brief Reads transctions using the record and index file - * - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "PipelineDataCallback.hpp" -#include "sparta/utils/SpartaException.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/LexicalCast.hpp" -#include "sparta/pipeViewer/Outputter.hpp" - -// #define READER_DBG 1 -// #define READER_LOG 1 - -#define _LOG_MSG(strm, x) strm << "READER: " << x << std::endl - -#if READER_DBG == 1 -#define READER_LOG 1 -#define READER_DBG_MSG(x) _LOG_MSG(std::cerr, x) -#else -#define READER_DBG_MSG(x) -#endif - -#if READER_LOG == 1 -#define READER_LOG_MSG(x) _LOG_MSG(std::cout, x) -#else -#define READER_LOG_MSG(x) -#endif - -namespace sparta::pipeViewer { - /*! - * \brief Sanity checker for records. Used when doing a dump of the index - * file to check that all transactions in the heartbeat actually belong - * there - */ - class RecordChecker : public PipelineDataCallback - { - private: - uint64_t start; - uint64_t end; - - public: - RecordChecker(const uint64_t _start, const uint64_t _end) : - start(_start), - end(_end) - {;} - - virtual void foundTransactionRecord(const transaction_t* r) override { - if (r->time_Start < start || - r->time_End > end) { - std::cout << "Bounds on transactions were outside of heartbeat range " << start - << ", " << end << ". transaction:" - << " idx: " << r->transaction_ID - << " disp: " << r->display_ID - << " loc: " << r->location_ID - << " start: " << r->time_Start - << " end: " << r->time_End - << " parent: " << r->parent_ID << std::endl; - } - } - - virtual void foundInstRecord(const instruction_t* r) override { - foundTransactionRecord(r); - } - - virtual void foundMemRecord(const memoryoperation_t* r) override { - foundTransactionRecord(r); - } - - virtual void foundAnnotationRecord(const annotation_t* r) override { - foundTransactionRecord(r); - } - - virtual void foundPairRecord(const pair_t* r) override { - foundTransactionRecord(r); - } - }; - - /*! - * \class Reader - * @ brief A class that facilitates reading transactions from disk that - * end in a given interval measured in cycles. - * - * The Reader will return the records found on disk by calling - * methods in PipelineDataCallback, passing pointers to the read - * transactions. - */ - class Reader - { - private: - class FileStream { - private: - const std::string filename_; - const std::ios_base::openmode mode_; - - protected: - std::ifstream fstream_; - - public: - FileStream(std::string&& filename, const std::ios_base::openmode mode) : - filename_(std::move(filename)), - mode_(mode), - fstream_(filename_, mode) - { - //Make sure the file opened correctly! - sparta_assert(fstream_.is_open(), - "Failed to open file " << filename_); - sparta_assert(fstream_.peek() != std::ifstream::traits_type::eof(), - filename_ << " is empty. Did Argos database collection complete?"); - } - - FileStream(FileStream&&) = default; - - ~FileStream() { - fstream_.close(); - } - - inline const auto& getFilename() const { - return filename_; - } - - inline bool read(char* const buf, const size_t num_bytes) { - return !!fstream_.read(buf, num_bytes); - } - - template - inline bool read(T* const buf, const size_t num_bytes) { - return read(reinterpret_cast(buf), num_bytes); - } - - template - inline bool read(T* const buf) { - return read(buf, sizeof(T)); - } - - template - inline bool read(T& buf) { - return read(&buf); - } - - inline bool seekg(const std::ifstream::pos_type pos) { - return !!fstream_.seekg(pos); - } - - inline bool seekg(const std::ifstream::off_type pos, const std::ios::seekdir dir) { - return !!fstream_.seekg(pos, dir); - } - - inline auto tellg() { - return fstream_.tellg(); - } - - inline auto gcount() const { - return fstream_.gcount(); - } - - inline bool good() const { - return fstream_.good(); - } - - inline void reopen() { - std::fstream::pos_type cur_pos = fstream_.tellg(); - fstream_.close(); - fstream_.clear(); - fstream_.open(filename_, mode_); - fstream_.seekg(cur_pos); - } - - inline void clear() { - fstream_.clear(); - } - - inline int64_t size() const { - struct stat stat_result; - stat(filename_.c_str(), &stat_result); - return stat_result.st_size; - } - - inline operator bool() const { - return !!fstream_; - } - }; - - /** - * \class ColonDelimitedFile - * \brief Class that knows how to read ':'-delimited files used by the Argos pair format - */ - class ColonDelimitedFile : public FileStream { - /** - * \struct CustomSpaceCType - * \brief Subclass of std::ctype that allows overriding which characters should be - * considered whitespace - */ - template - struct CustomSpaceCType : std::ctype { - private: - using CTypeArray = std::array; - - /** - * \brief Generates a constexpr character table with the characters in SpaceChars set as whitespace - */ - static inline constexpr CTypeArray genSpaceTable_() { - CTypeArray space_table{}; - // First set every character to std::ctype_base::mask - treating them as non-whitespace - for(size_t i = 0; i < table_size; ++i) { - space_table[i] = std::ctype_base::mask(); - } - - // Lambda that sets a character to whitespace - auto add_space_char = [&](const char c) { space_table[c] = std::ctype_base::space; }; - // Apply the lambda over all of the characters in SpaceChars - (add_space_char(SpaceChars), ...); - return space_table; - } - - public: - CustomSpaceCType() : - std::ctype(getTable()) - { - } - - static inline std::ctype_base::mask const* getTable() { - // Init the table - static constexpr CTypeArray space_table = genSpaceTable_(); - // Return a pointer to the table - return space_table.data(); - } - }; - - // The FileLineCType only treats newlines as whitespace, causing the >> operator to grab an entire line at a time - using FileLineCType = CustomSpaceCType<'\n'>; - - public: - /** - * \class LineStream - * \brief Subclass of std::istringstream with some useful specializations for the >> operator. This class is used to - * automatically tokenize each line in a ':'-delimited file using the >> operator. - */ - class LineStream : public std::istringstream { - private: - // The PairFormatCType treats ':' as whitespace, causing the >> operator to tokenize on - // any ':' in its internal string buffer - using PairFormatCType = CustomSpaceCType<':'>; - - /** - * \brief Reads an aribitrary tuple from the stream - */ - template - inline LineStream& readTuple_(std::tuple& val) { - // We reached the end of the tuple, so no more work to be done - if constexpr(idx >= sizeof...(Args)) { - return *this; - } - else { - // Grab the current index and then recursively grab the next one - // (this will get unrolled by the compiler) - *this >> std::get(val); - return readTuple_(val); - } - } - - /** - * \brief Sets the locale to a PairFormatCType - */ - inline void setLocale_() { - imbue(std::locale(std::locale(), new PairFormatCType())); - } - - public: - LineStream() : - std::istringstream() - { - setLocale_(); - } - - explicit LineStream(const std::string& str) : - std::istringstream(str) - { - setLocale_(); - } - - using std::istringstream::operator>>; - - /** - * \brief >> operator specialization for enums - */ - template - inline std::enable_if_t, LineStream&> operator>>(T& val) { - // Read into a temporary variable of the underlying type of the enum - std::underlying_type_t new_val; - *this >> new_val; - // Then cast it to the enum type - val = static_cast(new_val); - return *this; - } - - /** - * \brief >> operator specialization for std::pair - */ - template - inline LineStream& operator>>(std::pair& val) { - *this >> val.first >> val.second; - return *this; - } - - /** - * \brief >> operator specialization for std::tuple - */ - template - inline LineStream& operator>>(std::tuple& val) { - return readTuple_<0>(val); - } - - /** - * \brief >> operator specialization for std::unordered_map - * Reads in the key and value as an std::pair and then does a move-emplace into the map - * Note that this only adds 1 element at a time rather than attempting to read an entire map at once - */ - template, - typename KeyEqual = std::equal_to, - typename Allocator = std::allocator>> - inline LineStream& operator>>(std::unordered_map& map) { - std::pair new_pair; - *this >> new_pair; - map.emplace(std::move(new_pair)); - return *this; - } - - /** - * \brief >> operator specialization for std::vector - * Reads in the value then does a move-emplace into the vector - * Note that this only adds 1 element at a time rather than attempting to read an entire vector at once - */ - template> - inline LineStream& operator>>(std::vector& val) { - T new_val; - *this >> new_val; - val.emplace_back(std::move(new_val)); - return *this; - } - }; - - explicit ColonDelimitedFile(std::string&& filename, std::ios_base::openmode mode = std::ios_base::in) : - FileStream(std::move(filename), mode) - { - fstream_.imbue(std::locale(std::locale(), new FileLineCType)); - } - - /** - * \brief Iterates over the entire file line by line, tokenizing the line with a LineStream and then - * calling a callback function for further processing - */ - template - inline void processWith(CallbackFunc&& func) { - LineStream strm; - for(auto it = std::istream_iterator(fstream_); it != std::istream_iterator(); ++it) { - strm.str(*it); - strm.clear(); - func(strm); - } - } - - /** - * \brief Simplified form of processWith that reads an entire file into a single object - */ - template - inline void processInto(T& obj) { - processWith([&](LineStream& strm) { strm >> obj; }); - } - }; - - // Memory Layout of pair_struct. We build a map of such structs before we start reading record back from the database. - // This structs help us by telling us exactly how may pair values - // there are in the current pair type, what their names are and how much bytes each of the values occupy. - struct pair_struct{ - uint16_t length; - std::vector names; - std::vector sizes; - std::vector types; - PairFormatterVector formats; - - explicit pair_struct(ColonDelimitedFile::LineStream& strm) : - length(0), - names(1, "pairid"), - sizes(1, sizeof(uint16_t)), - types(1, 0), - formats(1, PairFormatter::DECIMAL) - { - strm >> length; - ++length; // Increment length to account for the default pairid field - - names.reserve(length); - sizes.reserve(length); - types.reserve(length); - - // Iterate through the rest of the delimiters in the stringline - // and build up the Name Strings and the Sizeof values for every field. - // Formats are read later from a different file - while(!strm.eof()) { - strm >> names >> sizes >> types; - } - - formats.reserve(length); - } - }; - - /** - * \brief return the correct position in the record file that corresponds - * to the start cycle. - * \param start The cycle number to start reading records from. - */ - inline uint64_t findRecordReadPos_(const uint64_t start) - { - //Figure out how far we need to seek into our index file. - const uint64_t step = first_index_ + (start/heartbeat_ * sizeof(uint64_t)); - - //Now read the pointer from this location. - if(!index_file_.seekg(step)) { - sparta_assert(!"Could not seekg in for the given position. Please report bug"); - } - uint64_t pos = size_of_record_file_; - - //It might be the case that our index file is too small - //to represent an end time that the user is requesting. - //in the above algorythm either reached the end of the index file. - - //notice we look too see if the index file seeker has passed - //size_of_index_file_ - 8 bytes b/c a special last index is written to the index file - //to point to only the start of the last transaction. - const int64_t filepos = index_file_.tellg(); - if(filepos >= size_of_index_file_ - 8 || filepos == -1) - { - //We need to reset end of file reach flags for the index file. - index_file_.clear(); - } - else - { - index_file_.read(pos); - } - return pos; - } - - /** - * \brief A helper method to round values up to the - * interval values. - * example 4600 rounds to 5000 when the interval is "1000" - */ - inline uint64_t roundUp_(const uint64_t num) - { - const auto sub_sum = num + heartbeat_ - 1; - return sub_sum - (sub_sum % heartbeat_); - } - - inline std::string readAnnotation_(const uint16_t length) { - std::string annt_buf(length, '\0'); - record_file_.read(annt_buf.data(), length); - return annt_buf; - } - - inline void acquireLock_() { - sparta_assert(!lock_, - "This reader class is not thread safe, " - "and this method cannot be called from multiple threads."); - lock_ = true; - } - - /** - * \brief Return the start time in the file. - */ - inline uint64_t findCycleFirst_() - { - //Make sure the user is not abusing our NON thread safe method. - acquireLock_(); - record_file_.seekg(0, std::ios::beg); - transaction_t transaction; - record_file_.read(transaction); - clearLock(); - return transaction.time_Start; - } - - /** - * \brief Return the last end time in the file. - * Our output saved the last index to point to - * the start of last record. - */ - inline uint64_t findCycleLast_() - { - //Make sure the user is not abusing our NON thread safe method. - acquireLock_(); - //Notice that we reset the index file, b/c if - //we had already read to the end of the file, we need to reset - //end of file flags. - index_file_.clear(); - //seek one entry back from the end of the index file - index_file_.seekg(size_of_index_file_-sizeof(uint64_t)); - uint64_t pos = 0; - index_file_.read(pos); - //read the transaction at the appropriate location. - record_file_.seekg(pos, std::ios::beg); - transaction_t transaction; - record_file_.read(transaction); - clearLock(); - if(record_file_.gcount() != sizeof(transaction_t)) - { - return highest_cycle_; - } - return transaction.time_End - 1; - - } - - template - inline size_t readRecordVersion_(const std::fstream::pos_type end_pos, - const uint64_t start, - const uint64_t end) - { - size_t recsread = 0; - for(auto pos = record_file_.tellg(); pos < end_pos && pos != -1; pos = record_file_.tellg()) - { - // Read, checking for chunk_end - readRecord_(start, end); - if constexpr(CountRecords) { - ++recsread; - } - } - return recsread; - } - - /*! - * \brief Read a record of any format. Older formats are upconverted to new format. - */ - template - inline size_t readRecords_(const std::fstream::pos_type end_pos, const uint64_t start, const uint64_t end) { - sparta_assert(version_ == 2, "Only version 2 is currenly supported"); - return readRecordVersion_(end_pos, start, end); - } - - /** - * \brief Read a single record at \a pos and increment pos - */ - inline void readRecord_(const uint64_t start, const uint64_t end) { - transaction_t transaction; - record_file_.read(transaction); - sparta_assert(record_file_.good(), "Previous read of the argos DB failed"); - - switch (transaction.flags & TYPE_MASK) - { - case is_Annotation : - { - annotation_t annot(std::move(transaction)); - record_file_.read(annot.length); - annot.annt = readAnnotation_(annot.length); - - //// Sanity check the transactions coming out - //if(start % heartbeat_ == 0 // Only try this sanity checking if start is a multiple of heartbeat_ - // && transaction.time_Start < start // Got a transaction starting before the first heartbeat containing this query - // && transaction.time_End > transaction.time_Start){ // ignore 0-length transactions - // std::cout << "Found a transaction where start < query-start && end <= query-start: (" - // << transaction.time_Start << " < " << start << ")" - // << " && " << transaction.time_End << "<= " << start << ") : \"" - // << annot.annt << "\"" - // << std::endl; - //} - - // Only send along transaction in the query range - if(transaction.time_End < start || transaction.time_Start > end){ - // Skip transactions by not sending them along to the callback. - // read is faster than seekg aparently. This DOES help performance - READER_DBG_MSG("skipped transaction outside of window [" << start << ", " - << end << "). start: " << transaction.time_Start << " end: " - << transaction.time_End - << " parent: " << transaction.parent_ID); - }else{ - READER_DBG_MSG("found annt. " << "loc: " << annot.location_ID << " start: " - << annot.time_Start << " end: " << annot.time_End - << " parent: " << annot.parent_ID); - data_callback_->foundAnnotationRecord(&annot); - } - } break; - - case is_Instruction: - { - instruction_t inst; - record_file_.seekg(-sizeof(transaction_t), std::ios::cur); - record_file_.read(inst); - - READER_DBG_MSG("found inst. start: " << inst.time_Start << " end: " << inst.time_End); - - data_callback_->foundInstRecord(&inst); - } break; - - case is_MemoryOperation: - { - memoryoperation_t memop; - record_file_.seekg(-sizeof(transaction_t), std::ios::cur); - record_file_.read(memop); - - READER_DBG_MSG("found inst. start: " << memop.time_Start << " end: " << memop.time_End); - - data_callback_->foundMemRecord(&memop); - } break; - - // If we have found a record which is of Pair Type, - // then we enter into this switch case - // which contains the logic to read back records of - // pair type using record file, map file - // and In-memory data structures and rebuild the pair - // record one by one. - case is_Pair : { - pair_t pairt(std::move(transaction)); - - // The loc_map is an In-memory Map which contains a mapping - // of Location ID to Pair ID. - // We are going to use this map to lookup the Location ID we - // just read from this current record - // and find out the pair id of such a record - const auto unique_id = loc_map_.at(transaction.location_ID); - - // We are now going to use the In-memory data structure we built - // during the Reader construction. - // This Data Structure contains information about the name strings - // and their sizeof data - // for every different type of pair we have collected. - // So, we retrieve records from this structure one by one, - // till we find that the pair id of the record we just read - // from the record file - // matches the pair id of the current record we retrieved from - // the In-memory data Structure. - const auto& st = map_.at(unique_id); - - // We lookup the length, the name strings and the sizeofs of - // every anme string from the retrieved - // record of the Data Struture and copy the values into out - // live Pair Transaction record - pairt.length = st.length; - pairt.nameVector = st.names; - pairt.sizeOfVector = st.sizes; - pairt.delimVector = st.formats; - - pairt.valueVector.reserve(pairt.length); - pairt.valueVector.emplace_back(std::make_pair(unique_id, false)); - - pairt.stringVector.reserve(pairt.length); - pairt.stringVector.emplace_back(std::to_string(unique_id)); - - for(std::size_t i = 1; i != st.length; ++i){ - if(st.types[i] == 0) { - // Type 0 = integer - const auto item_size = pairt.sizeOfVector[i]; - sparta_assert(item_size <= sizeof(pair_t::IntT), - "Data Type not supported for reading/writing."); - pair_t::IntT tmp = 0; - record_file_.read(&tmp, item_size); - pairt.valueVector.emplace_back(std::make_pair(tmp, true)); - - // Finally for a certain field "i", we check if there is a string - // representation for the integral value, by checking, - // if a key with current pair id, current field id, current integral value - // exists in the In-memory String Map. If yes, we grab the value - // and place it in the enum vector. - if(const auto it = stringMap_.find(std::make_tuple(pairt.valueVector[0].first, - i-1, // string map doesn't include the UID field, so index 0 == field index 1 - pairt.valueVector[i].first)); - it != stringMap_.end()) { - pairt.stringVector.emplace_back(it->second); - pairt.valueVector[i].second = false; - } - - // Else, we convert integer value to string - else { - const auto int_value = pairt.valueVector[i].first; - - if (int_value == std::numeric_limits::max()) { - // Max value, so probably bad...push empty string - pairt.stringVector.emplace_back(""); - } else { - const auto& format_str = pairt.delimVector[i]; - - std::ios_base::fmtflags format_flags = std::ios::dec; - std::string_view fmt_prefix = ""; - - if(format_str == PairFormatter::HEX) { - format_flags = std::ios::hex; - fmt_prefix = "0x"; - } - else if(format_str == PairFormatter::OCTAL) { - format_flags = std::ios::oct; - fmt_prefix = "0"; - } - - std::ostringstream int_str; - int_str << fmt_prefix; - int_str.setf(format_flags, std::ios::basefield); - int_str << int_value; - pairt.stringVector.emplace_back(int_str.str()); - } - } - } - else if(st.types[i] == 1){ - // Type 1 = string - uint16_t annotationLength; - record_file_.read(annotationLength); - - std::string annot_str(annotationLength, '\0'); - record_file_.read(annot_str.data(), annotationLength); - pairt.stringVector.emplace_back(std::move(annot_str)); - - // This bool value describes if this field has a string-only value. - // String only values are those values which are stored in database as - // strings and has no integral representation of itself. - const bool string_only_field = true; - pairt.valueVector.emplace_back( - std::make_pair( - std::numeric_limits::max(), - string_only_field - ) - ); - } else { - pairt.stringVector.emplace_back("none"); - pairt.valueVector.emplace_back(std::make_pair(0, false)); - } - } - - READER_DBG_MSG("found pair. start: " << pairt.time_Start << " end: " << pairt.time_End); - - data_callback_->foundPairRecord(&pairt); - } break; - - default: - { - throw sparta::SpartaException("Unknown Transaction Found. Data might be corrupt."); - } - } - } - - inline void checkIndexUpdates_() - { - const auto index_size = index_file_.size(); - const auto record_size = record_file_.size(); - - if(index_size != size_of_index_file_ && record_size != size_of_record_file_) - { - const int64_t record_remainder = record_size % heartbeat_; - - if(record_size - record_remainder == size_of_record_file_) - { - return; - } - - record_file_.reopen(); - index_file_.reopen(); - map_file_.reopen(); - data_file_.reopen(); - - size_of_index_file_ = index_size; - - if(record_remainder != 0) - { - size_of_record_file_ = record_size - record_remainder; - } - else - { - size_of_record_file_ = record_size; - } - - highest_cycle_ = findCycleLast_(); - - file_updated_ = true; - } - } - public: - /** - * \brief Construct a Reader - * \param filename the name of the record file - * \param cd a pointer to for the PipelineDataCallback to use. - */ - Reader(std::string filepath, std::unique_ptr&& data_callback) : - filepath_(std::move(filepath)), - record_file_(filepath_ + "record.bin", std::fstream::in | std::fstream::binary ), - index_file_(filepath_ + "index.bin", std::fstream::in | std::fstream::binary), - map_file_(filepath_ + "map.dat", std::fstream::in), - data_file_(filepath_ + "data.dat", std::fstream::in), - string_file_(filepath_ + "string_map.dat", std::fstream::in), - display_file_(filepath_ + "display_format.dat", std::fstream::in), - data_callback_(std::move(data_callback)), - size_of_index_file_(0), - size_of_record_file_(0), - lowest_cycle_(0), - highest_cycle_(0), - lock_(false), - file_updated_(false), - loc_map_(), - map_(), - stringMap_() - { - sparta_assert(data_callback_); - - READER_LOG_MSG("pipeViewer reader opened: " << record_file_.getFilename()); - - // Read header from index file - char header_buf[HEADER_SIZE]; - index_file_.read(header_buf, HEADER_SIZE); - // Assuming older version until header proves otherwise - version_ = 1; - if(static_cast(index_file_.gcount()) != HEADER_SIZE) { - // Assume old version because the file is too small to have a header - index_file_.clear(); // Clear error flags after failing to read enough data - index_file_.seekg(0, std::ios::beg); // Restore - } - else if(HEADER_PREFIX.compare(0, - HEADER_PREFIX.size(), - header_buf, - HEADER_PREFIX.size())) { - // Header prefix did not match. Assume old version - index_file_.seekg(0, std::ios::beg); // Restore - } - else { - // Header prefix matched. Read version - header_buf[HEADER_SIZE - 1] = '\0'; // Insert null - version_ = sparta::lexicalCast(header_buf + HEADER_PREFIX.size()); - } - sparta_assert(version_ > 0 && version_ <= Outputter::FILE_VERSION, - "pipeout file " << filepath_ << " determined to be format " - << version_ << " which is not known by this version of SPARTA. Version " - "expected to be in range [1, " << Outputter::FILE_VERSION << "]"); - sparta_assert(index_file_.good(), - "Finished reading index file header for " << filepath_ << " but " - "ended up with non-good file handle somehow. This is a bug in the " - "header-reading logic"); - - // Read the heartbeat size from our index file. - // This will be the first integer in the file except for the header if there is one - // Warning: this must be called while the index_file_ is already seeked to the - // start of file, which it is after opening. - index_file_.read(heartbeat_); - - // Save the first index entry position - first_index_ = index_file_.tellg(); - - READER_LOG_MSG("Heartbeat is: " << heartbeat_); - - sparta_assert(heartbeat_ != 0, - "Pipeout database \"" << filepath_ << "\" had a heartbeat of 0. This " - "would be too slow to actually load"); - - //Determine the size of our index file - index_file_.seekg(0, std::fstream::end); - size_of_index_file_ = index_file_.tellg(); - //Determine the size of our record file. - record_file_.seekg(0, std::ios::end); - size_of_record_file_ = record_file_.tellg(); - - //cache the earliest start and stop of the record file - lowest_cycle_ = findCycleFirst_(); - highest_cycle_ = findCycleLast_(); - - // Building the In-Memory LocationID -> PairID Lookup structure - // from the map_file_ which contains the same. - // We read this map_file_ just once during the construction - // of the Reader Class and store all the relationships - // in an unordered_map called loc_map. When we read each Pair Record, - // we quickly do a lookup with the Location ID - // of the record in this map, and retrieve the Pair ID of that record, - // so that we can go ahead and get all the information - // about its name strings and sizeof and pair length from other data structures. - //The fields in the file are separated by ":" - - // Read through the whole map File - map_file_.processInto(loc_map_); - - // Building the In-memory Pair Lookup structure, such that, - // in future when reading back record from transaction file, - // we can use this structure to know about the length, name strings - // and sizeof the values - // for that paritcular pair, instead of using a file on disk for this. - // We read through the Data file just once, and populate this structure. - //The fields in the file are separated by ":" - - data_file_.processWith([&](auto& strm) { - uint16_t unique_id; - strm >> unique_id; - - pair_struct pStruct(strm); - - //Finally, when we have completely parsed one line of this file, - // it means we have complete knowledge of one pair type. - // We then insert this pair into out Lookup structure. - map_.emplace(unique_id, pStruct); - }); - - display_file_.processWith([&](auto& strm) { - uint16_t pairId; - strm >> pairId; - - auto& fmt_vec = map_.at(pairId).formats; - - while(!strm.eof()) { - strm >> fmt_vec; - } - }); - - // Read every line of the String Map file - string_file_.processInto(stringMap_); - } - - Reader(Reader&& rhs) = default; - - //!Destructor. - virtual ~Reader() = default; - - template - inline static Reader construct(const std::string& filepath, CallbackArgs&&... cb_args) { - return Reader(filepath, std::make_unique(std::forward(cb_args)...)); - } - - /** - * \brief Clears the internal lock. This should be used ONLY when an - * exception occurs during loading - */ - inline void clearLock(){ - lock_ = false; - } - - /** - * \brief using our PipelineDataCallback pointer - * pass all of the transactions found in a given interval of cycles. - * \param start the intervals start cycle. Transactions with - * TMEN of "start" WILL be included in this window. - * "start" will also round down to the lower index closest to "start" - * \param end the intervals stop cycle. transactions - * with TMEN of "end" will NOT be included in the window. - * - * [start, end) where start rounded down and end rounded up - * - * so if our interval = 1000 - * getWindowls(3500, 4700) will essentially return all transactions - * ending between - * [3000, 5000) - * - * \warning start must be GREATER than end. - * \warning This method IS NOT thread safe. - */ - inline void getWindow(const uint64_t start, const uint64_t end) - { - READER_LOG_MSG("returning window. START: " << start << " END: " << end); - // sparta_assert(//start < end, "Cannot return a window where the start value is greater than the end value"); - - //Make sure the user is not abusing our NON thread safe method. - acquireLock_(); - //round the end up to the nearest interval. - const uint64_t chunk_end = roundUp_(end); - - READER_LOG_MSG("end rounded to: " << chunk_end); - - //First we will want to make sure we are ready to read at the correct - //position in the record file. - record_file_.seekg(findRecordReadPos_(start)); - - const auto read_pos = record_file_.tellg(); - //what space does this interval span in the record file. - const std::fstream::pos_type full_data_size = findRecordReadPos_(chunk_end) - read_pos; - //Now start processing the chunk. - const std::fstream::pos_type end_pos = read_pos + full_data_size; - - READER_LOG_MSG("start_pos: " << read_pos << " end_pos: " << end_pos); - - //Make sure we have not passed the end position, also make sure we - //are not at -1 bc that means we reached the end of the file! - //As we read records. Read each as a transaction. - //check the flags, seek back and then read it as the proper type, - //then pass the a pointer to the struct to the appropriate callback. - - -#if READER_LOG == 1 - READER_LOG_MSG("read " << std::dec << readRecords_(end_pos, start, end) << " records"); -#else - readRecords_(end_pos, start, end); -#endif - //unlock our assertion test. - sparta_assert(lock_); - clearLock(); - } - - /** - * \brief Reads transactions afer each index in the entire file - */ - inline void dumpIndexTransactions() { - auto prev_cb = std::move(data_callback_); - try{ - uint64_t tick = 0; - index_file_.seekg(0, std::ios::beg); - while(tick <= getCycleLast() + (heartbeat_-1)){ - int64_t pos; - - // Set up a record checker to ensure all transactions fall - // within the range being queried - data_callback_ = std::make_unique(tick, tick + heartbeat_); - - pos = findRecordReadPos_(tick); - - std::cout << "Heartbeat at t=" << std::setw(10) << tick << " @ filepos " << std::setw(9) - << pos << " first transaction:" << std::endl; - - - const uint64_t chunk_end = roundUp_(tick + heartbeat_); - std::cout << "chunk end rounded to: " << chunk_end << std::endl - << "record file pos before: " << record_file_.tellg() << std::endl; - record_file_.seekg(pos, std::ios::beg); - const auto read_pos = record_file_.tellg(); - std::cout << "record file pos after: " << read_pos << std::endl; - if(read_pos == EOF) { - std::cerr << "TellG says EOF!" << std::endl; - } - else { - - //what space does this interval span in the record file. - const uint64_t full_data_size = findRecordReadPos_(chunk_end) - read_pos; - - //Now start processing the chunk. - const int64_t end_pos = pos + full_data_size; - std::cout << "pos = " << pos << ", end_pos = " << end_pos << std::endl; - - - const auto recsread = readRecords_(end_pos, tick, chunk_end); - std::cout << "Records: " << recsread << std::endl; - } - std::cout << "record file pos after read: " << record_file_.tellg() << std::endl; - std::cout << "pos variable after read: " << read_pos << std::endl; - tick += heartbeat_; - std::cout << "\n"; - } - }catch(...){ - // Restore data callback before propogating exception - data_callback_ = std::move(prev_cb); - throw; - } - - // Restore callback - data_callback_ = std::move(prev_cb); - - if(uint64_t tmp; index_file_.read(tmp)) { - std::cout << "Read junk at the end of the index file:\n " << tmp; - while(index_file_.read(tmp)) { - std::cout << " " << tmp; - } - } - } - - /** - * \brief Gets the size of a data chunk. This is a minimum granularity - * of file reads when looking for any range. Chunks are in terms of - * ticks and chunks always begin at ticks which are chunk-size-aligned - */ - inline uint64_t getChunkSize() const - { - return heartbeat_; - } - - /** - * \brief Return the start time in the file. - */ - inline uint64_t getCycleFirst() const - { - READER_DBG_MSG("Returning first cycle: " << lowest_cycle_); - //This needs to be changed. - //BUG When this returns 0, we miss many transactions in the viewer. - return lowest_cycle_; - } - - /** - * \brief Return the last end time in the file. - * Our output saved the last index to point to - * the start of last record. - */ - inline uint64_t getCycleLast() const - { - READER_DBG_MSG("Returning last cycle: " << highest_cycle_); - return highest_cycle_; - } - - /** - * \brief Gets the version of the pipeout files loaded - */ - inline uint32_t getVersion() const - { - return version_; - } - - inline bool isUpdated() - { - checkIndexUpdates_(); - return file_updated_; - } - - inline void ackUpdated() - { - file_updated_ = false; - } - - template - inline CallbackType& getCallbackAs() { - return *static_cast(data_callback_.get()); - } - - template - inline const CallbackType& getCallbackAs() const { - return *static_cast(data_callback_.get()); - } - - private: - const std::string filepath_; /*!< Path to this file */ - FileStream record_file_; /*!< The record file stream */ - FileStream index_file_; /*!< The index file stream */ - ColonDelimitedFile map_file_; /*!< The map file stream */ - ColonDelimitedFile data_file_; /*!< The data file stream */ - ColonDelimitedFile string_file_; /*!< The string map file stream */ - ColonDelimitedFile display_file_; - std::unique_ptr data_callback_; /* loc_map_; - // In-memory data structure to hold the unique pairId and - // subsequent information about its name strings and - // sizeofs for every different pair type - std::unordered_map map_; - // In-memory data structure to hold the string map structure - // which maps a tuple of integers reflecting the pair type, - // field number and field value to the actual String - // we want to display in pipeViewer - std::unordered_map, - std::string, - hashtuple::hash>> - stringMap_; - }; - - // Formats a pair into an annotation-like string - used by transactionsearch and Python interface - // This version accepts the individual pair_t members as parameters so that it can be used with the - // transactionInterval type - inline static std::string formatPairAsAnnotation(const uint64_t transaction_ID, - const uint64_t display_ID, - const uint16_t length, - const std::vector& nameVector, - const std::vector& stringVector) { - std::ostringstream annt_preamble; - std::ostringstream annt_body; - - const uint16_t display_id = static_cast((display_ID < 0x1000 ? display_ID : transaction_ID) & 0xfff); - - std::ios flags(nullptr); - flags.copyfmt(annt_preamble); - annt_preamble << std::setw(3) << std::setfill('0') << std::hex << display_id; - annt_preamble.copyfmt(flags); - annt_preamble << ' '; - - for(size_t i = 1; i < length; ++i) { - const auto& name = nameVector[i]; - const auto& value = stringVector[i]; - - if(name != "DID") { - annt_body << name << '(' << value << ") "; - } - - if(name == "uid") { - annt_preamble << 'u' << (std::stoull(value) % 10000) << ' '; - } - else if(name == "pc") { - flags.copyfmt(annt_preamble); - annt_preamble << "0x" << std::setw(4) << std::setfill('0') << std::hex << (std::stoull(value, 0, 16) & 0xffff) << ' '; - annt_preamble.copyfmt(flags); - } - else if(name == "mnemonic") { - const std::string_view value_view(value); - annt_preamble << value_view.substr(0, 7) << ' '; - } - } - - return annt_preamble.str() + annt_body.str(); - } - - // Formats a pair into an annotation-like string - used by transactionsearch and Python interface - inline static std::string formatPairAsAnnotation(const pair_t* pair) { - return formatPairAsAnnotation(pair->transaction_ID, - pair->display_ID, - pair->length, - pair->nameVector, - pair->stringVector); - } -}//NAMESPACE:sparta::pipeViewer diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/SimpleOutputInterface.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/SimpleOutputInterface.hpp deleted file mode 100644 index d2bca3fca1..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/SimpleOutputInterface.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include "sparta/pipeViewer/Outputter.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/utils/SpartaException.hpp" - -namespace sparta{ - namespace pipeViewer{ - - class SimpleOutputInterface { - public: - static constexpr uint64_t BAD_DISPLAY_ID = 0x1000; // anything higher than 0xfff is out of bounds - - SimpleOutputInterface(const std::string& filepath, const uint64_t interval, bool debug = false) : - outputter_(filepath, interval) - { - debug_ = debug; - if(SPARTA_EXPECT_FALSE(debug_)) - { - std::cerr << "constructed output wrapper" << std::endl; - } - // default intialize annot_struct. - annot_struct_.time_Start = 0; - annot_struct_.time_End = 0; - annot_struct_.parent_ID = 0; - annot_struct_.control_Process_ID = 0; - annot_struct_.transaction_ID = 0; - annot_struct_.display_ID = BAD_DISPLAY_ID; - annot_struct_.location_ID = 0; - annot_struct_.flags = is_Annotation; - annot_struct_.length = 0; - annot_struct_.annt = nullptr; - } - - void writeTransaction(uint64_t start, - uint64_t end, - uint64_t location_id, - const std::string& dat, - bool continue_transaction = false) - { - std::cout << "SimplePutputterInterface" << std::endl; - annot_struct_.length = (uint16_t)(dat.size() + 1); - if(SPARTA_EXPECT_FALSE(debug_)) - { - std::cerr << " annotation length = " << annot_struct_.length << std::endl; - } - annot_struct_.annt = dat.c_str(); - annot_struct_.time_Start = start; - annot_struct_.time_End = end; - annot_struct_.transaction_ID = next_transaction_id_++; - annot_struct_.location_ID = location_id; - - if(continue_transaction) - { - annot_struct_.flags |= CONTINUE_FLAG; - } - - sparta_assert(start != end); - if(SPARTA_EXPECT_FALSE(debug_)) - { - std::cerr << " ----> wrote transaction: (" << start << ", "<< end << ") " << dat << std::endl; - } - outputter_.writeTransaction(annot_struct_); - - if(continue_transaction) - { - annot_struct_.flags &= ~CONTINUE_FLAG; - } - } - - void writeIndex() { - outputter_.writeIndex(); - } - - private: - Outputter outputter_; - annotation_t annot_struct_; - uint64_t next_transaction_id_ = 0; - bool debug_; - }; - } // namespace pipeViewer -} // namespace sparta - - diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp deleted file mode 100644 index 7c8356f347..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp +++ /dev/null @@ -1,1902 +0,0 @@ -// -*- C++ -*- - -#pragma once - -// Required to enable std::this_thread::sleep_for -#ifndef _GLIBCXX_USE_NANOSLEEP -#define _GLIBCXX_USE_NANOSLEEP -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "TransactionInterval.hpp" -#include "PipelineDataCallback.hpp" -#include "Reader.hpp" -#include "sparta/pipeViewer/transaction_structures.hpp" -#include "sparta/utils/SpartaAssert.hpp" -#include "sparta/utils/TimeManager.hpp" - -namespace sparta::pipeViewer { - -class TransactionDatabaseInterface -{ -public: - - /*! - * \brief Transaction type - */ - using Transaction = transactionInterval; - - using interval_ptr = Transaction*; - using interval_idx = uint32_t; - using const_interval_idx = const interval_idx; - - /*! - * \brief Constant representing ID Of NO transaction - */ - static constexpr interval_idx NO_TRANSACTION = 0xffffffff; - - /*! - * \brief Stop after exceeding this threshold. - * A low threshold is required for testing the sliding window algorithm - */ - //static constexpr uint64_t MEMORY_THRESHOLD_BYTES = 1000000000; // 1 GB - //static constexpr uint64_t MEMORY_THRESHOLD_BYTES = 100000000; // 100 MB - static constexpr uint64_t MEMORY_THRESHOLD_BYTES = 500000000; // 500 MB - - /*! - * \brief Background thread sleep period between checks - */ - static constexpr uint32_t BACKGROUND_THREAD_SLEEP_MS = 100; - - /*! - * \brief Interval between DB update checks in seconds - */ - static constexpr time_t DB_UPDATE_INTERVAL_S = 10; - -private: - - /*! - * \brief Window range definition - */ - struct Window { - uint64_t start; - uint64_t end; - }; - - /*! - * \brief Node containing a chunk of data, which may be sparsely populated - */ - class Node - { - public: - - /*! - * \brief Data stored at a single tick - */ - struct TickData { - - /*! - * \brief Constructor - * \param tick_offset Tick index of this object - relative to containing node's start - * tick - * \param num_locations Number of locations in this TransactionDatabase - * \param prev Previous TickData (if being inserted after another) - * \param next Next TickData (if being inserted before another) - */ - TickData(const uint64_t _tick_offset, const uint32_t num_locations, - const TickData* prev = nullptr, const TickData* next = nullptr) : - tick_offset(_tick_offset), - data(num_locations, NO_TRANSACTION) - { - sparta_assert(!prev || prev->tick_offset < _tick_offset); - sparta_assert(!next || next->tick_offset > _tick_offset); - - // Auto-fill each item in the array based on prev and next TickDatas. If there is a - // prev and next it means that this is being inserted into the Node::tick_content - // list. - - if(!prev && !next){ - return; // early-out, no existing transactions to look at - } - - for(uint32_t x = 0; x < num_locations; ++x){ - if(prev && next && (prev->data[x] == next->data[x])){ - // Has prev AND has same next or no next - data[x] = prev->data[x]; - } - } - } - - /*! - * \brief This tick's tick offset - */ - const uint64_t tick_offset; - - /*! - * \brief Indices into all_intervals_ for this tick - */ - std::vector data; - }; - - private: - - /*! - * \brief All intervals allocated in this Node - */ - std::vector all_intervals_; - - bool should_del_; //!< Should this node be deleted when next possible - - const uint64_t start_inclusive; - const uint64_t end_exclusive; - const uint32_t num_locations; - uint64_t transaction_bytes_; //!< Bytes used for transactions - volatile bool complete_; //!< Has this node been completely populated - - /*! - * \brief Transaction pointers per location for each relevant tick - */ - std::list tick_content; - - /*! - * \brief How sparsely the node stores its content (e.g. how ticks are - * skipped) - */ - uint32_t sparseness; - - /*! - * \brief How many entries were overwritten because they already had a - * transaction pointer - */ - uint32_t overwrites; - - /*! - * \brief Acquired when this block is being populated. To read it, this - * mutex must acquired - */ - mutable std::mutex loading_mutex_; - - public: - - /*! - * \brief Constructor - * \param num_locations Number of locations to support. Must be > 0 - */ - Node(const uint64_t start_inc, const uint64_t size, const uint32_t _num_locations) : - should_del_(false), - start_inclusive(start_inc), - end_exclusive(start_inc + size), - num_locations(_num_locations), - transaction_bytes_(0), - complete_(false), - sparseness(size), - overwrites(0) - { - sparta_assert(size > 0); - loading_mutex_.lock(); - sparta_assert(num_locations > 0, - "A transaction database node requires a location count of 1 or more"); - all_intervals_.reserve(512); - - // Insert the first node at tick-offset = 0 so that there always data to walk. - tick_content.emplace_back(0, num_locations, nullptr, nullptr); - sparseness--; - } - - /*! - * \brief Destructor - */ - ~Node() - { - if(!complete_){ - // Someone else may be manipulating this. so wait for it - loading_mutex_.lock(); - } - loading_mutex_.unlock(); - } - - /*! - * \brief Gets the mutex associated with loading data into this node. - * \note To load data into this node or read data from it, the mutex - * must be held. This allows node loading from a background thread while - * the forground thread queries the entire transaction database. Only - * when data is requested from a node (or it must be deleted), does the - * foreground thread have to wait for this mutex - */ - std::mutex& getMutex() { return loading_mutex_; } - - /*! - * \brief Dumps the content of this node where each row is a TickData - * \param location_start First location to show for each row. - * \param location_end Exclusive end of locaiton range to show. This - * location will not be shown. This value is always clamped to maximum - * number of locations. If < location_start, nothing will be shown. - * \param tick_entry_limit Number of tick entries to show. 0 if - * unlimited. Value is always clamped to total number of tick entries - */ - void dumpContent(std::ostream& o, - const uint32_t location_start = 0, - const uint32_t location_end = 0, - const uint32_t tick_entry_limit = 0) const { - auto itr = tick_content.begin(); - uint32_t tick_entries = 0; - const uint32_t real_loc_limit = std::min(num_locations, - location_end > 0 ? location_end : std::numeric_limits::max()); - o << std::setw(8) << "location: "; - for(uint32_t loc = location_start; loc < real_loc_limit; ++loc){ - o << std::dec << std::setw(4) << loc << ' '; - } - o << '\n'; - for(; itr != tick_content.end(); ++itr){ - o << std::dec << std::setw(8) << itr->tick_offset << ": "; - for(uint32_t loc = location_start; loc < real_loc_limit; ++loc){ - if(itr->data[loc] == NO_TRANSACTION){ - o << std::hex << std::setw(4) << "..." << ' '; - }else{ - o << std::hex << std::setw(4) << itr->data[loc] << ' '; - } - } - o << '\n'; - tick_entries++; - if(tick_entry_limit != 0 && tick_entries >= tick_entry_limit){ - break; - } - } - if(itr != tick_content.end()){ - o << "more...\n"; - } - - o << "Up to 20 transactions in location range\n"; - for(uint32_t tids = 0; tids < 20; ++tids){ - if(tids >= all_intervals_.size()){ - break; - } - const auto& trans = all_intervals_[tids]; - o << "Transaction " << std::dec << trans.transaction_ID - << " disp=" << trans.display_ID - << " loc=" << trans.location_ID - << " @ [" << trans.getLeft() - << ", " << trans.getRight() << ")\n"; - } - } - - /*! - * \brief Get the content string from dumpContent - */ - std::string getContentString(const uint32_t location_start = 0, - const uint32_t location_end = 0, - const uint32_t tick_entry_limit = 0) const { - std::ostringstream ss; - dumpContent(ss, location_start, location_end, tick_entry_limit); - return ss.str(); - } - - /*! - * \brief Returns the memory used by this node including the node itself - * and anything allocated my the node. - */ - uint64_t getSizeInBytes() const - { - return sizeof(*this) + - //(end_exclusive - start_inclusive) * num_locations * sizeof(interval_idx) + - (tick_content.size() * (num_locations * sizeof(interval_idx) + sizeof(TickData))) + - transaction_bytes_; - } - - /*! - * \brief Get low inclusive endpoint of this node - */ - uint64_t getStartInclusive() const { - return start_inclusive; - } - - /*! - * \brief Get high exclusive endpoint of this node - */ - uint64_t getEndExclusive() const { - return end_exclusive; - } - - /*! - * \brief Mark this node as completed and release the loading mutex. - * The thread that constructs this node must do this - */ - void markComplete() { - complete_ = true; - loading_mutex_.unlock(); - } - - /*! - * \brief Is this node fully loaded. If false, it means that it has not - * been loaded or there was an error loading it. - * \nhote This is NOT a worker-thread synchronization mechanism - */ - bool isComplete() const { - return complete_; - } - - /*! - * \brief Add a new transaction to this node - * \param time_Start inclusive start tick - * \param time_End exclusive end tick - */ - template - void addTransaction(const Transaction::IntervalDataT time_Start, - const Transaction::IntervalDataT time_End, - const uint16_t control_Process_ID, - const uint64_t transaction_ID, - const uint64_t display_ID, - const uint32_t location_ID, - _Args&&... __args) - { - // Interpret the time_End differently depending on whether the - // endpoint is inclusive or exclusive. This is a property of the - // pipeline collector. - static constexpr bool IS_TRANSACTION_END_INCLUSIVE = false; - - // Ensure this location fits within this Node. - sparta_assert(time_Start < end_exclusive); - - sparta_assert(location_ID < num_locations, - "Encountered a transaction with location ID=" << location_ID - << " when the database window was initialized expecting only " - << num_locations << " locations"); - - // Compute exclusive endpoint, clamping it to the valid range for this node - const uint64_t transaction_exclusive_end = std::min( - time_End - static_cast(IS_TRANSACTION_END_INCLUSIVE && (time_End > time_Start)), - end_exclusive - ); - - sparta_assert(transaction_exclusive_end > start_inclusive); - - // Track a copy locally in this node - all_intervals_.emplace_back(time_Start, - time_End, // Use original ending - control_Process_ID, - transaction_ID, - display_ID, - location_ID, - std::forward<_Args>(__args)...); - transaction_bytes_ += all_intervals_.back().getSizeInBytes(); - - // Index of this transaction within the node-scoped transaction list - const auto trans_pos = all_intervals_.size() - 1; - - // Place pointers to this transaction in every relevant slot - const uint64_t start_cycle_offset = std::max(time_Start, start_inclusive) - start_inclusive; - const uint64_t trans_end_exclusive = std::min(transaction_exclusive_end, end_exclusive); - const uint64_t end_entry_offset = trans_end_exclusive - start_inclusive; - - // Insert the tick data - static_assert(IS_TRANSACTION_END_INCLUSIVE == false, - "Some assumptions of the following routine assume exclusive endpoints"); - TickData* prev_td = nullptr; - bool marked_start = false; - bool marked_ending = false; - const bool single_tick_entry = end_entry_offset - start_cycle_offset == 1; - //! \todo Remove this linear search. Maybe turn it into a binary search? - //! Or insert backward from end point which will be close to most revent endpoint - auto itr = tick_content.begin(); - for(; itr != tick_content.end(); ++itr){ - TickData& td = *itr; - - if(td.tick_offset < start_cycle_offset){ - // Continue - haven't reached first cycle in the transaction - }else if(td.tick_offset == start_cycle_offset){ - // Reached a TickData matching the start of this transaction - if(td.data[location_ID] != NO_TRANSACTION){ - overwrites++; - } - td.data[location_ID] = trans_pos; - marked_start = true; - marked_ending |= single_tick_entry; - }else{ - // td.tick_offset > start_cycle_offset - if(!marked_start){ - // Passed the start of this transaction while missing the start point. Need to - // insert a TickData at the start of the transaction - const auto new_itr = tick_content.emplace(itr, start_cycle_offset, num_locations, prev_td, &td); - prev_td = &(*new_itr); - new_itr->data[location_ID] = trans_pos; - sparseness--; - marked_start = true; - } - - if(td.tick_offset == end_entry_offset - 1){ - // Reached final cycle in the transaction - // Just update the pointer in this TickData - if(td.data[location_ID] != NO_TRANSACTION && td.data[location_ID] != location_ID){ - overwrites++; - } - td.data[location_ID] = trans_pos; - marked_ending = true; - - }else if(td.tick_offset >= end_entry_offset){ - // Reached the (exclusive end) of the transaction or later - // Nothing to update here if it already exists. If the ending was never - // marked (had an entry added in a TickData), then a TickData must be - // inserted now immediately before this tick - // Mark last tick INSIDE transaction if not marked and start != ending - // single-tick entries are a special case where the last tick of the - // transacation IS the startpoint - if(!marked_ending && !single_tick_entry){ - // Insert the new tick which pulls from prev or next. - const auto new_itr = tick_content.emplace(itr, end_entry_offset-1, num_locations, prev_td, &td); - prev_td = &(*new_itr); - new_itr->data[location_ID] = trans_pos; - sparseness--; - } - - // Need to indicate that the transaction is no longer in this location at - // end_entry_offset. If there was aready an entry here, assume it has another - // transaction or NO_TRANSACTION at this location. - if(td.tick_offset > end_entry_offset){ - // Insert a new TickData before this indicating that there is NO transaction - // in this location after this transaction ends. This new tick data must be - // populated with pointers for its locations where the next and previous - // TickDatas have the same transaction. We're inserting a TickData in the - // middle of a transactions that was sparsely indicated in the file. - // Insert the new tick which pulls from prev and next. - // Note that td.data[location_ID] will already be NO_TRANSACTION - if(end_entry_offset < end_exclusive){ // Do not insert at exclusive endpoint of this NODE - const auto new_itr = tick_content.emplace(itr, end_entry_offset, num_locations, prev_td, &td); - prev_td = &(*new_itr); - sparta_assert(new_itr->data[location_ID] == NO_TRANSACTION); - sparseness--; - } - } - - break; - - }else{ - // Current tick-data is before the inclusive end of the transaction. Update - // it to point to this transaction - if(td.data[location_ID] != NO_TRANSACTION && td.data[location_ID] != location_ID){ - overwrites++; - } - td.data[location_ID] = trans_pos; - } - } - - prev_td = &td; - } - - if(time_Start == time_End){ - // This is a degenerate case and will not properly be handled by the insertion - // algorithm above. The else-case asumptions about marking the start and attempting - // to mark the last tick are invalid. - }else{ - - // If the loop terminated because the iterator hit the end. Insert a new TickContent - if(itr != tick_content.end()){ - sparta_assert(marked_start, - "Somehow made it through a transaction insertion into node \"" - << stringize() << "\" without marking the start in a TickData. " - "Transaction " << std::dec << transaction_ID - << " disp=" << std::dec << display_ID - << " loc=" << std::dec << location_ID - << " @ [" << time_Start << ',' << time_End << ")\n"); - }else{ - // Finished iterating all the TickDatas. Add remaining entries - - if(!marked_start){ - // Passed the start of this transaction while missing the start point. Need to - // insert a TickData at the start of the transaction - const auto new_itr = tick_content.emplace(itr, start_cycle_offset, num_locations, prev_td, nullptr); - prev_td = &(*new_itr); - new_itr->data[location_ID] = trans_pos; - sparseness--; - } - - // Mark last tick INSIDE transaction if not marked and start != ending - if(!marked_ending && !single_tick_entry){ - // Allow insertion at end_exclusive (no later) - const auto num_ticks_in_node = end_exclusive - start_inclusive; - const uint64_t entry_tick = std::min(num_ticks_in_node, end_entry_offset-1); - const auto new_itr = tick_content.emplace(itr, entry_tick, num_locations, prev_td, nullptr); - prev_td = &(*new_itr); - new_itr->data[location_ID] = trans_pos; - sparseness--; - } - - if(end_entry_offset < end_exclusive){ // Do not insert at exclusive endpoint. It will never be accessed - const auto new_itr = tick_content.emplace(itr, end_entry_offset, num_locations, prev_td, nullptr); - sparta_assert(new_itr->data[location_ID] == NO_TRANSACTION); - sparseness--; - } - } - } - - // Sanity check transactions for repeats (this is expensive) - int64_t last_off = -1; - itr = tick_content.begin(); - for(; itr != tick_content.end(); ++itr){ - const auto tick_offset = static_cast(itr->tick_offset); - sparta_assert(tick_offset > last_off); - last_off = tick_offset; - } - - // Dump data for a range of locations. Here, after each insertion - // The dump can show how the list of tick-datas was modified from - // the last insertion. - //dumpContent(std::cout, 4566, 4567, 0); - - //interval_idx * ref_ptr = &(tick_content[(start_cycle_offset*num_locations) + location_ID]); - //sparta_assert(ref_ptr >= &(tick_content[0])); - //sparta_assert(ref_ptr < &(tick_content[0]) + (end_exclusive - start_inclusive) * num_locations); - //uint32_t slots = 0; - //for(; ref_ptr < &(tick_content[0]) + (end_entry_offset * num_locations); ref_ptr += num_locations){ - // *ref_ptr = trans_pos; - // ++slots; - // sparta_assert(ref_ptr < &(tick_content[0]) + (end_exclusive - start_inclusive) * num_locations); - //} - //std::cout << "number of slots for " << location_ID << " @ [" << time_Start - // << ',' << time_End << ") in " << stringize() << " = " << slots << std::endl; - } - - std::string stringize() const { - std::ostringstream ss; - ss << ""; - return ss.str(); - } - - /*! - * \brief Flags this node for deletion. Node can then be freed within a - * query or by a worker thread if needed - */ - void flagForDeletion() { - should_del_ = true; - } - - /*! - * \brief Should this node be deleted when convenient? - * \return True if this block is not currently being loaded and was - * flagged for deletion - */ - bool canDelete() const { - if(complete_){ - return should_del_; - } - return false; - } - - /*! - * \brief Gets an iterator pointing to the TickData associated witih an - * absolute tick number of the nearest earlier TickData for that tick - * number. - * \return Iterator that is not equivalent to getTickDataEnd. This is - * allowed because the presence of at least 1 TickData is guaranteed at - * construction. - */ - std::list::const_iterator getTickData(const uint64_t abstime) const { - sparta_assert(abstime >= start_inclusive && abstime < end_exclusive, - "tick (" << abstime << ") being queried is not within range of node " - << stringize()); - const uint64_t t = abstime - start_inclusive; - - // Mutiple TickDatas Guaranteed by Node constructor. Required for following code to work - sparta_assert(!tick_content.empty()); - - // Find the locations - auto prev_itr = tick_content.cbegin(); - for(auto itr = prev_itr; itr != tick_content.cend(); ++itr){ - if(itr->tick_offset == t) { - return itr; - } - if(itr->tick_offset > t) { - return prev_itr; - } - prev_itr = itr; - } - - return prev_itr; - - //// Constructor guarantees 1 or more locations - //const_interval_idx * ref_ptr = &(tick_content[(t*num_locations)]); - //sparta_assert(ref_ptr >= &(tick_content[0])); - //sparta_assert(ref_ptr < &(tick_content[0]) + (end_exclusive - start_inclusive) * num_locations); - //return ref_ptr; - } - - /*! - * \brief Returns the end iterator associated with the tick data in this - * Node. This can be compared to the result of getTickData. - */ - std::list::const_iterator getTickDataEnd() const { - return tick_content.end(); - } - - /*! - * \bried Returns a vector containing all intervals known to this node. - * This can be indexed by offsets stored at each location - */ - const std::vector& getIntervals() const { - return all_intervals_; - } - }; - - - /*! - * \brief Manages pipeViewer reader callbacks and dumps transaction data into an - * ordered list of nodes - */ - class SmartReader - { - private: - class SmartReaderCallback : public PipelineDataCallback { - private: - /*! - * \brief Pointer to nodes to be populated from reader callbacks - */ - std::vector const * load_to_nodes_ = nullptr; - - /*! - * \brief Add a transaction to this smart reader - */ - template - void addTransaction_(const Transaction::IntervalDataT time_Start, - const Transaction::IntervalDataT time_End, - const uint16_t control_Process_ID, - const uint64_t transaction_ID, - const uint64_t display_ID, - const uint32_t location_ID, - const uint16_t flags, - _Args&&... __args) - { - //std::cout << "Interval " << std::setw(6) << location_ID << " @ [" - // << time_Start << ',' << time_End << ")" << std::endl; - - // Guaranteed not nullptr - std::vector::const_iterator node_itr = load_to_nodes_->begin(); - - while(node_itr != load_to_nodes_->end()){ - // Note: Assuming exclusove right endpoint of transactions - // Stop iterating when a node is encountered that starts after this - // transaction ends - if((*node_itr)->getStartInclusive() >= time_End){ - break; - }else if((*node_itr)->getEndExclusive() > time_Start){ - // This case should only accept nodes that contain part of this - // transaction. findNode can return a node preceeding this - // transaction. - // Furthermore, only incomplete nodes will be loaded - if(false == (*node_itr)->isComplete()){ - (*node_itr)->addTransaction(time_Start, - time_End, - control_Process_ID, - transaction_ID, - display_ID, - location_ID, - flags, - std::forward<_Args>(__args)...); - } - } - ++node_itr; - } - } - - public: - /*! - * \brief Callback from PipelineDataCallback - */ - void foundTransactionRecord(const transaction_t* loc) override { - addTransaction_(loc->time_Start, loc->time_End, loc->control_Process_ID, - loc->transaction_ID, loc->display_ID, loc->location_ID, loc->flags); - } - void foundInstRecord(const instruction_t* loc) override { - addTransaction_(loc->time_Start, loc->time_End, loc->control_Process_ID, - loc->transaction_ID, loc->display_ID, loc->location_ID, loc->flags, - loc->parent_ID, loc->operation_Code, loc->virtual_ADR, - loc->real_ADR); - } - void foundMemRecord(const memoryoperation_t* loc) override { - addTransaction_(loc->time_Start, loc->time_End, loc->control_Process_ID, - loc->transaction_ID, loc->display_ID, loc->location_ID, loc->flags, - loc->parent_ID, loc->virtual_ADR, loc->real_ADR); - } - void foundAnnotationRecord(const annotation_t* loc) override { - addTransaction_(loc->time_Start, loc->time_End, loc->control_Process_ID, - loc->transaction_ID, loc->display_ID, loc->location_ID, loc->flags, - loc->parent_ID, loc->length, std::move(loc->annt)); - } - - void foundPairRecord(const pair_t* loc) override { - addTransaction_(loc->time_Start, loc->time_End, loc->control_Process_ID, - loc->transaction_ID, loc->display_ID, loc->location_ID, loc->flags, - loc->parent_ID, loc->length, loc->pairId, loc->sizeOfVector, - loc->valueVector, loc->nameVector, - loc->stringVector, loc->delimVector); - } - - void loadDataToNodes(std::vector const * load_to) { - load_to_nodes_ = load_to; - } - }; - - /*! - * \brief Reader used to get intervals from the event file - */ - Reader event_reader_; - - - /*! - * \brief Mutex that must be locked loading through the reader - */ - std::mutex reader_load_mutex_; - - public: - - /*! - * \brief Constructor - * \param file_prefix Prefix of the pipeViewer database that will be opened - * \post Handle to pipeViewer database identified by \a file_prefix is open - */ - explicit SmartReader(const std::string& file_prefix) : - event_reader_(Reader::construct(file_prefix)), - reader(event_reader_) - {;} - - const Reader& reader; - - /*! - * \brief Resets the query state on the reader - */ - void resetQueryState() - { - event_reader_.clearLock(); - } - - /*! - * \brief Locks the non-recursive mutex for the current thread - */ - void lock() { reader_load_mutex_.lock(); } - - /*! - * \brief Unlocks the non-recursive mutex for the current thread - */ - void unlock() { reader_load_mutex_.unlock(); } - - /*! - * \brief Load a range of data from the reader to a specific set of nodes - * \param start Start reading from the file at this tick. This should - * generally be chunk aligned to avoid excess reading - * \param enbd Stop reading from the file at this tick. This should - * generally be chunk aligned to avoid excess reading - * \param load_to vector of Node* indicating which nodes should receive the - * data being loaded. Because of the nature of file reading, more than 1 - * node's worth of data must be read to see all transaction's that overlap - * the node (since they are sorted by end-time). - * \note This is a blocking call - * \note load_to must not be modified externally while within this call. - * \pre this reader must be locked via lock by the thread calling this - * method. - */ - void loadDataToNodes(const uint64_t start, - const uint64_t end, - std::vector const * load_to) - { - sparta_assert(load_to != nullptr, "cannot loadDataToNodes with a null load_to vector"); - auto& cb = event_reader_.getCallbackAs(); - cb.loadDataToNodes(load_to); - event_reader_.getWindow(start, end); - cb.loadDataToNodes(nullptr); - } - - bool isUpdated() - { - return event_reader_.isUpdated(); - } - - void ackUpdated() - { - lock(); - event_reader_.ackUpdated(); - unlock(); - } - }; - - SmartReader smart_reader_; - - /*! - * \brief Type for Node list. This must be a list so that objects are - * perserved - */ - using node_list_t = std::list; - - using node_iterator_t = node_list_t::iterator; - using node_const_iterator_t = node_list_t::const_iterator; - - const std::string file_prefix_; //!< Database filename prefix - - const uint32_t num_locations_; //!< Number of locations in the database - - /*! - * \brief Data nodes currently help in this class - * \warning This must be a list type - */ - node_list_t nodes_; - - /*! - * \brief Currently within a query? - */ - bool in_query_; - - /*! - * \brief Currently loaded window - */ - Window window_; - - /*! - * \brief Last query range - */ - Window last_query_; - - const uint64_t start_tick_; //!< Inclusive - uint64_t end_tick_; //!< Exclusive - - /*! - * \brief Size of a chunk in the transaction bd file. Nodes must be - * contained within a single chunk each - */ - uint64_t chunk_size_; - - uint64_t node_size_; //!< Size of a node in this window - - /*! - * \brief Background loader thread - */ - std::thread background_loader_; - - /*! - * \brief Mutex that must be locked when manipulating the node list and window - */ - mutable std::recursive_mutex node_list_mutex_; - - /*! - * \brief Should the background thread exit? - */ - volatile bool background_thread_should_exit_; - - /*! - * \brief Is this interface in verbose mode - */ - bool verbose_; - - /*! - * \brief Is this interface polling for database updates - */ - bool update_enabled_; - - /*! - * \brief Incremented every time this interface detects a database update - */ - uint64_t update_ready_; - - /*! - * \brief Timestamp of last check for DB updates - */ - time_t last_updated_; - -public: - - /*! - * \brief Callback function signature for each tick encountered during a - * cycle - * \null values for the pointers (other than user_data) indicate no data at - * the specified tick - */ - using callback_fxn = void (*)(void* user_data, - uint64_t tick, - const_interval_idx * location_contents, - const TransactionDatabaseInterface::Transaction* transactions, - uint32_t num_locations); - - TransactionDatabaseInterface() = delete; - TransactionDatabaseInterface(const TransactionDatabaseInterface&) = delete; - const TransactionDatabaseInterface& operator=(const TransactionDatabaseInterface&) = delete; - - /*! - * \brief Constructor - * \param file_prefix Path and prefix of database files - * \param num_locations Number of locations in the tree - */ - TransactionDatabaseInterface(const std::string& file_prefix, - const uint32_t num_locations, - const bool update_enabled = false) : - smart_reader_(file_prefix), - file_prefix_(file_prefix), - num_locations_(num_locations), - in_query_(false), - window_{0,0}, - last_query_{0,0}, - start_tick_(smart_reader_.reader.getCycleFirst()), - end_tick_(smart_reader_.reader.getCycleLast()), - chunk_size_(smart_reader_.reader.getChunkSize()), - node_size_(0), - background_thread_should_exit_(false), - verbose_(false), - update_enabled_(update_enabled), - update_ready_(0), - last_updated_(0) - { - // Find a node size that is an even integer division of chunk_size_ - static constexpr uint64_t MAX_NODE_SIZE = 200000; - node_size_ = chunk_size_; - for(uint32_t i = 1; i < 2000; ++i) { - const uint64_t temp_size = chunk_size_ / i; - if(temp_size * i == chunk_size_){ - if(temp_size <= MAX_NODE_SIZE){ - node_size_ = temp_size; - break; - } - } - } - sparta_assert(node_size_ >= 100, - "Size of node could not be determined. Heartbeat (" - << chunk_size_ << ") is not a multiple of 100"); - if(node_size_ > MAX_NODE_SIZE){ - std::cerr << "Warning: unable to find a suitable node size evenly divisible by chunk " - "size (" << chunk_size_ << ")" << std::endl; - } - - // DO NOT Load an initial window - // This is just a waste of time whenever the user wants to start at a non-zero tick and - // because of how close construction of this class tends to be to its usage, there is almost - // no opportunity to preload data. - //uint64_t load_end = std::min(start_tick_+1000, end_tick_); - //load_(start_tick_, load_end); - - // Start background thread - background_loader_ = std::thread(std::bind(&TransactionDatabaseInterface::backgroundLoader_, this)); - } - - /*! - * \brief Destructor - */ - ~TransactionDatabaseInterface() - { - background_thread_should_exit_ = true; - if(background_loader_.get_id() != std::thread::id()){ - background_loader_.join(); - } - } - - /*! - * \brief Sets the current verbose logging state of this interface - * \param verbose New verbose logging state - */ - void setVerbose(const bool verbose) { verbose_ = verbose;} - - /*! - * \brief Gets the current verbose logging state of this interface - */ - bool getVerbose() const { return verbose_; } - - /*! - * \brief Gets the number of ticks in each node - */ - uint32_t getNodeLength() const { return node_size_; } - - /*! - * \brief Gets the number of ticks in each heartbeat - */ - uint64_t getChunkSize() const { return chunk_size_; } - - /*! - * \brief Resets any temporary query state. - * \note this is mainly a debugging feature - */ - void resetQueryState() { - - // TODO: lock access - in_query_ = false; - smart_reader_.resetQueryState(); // In case exception occurred during reading - } - - /*! - * \brief Returns the inclusive start point of the last query made. - */ - uint64_t getLastQueryStart() const { - return last_query_.start; - } - - /*! - * \brief Returns the exclusive end point of the last query made. This is - * equal to 0 if no query has been made yet - */ - uint64_t getLastQueryEnd() const { - return last_query_.end; - } - - /*! - * \brief Completely unload all nodes and reset the window. - * Database connection is still maintained. The next query will reload any - * necessary data - */ - void unload() { - std::lock_guard lock(node_list_mutex_); - - window_.start = 0; - window_.end = 0; - - nodes_.clear(); - } - - /*! - * \brief Perform a range query - * \param _start_inclusive_inclusive Start tick of query (inclusive) - * \param _end_inclusive_inclusive End tick of query (exclusive) - * \param cb Callback funtion for each cycle at which query results are - * found. Callback will be invoked with \a user_data as the first argument. - * Callback will be invoked for all ticks in the requested range. It is the - * callback's responsibility to filter. If there is no data in the - * transaction database for a given tick, the callback location_content and - * transaction pointers will be nullptr and num_locations must be ignored. - * \param user_data user-defined data pointer. Given as first argument in - * \a cb function. This is often a class pointer to which the method is - * forwarded - * \param modify_tracking Treat the current query as the new query range. - * Normally, the last query is used to predict the next data that will be - * needed. If a small query is made after a large query (and inside that - * previous query's range), some data from the large query may be dropped in - * the background. Normally these large queries should have their data held - * in memory because the next query will tend to be that range shifted by a - * few ticks in either direction. To preven small queries within these - * ranges from breaking this prediction, modify_tracking can be set to false - * Data outside this new range may be unloaded and replaced with data closer - * to one endpoint of the range. If false, data will NOT be loaded or - * unloaded, but the queried range must be a subset of the prior query range. - * \pre Must not be call from within a callback of another query. - */ - void query(const uint64_t _start_inclusive, - const uint64_t _end_inclusive, - callback_fxn&& cb, - void* user_data=nullptr, - const bool modify_tracking=true) - { - std::lock_guard lock(node_list_mutex_); - - sparta_assert(_end_inclusive >= _start_inclusive, - "end point in query must be >= start point"); - - // Filter invalid accesses - if(_start_inclusive >= end_tick_ || _end_inclusive < start_tick_){ - return; - } - - // Clamp to file range - const uint64_t start_inclusive = std::max(_start_inclusive, start_tick_); - const uint64_t end_exclusive = std::min(_end_inclusive+1, end_tick_); - - // This is to prevent recursion when querying from within callback - // functions. This has nothing to do with thread safety. - sparta_assert(false == in_query_, - "Cannot query transaction database from within another query. If a query " - "threw an exception, use resetQueryState() to recover before the next " - "query") - - if(!modify_tracking){ - // Ensure that this query range is inside the last_query range - sparta_assert(start_inclusive >= window_.start, - "Pipeout database query with modify_tracking=false was not inside " - "prior query range. query start = " << start_inclusive << " while " - "previous (loaded) start = " << window_.start); - sparta_assert(end_exclusive <= window_.end, - "Pipeout database query with modify_tracking=false was not inside " - "prior query range. query end exclusive = " << end_exclusive - << " while previous (loaded) end exclusive = " << window_.end); - } - - // Set in query after all non-fatal exceptions - in_query_ = true; - - if(modify_tracking){ - // Store this latest query for later use. Ensure this is modified and - // read only with node_list_mutex acquired - last_query_.start = start_inclusive; - last_query_.end = end_exclusive; - - // Ensure that necessary data is loaded immediately. - if(start_inclusive < window_.start || end_exclusive > window_.end) { - - load_(start_inclusive, - end_exclusive); // exclusive end - } - } - - uint64_t t = _start_inclusive; - node_iterator_t itr = findNode(t); - - if(itr != nodes_.end()){ - // Now make some callbacks with no data for the start of the - // requested range up to the first block found - while(t < itr->getStartInclusive()){ - cb(user_data, t, nullptr, nullptr, 0); - ++t; - } - - //itr->dumpContent(std::cout, 890, 905, 0); - } - - for(; itr != nodes_.end(); ++itr){ - //std::cout << "Looking at node " << itr->stringize() << std::endl; - // Assuming exclusive right endpoint of transactions - - if(itr->getStartInclusive() <= t){ - // Calculate limit for iterating in this node - const uint64_t endpoint_exclusive = std::min(itr->getEndExclusive(), end_exclusive); - - // Iterate through necessary ticks in this node. - // Wait for it to be loaded first - if(itr->getMutex().try_lock() == false){ - std::cout << "*** Waiting for node to finish loading..." << itr->stringize() << std::endl; - itr->getMutex().lock(); - } - - auto tick_itr = itr->getTickData(t); - const auto tick_itr_end = itr->getTickDataEnd(); - sparta_assert(tick_itr != tick_itr_end); - const Node::TickData* td = &(*tick_itr); - sparta_assert(td->tick_offset + itr->getStartInclusive() <= t); - while(t < endpoint_exclusive && tick_itr != itr->getTickDataEnd()){ - if(t > tick_itr->tick_offset + itr->getStartInclusive()){ - // Current callback tick has passed this tick iterator. - tick_itr++; - } - if(tick_itr != tick_itr_end && tick_itr->tick_offset + itr->getStartInclusive() <= t){ - // Current callback tick has caught up with the tick iterator. Point - // "td" to the current iterator's TickData because it is at or before - // the current callback time (t) - td = &(*tick_itr); - } - - // VERY SLOW SANITY CHECKING. - // Ensures all valid transactions for the current callback tick (t). - // Re-enable only for debugging. - //for(uint32_t loc = 0; loc < num_locations_; loc++){ - // interval_idx idx = td->data[loc]; - // if(idx != NO_TRANSACTION){ - // const Transaction* trans = &itr->getIntervals()[idx]; - // sparta_assert(trans->getLeft() <= t && trans->getRight() > t); - // } - //} - - cb(user_data, t, td->data.data(), &itr->getIntervals()[0], num_locations_); - ++t; - } - - // Finish up callbacks with null data because there is no more tick data in this node - if(tick_itr == itr->getTickDataEnd()){ - while(t < endpoint_exclusive){ - cb(user_data, t, nullptr, nullptr, 0); - ++t; - } - } - - itr->getMutex().unlock(); - - // Has t passed the end time of the query within the file range? - // Note that the query endpoint is inclusive, so t must get to end_exclusive - if(t >= end_exclusive){ - sparta_assert(t == end_exclusive); - - // Now make some callbacks with no data for the rest of the - // requested range so that the viewer can just blank out the - // remaining transactions - while(t <= _end_inclusive){ - cb(user_data, t, nullptr, nullptr, 0); - ++t; - } - - in_query_ = false; - - verifyValidWindow_(); // Check that the window was properly updated - - return; // Done - } - - // Falling through to here meant that iteration reached the end - // of this node - sparta_assert(t == itr->getEndExclusive()); - }else{ - in_query_ = false; - - // This probably indicates that window was wrong and something - // should have been loaded for this query. - sparta_assert(false, - "Exceeded end of blocks at " << t << " where block start is " - << itr->getStartInclusive()); - } - } - - in_query_ = false; - sparta_assert(false, - "Unexpected end of iteration when querying for " << _start_inclusive - << " to " << _end_inclusive << " clamped down to [" << start_inclusive - << ", " << end_exclusive << ")"); - } - - /*! - * \brief Get the inclusive start cycle in the event file - */ - uint64_t getFileStart() const { - return start_tick_; - } - - /*! - * \brief Get the exclusive end cycle in the event file - */ - uint64_t getFileEnd() const { - // We only need to acquire a lock if there's a chance that the value of end_tick_ could change. - // This can only happen if update_enabled_ == true - if(update_enabled_) - { - std::lock_guard lock(node_list_mutex_); - } - return end_tick_; - } - - /*! - * \brief Get the inclusive start cycle in the currently loaded window - */ - uint64_t getWindowStart() const { - return window_.start; - } - - /*! - * \brief Get the exclusive end cycle in the currently loaded window - */ - uint64_t getWindowEnd() const { - return window_.end; - } - - uint32_t getFileVersion() const { - return smart_reader_.reader.getVersion(); - } - - std::ostream& writeNodeStates(std::ostream& o) const { - std::lock_guard lock(node_list_mutex_); - - uint32_t idx = 0; - for(const auto& n : nodes_){ - o << std::setw(5) << idx << ' ' << n.stringize() << std::endl; - ++idx; - } - return o; - } - - std::string getNodeStates() const { - std::ostringstream ss; - writeNodeStates(ss); - return ss.str(); - } - - std::string getNodeDump(const uint32_t node_idx, - const uint32_t location_start = 0, - const uint32_t location_end = 0, - const uint32_t tick_entry_limit = 0) const { - uint32_t idx = 0; - for(const auto& n : nodes_){ - if(idx == node_idx){ - return n.getContentString(location_start, location_end, tick_entry_limit); - } - ++idx; - } - return ""; - } - - std::string stringize() const { - std::lock_guard lock(node_list_mutex_); - - std::ostringstream ss; - ss << ""; - return ss.str(); - } - - uint64_t getSizeInBytes() const { - std::lock_guard lock(node_list_mutex_); - return std::accumulate(nodes_.begin(), - nodes_.end(), - 0, - [](const uint64_t sum, const Node& n){ return sum + n.getSizeInBytes(); }); - } - - bool isFileUpdated(const bool force = false) - { - const time_t cur_time = time(NULL); - if(!force) - { - if((cur_time - last_updated_) >= DB_UPDATE_INTERVAL_S) - { - last_updated_ = cur_time; - } - else - { - return false; - } - } - else - { - last_updated_ = cur_time; - } - const bool result = smart_reader_.isUpdated(); - if(result) - { - smart_reader_.ackUpdated(); - } - return result; - } - - bool updateReady() - { - std::lock_guard lock(node_list_mutex_); - - return update_ready_ > 0; - } - - void ackUpdate() - { - std::lock_guard lock(node_list_mutex_); - - if(update_ready_ > 0) - { - update_ready_--; - } - } - - void enableUpdate() - { - std::lock_guard lock(node_list_mutex_); - - update_enabled_ = true; - } - - void disableUpdate() - { - std::lock_guard lock(node_list_mutex_); - - update_enabled_ = false; - } - - void forceUpdate() - { - std::lock_guard lock(node_list_mutex_); - - if(isFileUpdated(true)) - { - end_tick_ = smart_reader_.reader.getCycleLast(); - unload(); - } - } - -private: - - /*! - * \brief Backround thread loader - */ - void backgroundLoader_() { - // NOTE: cannot perform this check immediately since the background_loader_ object might not - // be complete - //// Ensue that this function is called only in the background loader thread - //sparta_assert(background_loader_.get_id() == std::this_thread::get_id(), - // background_loader_.get_id() << " != " std::this_thread::get_id()); - - while(true) { - std::this_thread::sleep_for(std::chrono::milliseconds(BACKGROUND_THREAD_SLEEP_MS)); - if(background_thread_should_exit_){ - return; // End of thread - } - - if(node_list_mutex_.try_lock()){ - bool needs_update = false; - if(update_enabled_ && isFileUpdated()) - { - end_tick_ = smart_reader_.reader.getCycleLast(); - unload(); - needs_update = true; - } - - try{ - sparta_assert(window_.start % node_size_ == 0); - sparta_assert(window_.end % node_size_ == 0); - - // Pick an end to load on based on how close the last - // query window was to the loaded window edges - const uint64_t low_distance = (last_query_.start <= start_tick_) ? 0 : last_query_.start - window_.start; - const uint64_t high_distance = (last_query_.end >= end_tick_) ? 0 : window_.end - last_query_.end; - - // Did we hit the memory ceiling for this window. If so, loading new nodes will - // be prevented unless there is a significant imbalance in the window size - // around the last query. - // Note that we're guaranteed to contain all necessary data from the last query - // in the window because that is handled in the foreground. Background loading - // (here) is for additional window loading only - const bool hit_memory_ceiling = getSizeInBytes() >= MEMORY_THRESHOLD_BYTES; - - static constexpr uint64_t INVALID_NODE = std::numeric_limits::max(); - uint64_t load_node_pos = INVALID_NODE; - const uint64_t low_pos = window_.start - node_size_; - const uint64_t high_pos = window_.end; - const bool low_limited = window_.start <= start_tick_; // No more to load on the low-end, window reached start_tick - const bool high_limited = window_.end >= end_tick_; // No more to load on the high-end, window reached end_tick - if(low_distance < high_distance && !low_limited){ - // Closer to low side and there is data to load here - // If we hit the memory ceiling, ensure that loading on the low-end of the - // window (and evicting the high end) will not make the high-end closer to - // the last query end and cause a ping-pong loading effect. - if(!hit_memory_ceiling || (low_distance + node_size_ < high_distance)){ - load_node_pos = low_pos; - } - }else if(high_distance > 0 && !high_limited){ - // Closer on high side and there is data to load here - // If we hit the memory ceiling, ensure that loading on the high-end of the - // window (and evicting the low end) will not make the low-end closer to the - // last query start and cause a ping-pong loading effect. - if(!hit_memory_ceiling || (high_distance + node_size_ < low_distance)){ - load_node_pos = high_pos; - } - } - - //int64_t dist_diff; - //if(high_distance == ~(uint64_t)0){ - // if(low_distance == ~(uint64_t)0){ - // dist_diff = 0; // Full file is loaded. Do not evict anything - // }else{ - // dist_diff = low_distance; // window reaches upper limit. Difference is lower_distance - // } - //}else if(low_distance == ~(uint64_t)0){ - // dist_diff = high_distance; - //}else{ - // dist_diff = std::abs(((int64_t)high_distance) - ((int64_t)low_distance)); - //} - const int64_t dist_diff = std::abs(static_cast(high_distance) - - static_cast(low_distance)); - - if(load_node_pos < INVALID_NODE){ - // Check whether it is feasible to add more nodes - if(hit_memory_ceiling){ - // Exceeded memory threshold. May need to load new - // blocks, but others must be dropped first. Determine which - // to add and which can be dropped. First, a switch cannot be made - // unless one side is much further from the last query than the other. - // Otherwise, the extra node will be moved from the low to high end - // every time. - - //std::cout << "|<--" << low_distance << "--[" << last_query_.start << ", " - // << last_query_.end << "]--" << high_distance << "-->| (dist_diff=" - // << dist_diff << ")" << std::endl; - - if(nodes_.size() == 1 || - dist_diff <= static_cast(node_size_)){ - // Cannot possibly drop anything and maintain window, so cannot load anything new - load_node_pos = INVALID_NODE; - }else{ - //if(verbose_){ - // std::cout << "(background)" << window_.start << "|<--" << low_distance << "--[" << last_query_.start << ", " - // << last_query_.end << "]--" << high_distance << "-->|" << window_.end << " (dist_diff=" - // << dist_diff << ")" << std::endl; - //} - // See if the opposite block is outside of the query range - if(load_node_pos == low_pos && (!high_limited || high_distance >= 2*node_size_)){ - // Get opposite node - const node_iterator_t delitr = findNode(window_.end - 1); - if(nodes_.end() != delitr){ - if(delitr->isComplete() && delitr->getStartInclusive() >= last_query_.end){ - if(verbose_){ - std::cout << "(background) removing node:" - << delitr->stringize() << std::endl; - } - window_.end = delitr->getStartInclusive(); // Move end lower - nodes_.erase(delitr); - }else{ - if(verbose_){ - std::cout << "(background) want to slide left, but cannot delete " - << delitr->stringize() << std::endl; - } - load_node_pos = INVALID_NODE; // Cannot delete this node - } - }else{ - if(verbose_){ - std::cout << "(background) want to slide left, but cannot find node containing " - << window_.end - 1 << std::endl; - } - load_node_pos = INVALID_NODE; // Cannot delete this node - } - }else if(!low_limited || low_distance >= 2*node_size_){ // given: load_node_pos == high_pos - const node_iterator_t delitr = findNode(window_.start); - if(nodes_.end() != delitr){ - if(delitr->isComplete() && delitr->getEndExclusive() <= last_query_.start){ - if(verbose_){ - std::cout << "(background) removing node:" - << delitr->stringize() << std::endl; - } - window_.start = delitr->getEndExclusive(); // Move start higher - nodes_.erase(delitr); - }else{ - if(verbose_){ - std::cout << "(background) want to slide right, but cannot delete " - << delitr->stringize() << std::endl; - } - load_node_pos = INVALID_NODE; // Cannot delete this node - } - }else{ - if(verbose_){ - std::cout << "(background) want to slide right, but cannot find node containing " - << window_.start << std::endl; - } - load_node_pos = INVALID_NODE; // Cannot delete this node - } - }else{ - load_node_pos = INVALID_NODE; - } - } - } - } - - if(load_node_pos < INVALID_NODE){ - if(verbose_){ - std::cout << "(background) memory use is " << getSizeInBytes() / 1000000000.0 << " GB" << std::endl; - } - - const node_iterator_t itr = findNode(load_node_pos); - - nodes_.emplace(itr, load_node_pos, node_size_, num_locations_); - Node& added = *std::prev(itr); - if(verbose_){ - std::cout << "(background) inserting Node @ " << load_node_pos << " size " << node_size_ << std::endl; - } - - // Update window to contain this new node - if(window_.start > load_node_pos){ - window_.start = load_node_pos; - } - - if(window_.end < load_node_pos + node_size_){ - window_.end = load_node_pos + node_size_; - } - - // Defferred read to avoid walking over the same chunks when - const uint64_t chunk_start = chunk_size_ * (load_node_pos/chunk_size_); - if(verbose_){ - std::cout << "(background) loading @ " << chunk_start << " size " << chunk_size_ << std::endl; - } - - node_list_mutex_.unlock(); - - std::vector added_nodes{&added}; - smart_reader_.lock(); - - double t_start = 0; - if(verbose_){ - t_start = sparta::TimeManager::getTimeManager().getAbsoluteSeconds(); - } - - smart_reader_.loadDataToNodes(chunk_start, chunk_start + chunk_size_, &added_nodes); - if(verbose_){ - const auto t_delta = sparta::TimeManager::getTimeManager().getAbsoluteSeconds() - t_start; - std::cout << "(background) took " << t_delta << " seconds" << std::endl; - } - - added.markComplete(); - if(verbose_){ - std::cout << "(background) marking complete: " << added.stringize() << std::endl; - } - - smart_reader_.unlock(); - - node_list_mutex_.lock(); - - if(verbose_){ - std::cout << "(background) transactiondb: " << stringize() << std::endl; - std::cout << "(background) " << window_.start << "|<--" << low_distance << "--[" << last_query_.start << ", " - << last_query_.end << "]--" << high_distance << "-->|" << window_.end << " (dist_diff=" - << dist_diff << ")" << std::endl; - } - } - }catch(...){ - node_list_mutex_.unlock(); - throw; - } - - verifyValidWindow_(); - - if(update_enabled_ && needs_update) - { - update_ready_++; - } - - node_list_mutex_.unlock(); - } - } - } - - /*! - * \brief - * \pre Requiresthe node_list_mutex_ - */ - void verifyValidWindow_() - { - // Verify that the update algorithm updated the window range correctly - if(nodes_.size() > 1){ - sparta_assert(nodes_.begin()->getStartInclusive() == window_.start); - sparta_assert(nodes_.rbegin()->getEndExclusive() == window_.end); - } - } - - /*! - * \brief Loads data necessary to populate the given range - */ - void load_(const uint64_t start_inclusive, const uint64_t end_exclusive) - { - if(verbose_){ - std::cout << "(main) Attempting to load [" << start_inclusive << ", " - << end_exclusive << ")" << std::endl; - } - - std::lock_guard lock(node_list_mutex_); - - sparta_assert(start_inclusive >= start_tick_); - sparta_assert(end_exclusive <= end_tick_); - - // Find nodes. Round down to nearest chunk to start - //uint64_t chunk_start = uint64_t(start_inclusive / chunk_size_) * chunk_size_; - const uint64_t load_start = uint64_t(start_inclusive / node_size_) * node_size_; - - // cur_pos is current loading tick. It increases in multiples of - // chunk_size_. As loaded chunks are encountered or new chunks are loaded, - // cur_pos is increased. - uint64_t cur_pos = load_start; - - //std::cout << "Loading [" << start_inclusive << "," << end_exclusive << ")" - // << " => starting with "<< chunk_start << std::endl; - - // This scope block ensures we don't try to reuse itr after we're finished with it - { - // Flag any nodes before the load range for deletion - node_iterator_t itr = nodes_.begin(); - if(itr != nodes_.end()){ - // Window begins where there is valid data - window_.start = itr->getStartInclusive(); - for(; itr != nodes_.end(); ++itr) { - if(itr->getEndExclusive() <= cur_pos){ - if(verbose_){ - std::cout << "(main) Flagging for deletion: " << itr->stringize() << std::endl; - } - itr->flagForDeletion(); - }else{ - // Stop walking through these nodes because this one is after - // (or overlaps) the current load position - break; - } - } - }else{ - // Window begins where data will first be loaded - window_.start = cur_pos; - } - - // Insert nodes until cur_pos reaches the end of the range - std::vector chunks_to_read; // Chunks containing newly created nodes - std::vector added_nodes; - while(cur_pos < end_exclusive){ - - // Incomplete nodes are ok. Just don't read their data - //// Check for incomplete (BAD) nodes. - //// These can occur if an exception occurs during loading - //if(itr != nodes_.end() && itr->isComplete() == false){ - // // Remove this incomplete node - // std::cerr << "Incomplete node encountered: " << itr->stringize() - // << ", removing and reloading." << std::endl; - // node_iterator_t temp = itr; - // ++itr; // Move to next, allowing a replacement to be added - // nodes_.erase(temp); - //} - - // Add or skip the current element - if(itr == nodes_.end() || - cur_pos < itr->getStartInclusive()){ - // Assuming exclusive right endpoint of transactions - //nodes_.emplace(itr, Node(cur_pos, chunk_size_, num_locations_)); - sparta_assert(cur_pos % node_size_ == 0); - const auto new_itr = nodes_.emplace(itr, cur_pos, node_size_, num_locations_); - added_nodes.push_back(&*(new_itr)); - if(verbose_) { - std::cout << "(main) Inserting Node @ " << cur_pos << " size " << node_size_ << std::endl; - } - - // Update window - if(window_.start > cur_pos){ - window_.start = cur_pos; - } - - // Defferred read to avoid walking over the same chunks when - const uint64_t chunk_start = chunk_size_ * (cur_pos/chunk_size_); - if(chunks_to_read.size() == 0 or chunks_to_read[chunks_to_read.size() - 1] != chunk_start){ - chunks_to_read.push_back(chunk_start); - } - }else{ - if(verbose_){ - std::cout << "(main) Skipping insertion @ " << cur_pos << " ended=" - << (itr==nodes_.end()) << ", node start=" << itr->getStartInclusive() << std::endl; - } - ++itr; - } - cur_pos += node_size_; - } - - smart_reader_.lock(); - - // Load the chunks which broadcast to 1 or more Nodes - // Because node_size_ is an even division of chunk_size_, only the chunk - // at chunk_start must be read - for(uint64_t chunk_start : chunks_to_read) { - double t_start = 0; - if(verbose_){ - std::cout << "(main) Loading @ " << chunk_start << " size " << chunk_start + chunk_size_ << std::endl; - t_start = sparta::TimeManager::getTimeManager().getAbsoluteSeconds(); - } - smart_reader_.loadDataToNodes(chunk_start, chunk_start + chunk_size_, &added_nodes); - if(verbose_){ - auto t_delta = sparta::TimeManager::getTimeManager().getAbsoluteSeconds() - t_start; - std::cout << "(main) took " << t_delta << " seconds" << std::endl; - } - } - - // Mark fully loaded, newly constructed nodes as valid - for(auto& n : added_nodes){ - n->markComplete(); - if(verbose_){ - std::cout << "(main) marking complete: " << n->stringize() << std::endl; - } - } - - smart_reader_.unlock(); - - if(verbose_){ - std::cout << "(main) Added nodes marked as complete" << std::endl; - } - - // Walk through nodes out of the load range and flag them for deletion - if(itr != nodes_.end()) { - if(cur_pos == itr->getStartInclusive()) { - // Contiguous nodes, flag for deletion - - for(; itr != nodes_.end(); ++itr) { - if(verbose_) { - std::cout << "(main) Flagging for deletion: " << itr->stringize() << std::endl; - } - itr->flagForDeletion(); - cur_pos = itr->getEndExclusive(); - } - } - else{ - // Non-contiguous nodes. Gaps are not allowed. Remove immediately - if(verbose_) { - while(itr != nodes_.end()) { - std::cout << "(main) Erasing non-contiguous node following data: " - << itr->stringize() << std::endl; - itr = nodes_.erase(itr); - } - } - else { - // Use the range erase method if we don't need to print any messages - nodes_.erase(itr, nodes_.end()); - } - } - } - // We should be done with itr now, so we don't bother updating it in the above block - } - - // New cur pos at the end of the data - window_.end = cur_pos; - - // Verify window endpoints surround load endpoints - sparta_assert(window_.start <= start_inclusive); - sparta_assert(window_.end >= end_exclusive); - - // Clean up deleted nodes for now. Result must be a contiguous series of - // nodes, implying that cleanup can only remove from the ends. - // Also adjusts window_ - // - // This will be done by a thread later and controlled by a memory and - // idle threshold - node_iterator_t n = nodes_.begin(); - bool keeper_encountered = false; // Have any nodes been kept yet? - while(n != nodes_.end()){ - if(n->canDelete()){ - // Encountered a node flagged for deletion - if(verbose_){ - std::cout << "(main) CAN delete node " << n->stringize() << std::endl; - } - if(keeper_encountered){ - // Move window end to beginning of this node - // This should be hit on the first node after a stream of 1 - // or more nodes that were kept (not deleted) - if(window_.end > n->getStartInclusive()){ - window_.end = n->getStartInclusive(); - } - } - - const node_iterator_t temp = n; - ++n; - nodes_.erase(temp); - }else{ - if(verbose_){ - std::cout << "(main) can NOT delete node " << n->stringize() << std::endl; - } - // Encountered first node that will be kept in the list - if(!keeper_encountered){ - // This is the first node to keep. - // Move window start to start of this node - window_.start = n->getStartInclusive(); - } - - ++n; - keeper_encountered = true; - } - } - - // In the case where this happens (whicih it really shouldn't), reset - // the window so the next load will be forced to load everything - if(nodes_.size() == 0){ - window_.start = 0; - window_.end = 0; - } - - if(verbose_){ - std::cout << "(main) transactiondb: " << stringize() << std::endl; - std::cout << "(main) " << window_.start << "|<--" << " ... " << "--[" << last_query_.start << ", " - << last_query_.end << "]--" << " ... " << "-->|" << window_.end << std::endl; - } - } - - /*! - * \brief Finds first node containing tick or, if no nodes contain tick, the - * first node node after the tick. - */ - node_iterator_t findNode(const uint64_t tick) - { - node_iterator_t n = nodes_.begin(); - if(n == nodes_.end()){ - return n; - } - if(n->getStartInclusive() > tick){ - return nodes_.begin(); - } - for(; n != nodes_.end(); ++n){ - if(n->getEndExclusive() > tick){ - return n; - } - } - return nodes_.end(); - } -}; - -} // sparta::pipeViewer - diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/TransactionInterval.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/TransactionInterval.hpp deleted file mode 100644 index 87cc954c2d..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/TransactionInterval.hpp +++ /dev/null @@ -1,263 +0,0 @@ - -/** - * \file TransactionInterval.h - * - * \copyright - */ - -#pragma once - -#include -#include -#include -#include -#include "sparta/pairs/PairFormatter.hpp" -#include "sparta/utils/SpartaAssert.hpp" - -namespace sparta::pipeViewer { -template -class transactionInterval { -private: - const Dat_t time_Start_; /*! Left Boundary of Interval */ - const Dat_t time_End_; /*! Right Boundary of Interval */ - - transactionInterval(const Dat_t lval, - const Dat_t rval, - const uint16_t cpid, - const uint64_t trid, - const uint64_t dispid, - const uint64_t lctn, - const uint16_t flgs, - const uint64_t ptid, - const uint32_t opcd, - const uint64_t vadr, - const uint64_t radr, - const uint16_t lngt, - std::string&& iannt, - const uint16_t pair_id, - const std::vector& sz, - const std::vector>& vals, - const std::vector& nams, - const std::vector& str, - const PairFormatterVector& del) : - time_Start_(lval), - time_End_(rval), - control_ProcessID(cpid), - transaction_ID(trid), - display_ID(dispid), - location_ID(lctn), - flags(flgs), - parent_ID(ptid), - operation_Code(opcd), - virtual_ADR(vadr), - real_ADR(radr), - length(lngt), - annt(std::move(iannt)), - pairId(pair_id), - sizeOfVector(sz), - valueVector(vals), - nameVector(nams), - stringVector(str), - delimVector(del) - { - sparta_assert( time_Start_ <= time_End_); - } - - transactionInterval(const Dat_t lval, - const Dat_t rval, - const uint16_t cpid, - const uint64_t trid, - const uint64_t dispid, - const uint64_t lctn, - const uint16_t flgs, - const uint64_t ptid, - const uint32_t opcd, - const uint64_t vadr, - const uint64_t radr, - const uint16_t lngt, - std::string&& iannt) : - transactionInterval(lval, - rval, - cpid, - trid, - dispid, - lctn, - flgs, - ptid, - opcd, - vadr, - radr, - lngt, - std::move(iannt), - 0, - {}, - {}, - {}, - {}, - {}) - { - } - -public: - using IntervalDataT = Dat_t; - const uint16_t control_ProcessID; /*! Core ID*/ - const uint64_t transaction_ID; /*! Transaction ID*/ - const uint64_t display_ID; /*! Use to control display character and color */ - const uint16_t location_ID; /*! Location ID*/ - const uint16_t flags; /*! Assorted Transaction Flags*/ - const uint64_t parent_ID; /*! Parent Transaction ID*/ - const uint32_t operation_Code; /*! Operation Code*/ - const uint64_t virtual_ADR; /*! Virtual Address*/ - const uint64_t real_ADR; /*! Real Address*/ - const uint16_t length; /*! Annotation Length or Name Value Pair count*/ - const std::string annt; /*! Annotation Pointer*/ - const uint16_t pairId; /*! Unique id required for pipeline collection*/ - const std::vector sizeOfVector; /*! Vector of integers representing Sizeof of every field */ - const std::vector> valueVector; /*! Vector of integers containing the actual data of every field */ - const std::vector nameVector ; /*! Vector of strings containing the actual Names of every field */ - const std::vector stringVector; /*! Vector of strings containing the actual string value of every field */ - const PairFormatterVector delimVector; - - // Constructor for transaction_t - transactionInterval( const Dat_t lval, const Dat_t rval, - const uint16_t cpid, const uint64_t trid, const uint64_t dispid, - const uint64_t lctn, const uint16_t flgs) : - transactionInterval(lval, rval, cpid, trid, dispid, lctn, flgs, 0, 0, 0) - { - } - - // Constructor for annotation_t - transactionInterval( const Dat_t lval, const Dat_t rval, - const uint16_t cpid, const uint64_t trid, const uint64_t dispid, - const uint64_t lctn, const uint16_t flgs, - const uint64_t ptid, const uint16_t lngt, std::string iannt) : - transactionInterval(lval, - rval, - cpid, - trid, - dispid, - lctn, - flgs, - ptid, - 0, - 0, - 0, - lngt, - std::move(iannt)) - { - sparta_assert(lngt!=0); - } - - // Constructor for instrucation_t - transactionInterval( const Dat_t lval, const Dat_t rval, - const uint16_t cpid, const uint64_t trid, const uint64_t dispid, - const uint64_t lctn, const uint16_t flgs, - const uint64_t ptid, const uint32_t opcd, - const uint64_t vadr, const uint64_t radr) : - transactionInterval(lval, - rval, - cpid, - trid, - dispid, - lctn, - flgs, - ptid, - opcd, - vadr, - radr, - 0, - "") - { - } - - // Constructor for memoryoperation_t - transactionInterval( const Dat_t lval, const Dat_t rval, - const uint16_t cpid, const uint64_t trid, const uint64_t dispid, - const uint64_t lctn, const uint16_t flgs, - const uint64_t ptid, const uint64_t vadr, - const uint64_t radr) : - transactionInterval(lval, - rval, - cpid, - trid, - dispid, - lctn, - flgs, - ptid, - 0, - vadr, - radr) - { - } - - // Constructor for pair_t - transactionInterval(const Dat_t lval, const Dat_t rval, - const uint16_t cpid, const uint64_t trid, const uint64_t dispid, - const uint64_t lctn, const uint16_t flgs, - const uint64_t ptid, const uint16_t lngt, - const uint16_t pair_id, const std::vector& sz, - const std::vector>& vals, - const std::vector& nams, - const std::vector& str, - const PairFormatterVector& del) : - transactionInterval(lval, - rval, - cpid, - trid, - dispid, - lctn, - flgs, - ptid, - 0, - 0, - 0, - lngt, - "", - pair_id, - sz, - vals, - nams, - str, - del) - { - } - - // Copy Constructor - transactionInterval(const transactionInterval& rhp) = default; - - // Move Constructor - transactionInterval(transactionInterval&& rhp) = default; - - /*! - * \brief Compute size of this interval including memory allocated for the annotation or the memory allocated by all the values for the pair - */ - uint64_t getSizeInBytes() const { - uint64_t size = sizeof(*this); - if(!annt.empty()) { - size += length; - } - else { - size = std::accumulate(std::next(sizeOfVector.begin()), sizeOfVector.end(), size); - } - return size; - } - - /*! Function: Return the Left value of the event*/ - Dat_t getLeft() const noexcept{ return( time_Start_ ); } - /*! Function: Return the Right value of the event*/ - Dat_t getRight() const noexcept{ return( time_End_ ); } - - /*! Function: Check if &V is within the event range*/ - bool contains(const Dat_t V) const noexcept { - return( (V >= time_Start_) && ( V < time_End_) ); - } - - /*! Function: Check if &l,&r are within the event range*/ - bool containsInterval(const Dat_t l, const Dat_t r) const noexcept { - return ( (time_Start_ <= l) && (time_End_ >= r) ); - } -}; // transactionInterval - -}//NAMESPACE:sparta::pipeViewer - -#pragma once diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/common.pxd b/helios/pipeViewer/pipe_view/transactiondb/src/common.pxd deleted file mode 100644 index cd92d02782..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/common.pxd +++ /dev/null @@ -1,59 +0,0 @@ -# distutils: language = c++ - - -## @package Common files for transction database interface - -# import some common integer and string c++ type definitions - -import sys -sys.path.append("../../") -from libcpp.vector cimport vector -from libcpp.utility cimport pair -from libcpp.string cimport string -from libc.stdint cimport * - -cdef extern from "helpers.hpp" namespace "transactiondb": - - ctypedef long ptr_t - -#ctypedef ptr_t void_ptr "void*" -#cdef ptr_t void_ptr "void*" - -cdef extern from "sparta/pipeViewer/transaction_structures.hpp": - - cdef extern int ANNOTATION "is_Annotation" - cdef extern int INSTRUCTION "is_Instruction" - cdef extern int MEMORY_OP "is_MemoryOperation" - cdef extern int PAIR "is_Pair" - -cdef extern from "TransactionInterval.hpp" namespace "sparta::pipeViewer": - - cdef cppclass c_TransactionInterval_uint64_t "sparta::pipeViewer::transactionInterval": - - c_TransactionInterval_uint64_t(c_TransactionInterval_uint64_t) # Const ref arg - - const uint16_t control_ProcessID - const uint64_t transaction_ID - const uint64_t display_ID - const uint32_t location_ID - const uint16_t flags - const uint64_t parent_ID - const uint32_t operation_Code - const uint64_t virtual_ADR - const uint64_t real_ADR - const uint16_t length - const string annt - const uint16_t pairId - const vector[uint16_t] sizeOfVector - const vector[pair[uint64_t, bint]] valueVector - const vector[string] nameVector - const vector[string] stringVector - const vector[string] delimVector - - uint64_t getLeft() # const - uint64_t getRight() # const - bint contains(uint64_t V) # const # V is a const ref - bint containsInterval(uint64_t l, uint64_t r) # l and r are const refs - -cdef extern from *: - ctypedef c_TransactionInterval_uint64_t c_TransactionInterval_uint64_const_t "sparta::pipeViewer::transactionInterval const" diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/helpers.hpp b/helios/pipeViewer/pipe_view/transactiondb/src/helpers.hpp deleted file mode 100644 index a078900c83..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/helpers.hpp +++ /dev/null @@ -1,13 +0,0 @@ - -/*! - * \file helpers.hpp - * \brief Helpers for wrapper of transaction database structures - */ - -#pragma once - -namespace transactiondb { - - //! \brief Target-dependent pointer type for use by Cython - using ptr_t = void*; -} diff --git a/helios/pipeViewer/pipe_view/transactiondb/src/transactiondb.pyx b/helios/pipeViewer/pipe_view/transactiondb/src/transactiondb.pyx deleted file mode 100644 index de9858926e..0000000000 --- a/helios/pipeViewer/pipe_view/transactiondb/src/transactiondb.pyx +++ /dev/null @@ -1,738 +0,0 @@ -# distutils: language = c++ - - -## @package Setup file for transactiondb Python module - -import sys -import re - -from common cimport * -from libcpp.vector cimport vector -from libcpp.utility cimport pair -from libcpp.string cimport string -import inspect - -cdef extern from "TransactionDatabaseInterface.hpp" namespace "sparta::pipeViewer": - - ctypedef uint32_t const_interval_idx "sparta::pipeViewer::TransactionDatabaseInterface::const_interval_idx const" - ##ctypedef callback_fxn "sparta::pipeViewer::TransactionDatabaseInterface::callback_fxn" - ##cdef void (__stdcall *callback_fxn)(void_ptr obj, \ - ## uint64_t tick, \ - ## const_interval_idx * content, \ - ## c_TransactionInterval_uint64_t* transactions, \ - ## uint32_t content_len) - - cdef cppclass c_TransactionDatabaseInterface "sparta::pipeViewer::TransactionDatabaseInterface": - c_TransactionDatabaseInterface(string, uint32_t, bint) except +IOError # Can throw if file not opened - - void unload() - void resetQueryState() - void query(const uint64_t start_inc, \ - const uint64_t end_inc, \ - void (*cb)(void* obj, \ - uint64_t tick, \ - const_interval_idx * content, \ - c_TransactionInterval_uint64_const_t* transactions, \ - uint32_t content_len) except +, \ - void* user_data, \ - const bint modify_tracking) except + - - uint64_t getFileStart() - uint64_t getFileEnd() - uint64_t getWindowStart() - uint64_t getWindowEnd() - - uint32_t getFileVersion() # const - - uint32_t getNodeLength() # const - uint64_t getChunkSize() # const - - void setVerbose(bint verbose) - bint getVerbose() # const - - string getNodeStates() - string getNodeDump(uint32_t node_idx, \ - uint32_t location_start, \ - uint32_t location_end, \ - uint32_t tick_entry_limit) # const - string stringize() - uint64_t getSizeInBytes() - bint updateReady() - void ackUpdate() - void enableUpdate() - void disableUpdate() - void forceUpdate() - -cdef extern from "TransactionDatabaseInterface.hpp" namespace "sparta::pipeViewer::TransactionDatabaseInterface": - const_interval_idx NO_TRANSACTION # static const - -cdef extern from "Reader.hpp" namespace "sparta::pipeViewer": - string formatPairAsAnnotation(const uint64_t transaction_ID, \ - const uint64_t display_ID, \ - const uint16_t length, \ - const vector[string]& nameVector, \ - const vector[string]& stringVector); - -cdef class Transaction: - - FLAGS_MASK_TYPE = 0b111 - CONTINUE_FLAG = 0x10 - ANNOTATION_TYPE_STR = 'annotation' - INSTRUCTION_TYPE_STR = 'instruction' - MEMORY_OP_TYPE_STR = 'memory_op' - PAIR_TYPE_STR = 'pair' - - cdef c_TransactionInterval_uint64_const_t * __trans # Pointer to transaction data - cdef bint __is_proxy # Is this transaction a proxy, or does it own a copy - - def __cinit__(self, *args): - self.__trans = NULL - self.__is_proxy = False - - def __init__(self, trans_ptr, is_proxy): - """Internally construct a new transaction object (or proxy) - DO NOT CALL this method outside of transaction database code. - - trans_ptr: Pointer (integer) to transaction object or None - is_proxy: Is this object a proxy, If False, Makes a new Transaction - based on trans_ptr and trans_ptr must not be None/0. If True, - trans_ptr may be None or any integer - """ - if (trans_ptr is not None) and (not isinstance(trans_ptr, (int, long))): - raise TypeError('trans_ptr must be either None or an integer or long to be converted to a C pointer, is type {0}' \ - .format(type(trans_ptr))) - if not isinstance(is_proxy, bool): - raise TypeError('is_proxy must be a bool, is type {0}'.format(type(is_proxy))) - - self.__is_proxy = is_proxy - if self.__is_proxy == False: - if trans_ptr is None or trans_ptr == 0: - raise ValueError('is_proxy was False but trans_ptr did not refer to a valid object, was {0}' \ - .format(trans_ptr)) - - # Not a proxy, make and own a copy - #self.__trans = new c_TransactionInterval_uint64_t((trans_ptr)[0]) - self.__trans = new c_TransactionInterval_uint64_t((trans_ptr)[0]) - else: - if trans_ptr is None: - self.__trans = NULL - else: - #self.__trans = trans_ptr - self.__trans = trans_ptr - - - def __dealloc__(self): - if not self.__is_proxy and self.__trans != NULL: - del self.__trans - - def __str__(self): - if self.__trans == NULL: - return '' - - type_flags = self.getFlags() & 0b111 - if type_flags == ANNOTATION: - return '' \ - .format(self.getTransactionID(), self.getLocationID(), self.getLeft(), self.getRight(), \ - self.getParentTransactionID(), self.getAnnotation()) - elif type_flags == INSTRUCTION: - return '' \ - .format(self.getTransactionID(), self.getLocationID(), self.getLeft(), self.getRight(), \ - self.getOpcode(), self.getVirtualAddress(), self.getRealAddress(), \ - self.getParentTransactionID()) - elif type_flags == MEMORY_OP: - return '' \ - .format(self.getTransactionID(), self.getLocationID(), self.getLeft(), self.getRight(), \ - self.getVirtualAddress(), self.getRealAddress(), \ - self.getParentTransactionID()) - elif type_flags == PAIR: - return ''\ - .format(self.getTransactionID(), self.getPairID(), self.getLocationID(), self.getLeft(), self.getRight(),\ - self.getParentTransactionID(), self.getAnnotation()) - else: - return '' \ - .format(self.getTransactionID(), self.getLocationID(), self.getLeft(), self.getRight(), \ - self.getOpcode(), self.getVirtualAddress(), self.getRealAddress(), \ - self.getParentTransactionID()) - - def __repr__(self): - return self.__str__() - - - # Proxy-Related Methods - def makeRealCopy(self): - """This object may only be a proxy (see isProxy) to a transaction, so - modifying or destroying the related IntervalList object can cause this - object to refer to a different transaction, A null-transaction (see - isValid) or even be illegal to access because the proxied memory was - deleted externally - - This method makes a full copy of the current transaction which is - completely independet of any IntervalWindow or IntervalList. This is - the safest way to hold onto a transaction object, but there is memory - and time overhead in making a real copy. - """ - return Transaction(self.__trans, False) - - cdef void _setProxiedTransaction(self, c_TransactionInterval_uint64_const_t* trans): - """Updates the underlying translation proxied by this object. - It is an error to call this method if this Transaction is not a proxy - (see isProxy) - """ - if self.__is_proxy == False: - raise RuntimeError('Cannot set new proxied Transaction on a Transaction object that is not a proxy') - - self.__trans = trans - - cpdef bint isProxy(self): - """Returns True if this object is a proxy for some transaction, False if - it is a copy, which implies it has its own allocated object - """ - return self.__is_proxy - - cpdef bint isValid(self): - """Returns True if this Transaction is currently pointing to a valid - object. If this Transaction is a proxy, this can be True or False. If - this Transaction is not a proxy and is a real transaction copy, - value will always be True. - - If False, do not attempt to call data accessors or copy this object - """ - return self.__trans != NULL - - - # Transaction Attributes - def getComponentID(self): - if self.__trans == NULL: - return None - return self.__trans.control_ProcessID - - def getTransactionID(self): - if self.__trans == NULL: - return None - return self.__trans.transaction_ID - - def getPairID(self): - if self.__trans == NULL: - return None - return self.__trans.pairId - - def getDisplayID(self): - if self.__trans == NULL: - return None - return self.__trans.display_ID - - def getLocationID(self): - if self.__trans == NULL: - return None - return self.__trans.location_ID - - def getFlags(self): - if self.__trans == NULL: - return None - return self.__trans.flags - - def getParentTransactionID(self): - if self.__trans == NULL: - return None - return self.__trans.parent_ID - - def getOpcode(self): - if self.__trans == NULL: - return None - return self.__trans.operation_Code - - def getVirtualAddress(self): - if self.__trans == NULL: - return None - return self.__trans.virtual_ADR - - def getRealAddress(self): - if self.__trans == NULL: - return None - return self.__trans.real_ADR - - def getAnnotationLength(self): - if self.__trans == NULL: - return None - return self.__trans.length - - def getAnnotation(self): - if self.__trans == NULL: - return None - cdef bytes py_str_preamble - cdef bytes py_str_body - cdef int i - - bytes_re = re.compile("b'(.*?)'") # Look for b'xxx' - - py_str_preamble = b'' - py_str_body = b'' - if self.getType() == ANNOTATION: - if not self.__trans.annt.empty(): - value_str = bytes(self.__trans.annt) - if str(value_str).isnumeric(): - hex_string = format(int(value_str) & 0xf, 'x') - my_display_id = "R" + hex_string + hex_string - py_str_body = my_display_id.encode('utf-8') + b' ' + value_str - else: - py_str_body = value_str - else: - py_str_body = formatPairAsAnnotation(self.__trans.transaction_ID, - self.__trans.display_ID, - self.__trans.length, - self.__trans.nameVector, - self.__trans.stringVector) - - # TODO this could already be a string to avoid decoding over and over - - decoded_str = bytes_re.sub(r'\1', (py_str_preamble + py_str_body).decode('utf-8')) # Replace b'xxx' with xxx -# decoded_str = decoded_str.replace('\x00', '') # Remove null bytes - decoded_str = decoded_str.replace(r'\x00', '') # Remove null byte strings - return decoded_str - - def getLeft(self): - """ - Gets the left endpoint of the transaction in hypercycles - """ - return self.__trans.getLeft() - - def getRight(self): - """ - Gets the right endpoint of the transaction in hypercycles - """ - return self.__trans.getRight() - - def getType(self): - """ - Returns the integer type of this transaction extracted from flags - """ - return self.getFlags() & self.FLAGS_MASK_TYPE - - def getTypeString(self): - """ - Returns the string type of this transaction based on getType. - Possible return values are ANNOTATION_TYPE_STR, MEMORY_OP_TYPE_STR, or - INSTRUCTION_TYPE_STR - """ - type_flags = self.getType() - if type_flags == ANNOTATION: - return self.ANNOTATION_TYPE_STR - elif type_flags == INSTRUCTION: - return self.INSTRUCTION_TYPE_STR - elif type_flags == MEMORY_OP: - return self.MEMORY_OP_TYPE_STR - elif type_flags == PAIR: - return self.PAIR_TYPE_STR - else: - raise RuntimeError('Got a transaction type value {0:#x} from flags ' \ - '{1:#x} that did not map to a known transaction type.' \ - .format(type_flags, self.getFlags())) - - def contains(self, hc): - """ - Determines if this Transaction interval contains the given hypercycle - """ - return self.__trans.contains(hc) - - def containsInterval(self, l, r): - """ - Determines if this Transaction interval contains the given interval - defined by endpoint integers [l, r] in hypercycles - """ - return self.__trans.containsInterval(l, r) - - def isContinued(self): - return self.getFlags() & self.CONTINUE_FLAG != 0 - - -cdef class TransactionDatabase: - """ - Represents a sliding window view into a transaction database file. - Can be used to perfom queries of active transactions at a particular - hypercycle (tick number). - """ - - OBJECT_DESTROYED_ERROR = 'Cannot operate on a TransactionDatabase once _destroy()\'ed' - - cdef c_TransactionDatabaseInterface * __window # C implementation. Freed at destruction - cdef object __filename # Name of file/dir containing the transaction database - - cdef const_interval_idx* __cur_content - cdef c_TransactionInterval_uint64_const_t* __cur_transactions - cdef uint32_t __cur_content_len - cdef object __cur_callback - cdef dict __cached_annotations - - cdef Transaction __trans_proxy - - def __cinit__(self, *args, **kwargs): - self.__window = NULL - self.__filename = None - self.__cur_content = NULL - self.__cur_transactions = NULL - self.__cur_content_len = 0 - self.__cur_callback = None - self.__cached_annotations = {} - self.__trans_proxy = Transaction(None, True) # Create a Proxy - - def __init__(self, filename, num_locs, update_enabled): - """ - Create a c_TransactionDatabaseInterface* based on the chosen filename - """ - if not isinstance(filename, (str, unicode)): - raise TypeError('filename must be a str, is type {0}'.format(type(filename))) - - self.__filename = filename - cdef uint32_t c_num_locs = num_locs - cdef char* c_str = filename - cdef string c_s = filename.encode('utf-8') - self.__window = new c_TransactionDatabaseInterface(c_s, num_locs, update_enabled) - - def __dealloc__(self): - self._destroy() - - ## Handles query result callbacks (per tick) from C++ transaction database - # library - cdef void handleTickCallback(self, \ - uint64_t tick, \ - const_interval_idx * content, \ - c_TransactionInterval_uint64_const_t* transactions, \ - uint32_t content_len) except *: - # Store current query results so that Python callback may access them - # NOTE: Must update same set of values as clearCurrentTickContent - self.__cur_content = content - self.__cur_transactions = transactions - self.__cur_content_len = content_len - - # Invoke python callback with current tick and self. Callback will - # access this instance to look into result data - self.__cur_callback(tick, self) - - def clearCurrentTickContent(self): - """ - Clears the all pointers to content of the current tick. This will - result in all queries for a proxied transaction to return None which - effectively allows dummy callbacks with empty data to be made - """ - # NOTE: Must update same set of values ad handleTickCallback - self.__cur_content = NULL - self.__cur_transactions = NULL - self.__cur_content_len = 0 - - def __str__(self): - if self.__window == NULL: - return '' - - ##return '' \ - ## .format(self.__filename, self.getFileStart(), self.getFileEnd(), \ - ## self.getWindowLeft(), self.getWindowRight()) - - #cdef bytes py_s = self.__window.stringize().c_str() - #cdef str s = py_s - s = self.__window.stringize().c_str() - s += ' + {0}B cached annotations ({1})' \ - .format(int(self._getSizeOfCachedAnnotations()), len(self.__cached_annotations)).encode('utf-8') - return s.decode('utf-8') - - def __repr__(self): - return self.__str__() - - def getFileVersion(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getFileVersion() - - def getNodeLength(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getNodeLength() - - def getChunkSize(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getChunkSize() - - def setVerbose(self, bint verbose): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - self.__window.setVerbose(verbose) - - def getVerbose(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return bool(self.__window.getVerbose()) - - def _destroy(self): - """Manually destoy this object by freeing the contained window data - structures. Otherwise, internal memory is freed at Python destruction - (__del__). - - This is meant for use when discarding a window to ensure that the - memory taken up by the interval window is freed regardless of Python - - Calling this method when this IntervalWindow is no longer - needed is recommended. - """ - if self.__window != NULL: - del self.__window - self.__window = NULL - - def getNodeStates(self): - """Returns a string containing state of each node current loaded - """ - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - cdef bytes py_s = self.__window.getNodeStates().c_str() - return py_s.decode('utf-8') - - def getNodeDump(self, node_idx, loc_start=0, loc_end=0, tick_entry_limit=0): - """Gets a string representing a node's content. To fit this on the - screen, the number of locations (length of a row in the output) can be - limited by loc_start and loc_end. The number of tick entries (number of - rows) can be limited by tick_entry_limit - """ - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - cdef bytes py_s = self.__window.getNodeDump(node_idx, loc_start, loc_end, tick_entry_limit).c_str() - return py_s.decode('utf-8') - - def getSizeInBytes(self): - """Returns the current (approximate) memory used by this structure in - bytes - """ - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - cdef uint64_t size = self.__window.getSizeInBytes() - size += self._getSizeOfCachedAnnotations() - return size - - cdef uint64_t _getSizeOfCachedAnnotations(self): - getsizeof = sys.getsizeof - cdef uint64_t size = getsizeof(self.__cached_annotations) - size += sum([getsizeof(a) + getsizeof(b) for (a,b) in self.__cached_annotations.iteritems()]) - return size; - - def unload(self): - """Unloads current windowed data - """ - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - self.__window.unload() - - def query(self, start_inc, end_inc, callback, modify_tracking=True): - """Performs a query in the transaction database at the given tick range - and Makes a callback for each tick - Callback must accept (tick_num, StateSnapshot) - Queries - modify_tracking should be set to false when a query is made that is - smaller than the previous query and should not affect the transactiondb - library's prediction for the next query. Generally, set this to false - when querying specific data within the visual range of data. - """ - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - self.__cur_callback = callback - - try: - self.__window.query(start_inc, end_inc, \ - c_callback, self, \ - modify_tracking) - except: - self.__window.resetQueryState() - raise - - cdef c_TransactionInterval_uint64_const_t* _getTransaction(self, uint32_t c_loc): - if self.__cur_content == NULL: - return NULL - if c_loc >= self.__cur_content_len: - raise IndexError('Cannot access location {0} because only {1} locations are available' \ - .format(c_loc, self.__cur_content_len)) - cdef uint32_t c_trans_idx = self.__cur_content[c_loc] - if c_trans_idx == NO_TRANSACTION: - return NULL - - return &self.__cur_transactions[c_trans_idx] - - ##def getTransactionProxy(self, loc): - ## cdef c_TransactionInterval_uint64_const_t* c_trans - ## c_trans = self._getTransaction(loc) - ## return Transaction(c_trans, True) # Create a Proxy - - ## Gets the transaction proxy for the transaction at the selected - # location based on the latest data received from a query calback - # @return None if there is no current location/transaction data from an - # internal query callback of if there is simply no transction at the - # specified locaiton at the current time - def getTransactionProxy(self, loc): - if loc < 0 or loc > 0xffffffff: - return None # Implicitly invalid - cdef c_TransactionInterval_uint64_const_t* c_trans - c_trans = self._getTransaction(loc) - if c_trans == NULL: - return None - self.__trans_proxy._setProxiedTransaction(c_trans) - return self.__trans_proxy - - def getTransactionAnnotation(self, uint32_t loc): - cdef c_TransactionInterval_uint64_const_t* c_trans - c_trans = self._getTransaction(loc) - if not c_trans: - return None - - cached_str = self.__cached_annotations.setdefault(c_trans.transaction_ID, None) - if cached_str: - return cached_str - - cdef bytes py_str_preamble - cdef bytes py_str_body - cdef int i - - bytes_re = re.compile("b'(.*?)'") # Look for b'xxx' - - py_str_preamble = b'' - py_str_body = b'' - if self.getType() == ANNOTATION: - if not self.__trans.annt.empty(): - value_str = bytes(self.__trans.annt) - value_string = bytes_re.sub('r\1', str(value_str)) # Replace b'xxx' with xxx - hex_string = format(int(value_str) & 0xf, 'x') - my_display_id = "R" + hex_string + hex_string - py_str_body = my_display_id.encode('utf-8') + b' ' + value_str - else: - my_display_id = self.__trans.display_ID if self.__trans.display_ID < 0x1000 else self.__trans.transaction_ID - py_str_preamble += format(my_display_id, '>03x').encode('utf-8') + b' ' - - for i in range(1, self.__trans.length): - my_name = bytes_re.sub(r'\1', str(self.__trans.nameVector[i])) - my_value = bytes_re.sub(r'\1', str(self.__trans.stringVector[i])) - if my_name != "DID": - py_str_body += my_name.encode('utf-8') + b'(' + my_value.encode('utf-8') + b')' + b' ' - - if my_name == "uid": - value_string = "u" + format(int(my_value) % 10000, '>4d') + " " - py_str_preamble += value_string.encode('utf-8') - - elif my_name == "pc": - value_string = "0x" + format(int(my_value, 16) & 0xffff, '>04x') + " " - py_str_preamble += value_string.encode('utf-8') - - elif my_name == "mnemonic": - value_string = format(my_value[0:7], '<7') + " " - py_str_preamble += value_string.encode('utf-8') - - - # Randomly remove from cache - if len(self.__cached_annotations) > 30000: - # TODO: consider whether this can be improved using a replacement policy - self.__cached_annotations.clear() - - # Update cache - decoded_str = bytes_re.sub(r'\1', (py_str_preamble + py_str_body).decode('utf-8')) # Replace b'xxx' with xxx - self.__cached_annotations[c_trans.transaction_ID] = decoded_str - return decoded_str - - def getPairID(self, uint32_t loc): - cdef c_TransactionInterval_uint64_const_t* c_trans - c_trans = self._getTransaction(loc) - if c_trans == NULL: - return None - return c_trans.pairId - - def getTransactionID(self, uint32_t loc): - cdef c_TransactionInterval_uint64_const_t* c_trans - c_trans = self._getTransaction(loc) - if c_trans == NULL: - return None - return c_trans.transaction_ID - - def getDisplayID(self, uint32_t loc): - cdef c_TransactionInterval_uint64_const_t* c_trans - c_trans = self._getTransaction(loc) - if c_trans == NULL: - return None - return c_trans.display_ID - - def getLocationMap(self): - cdef list results = []*self.__cur_content_len - cdef uint32_t i - cdef uint32_t c_trans_idx - for i in range(self.__cur_content_len): - c_trans_idx = self.__cur_content[i] - if c_trans_idx == NO_TRANSACTION: - results.append('no') - else: - results.append(self.__cur_transactions[c_trans_idx].transaction_ID) - return results - - ## Get the number of cached annotations - def getNumCachedAnnotations(self): - return len(self.__cached_annotations) - - def getWindowLeft(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getWindowStart() - - def getWindowRight(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getWindowEnd() - - def getFileStart(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getFileStart() - - ## Get the exclusive endpoint - def getFileEnd(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - return self.__window.getFileEnd() - - ## Get the inclusive end point. If 0, file has no data - def getFileInclusiveEnd(self): - if self.__window == NULL: - raise RuntimeError(self.OBJECT_DESTROYED_ERROR) - - cdef uint64_t end = self.__window.getFileEnd() - if end == 0: - return 0 - return end - 1 - - def isUpdateReady(self): - return self.__window.updateReady() - - def ackUpdate(self): - self.__window.ackUpdate() - - def enableUpdate(self): - self.__window.enableUpdate() - - def disableUpdate(self): - self.__window.disableUpdate() - - def forceUpdate(self): - self.__window.forceUpdate() - -cdef void c_callback(void* obj, - uint64_t tick, - const_interval_idx * content, - c_TransactionInterval_uint64_const_t* transactions, - uint32_t content_len): - cdef TransactionDatabase c_tdbi - c_tdbi = obj - c_tdbi.handleTickCallback(tick, content, transactions, content_len) diff --git a/helios/pipeViewer/scripts/alf_gen/ALFLayout.py b/helios/pipeViewer/scripts/alf_gen/ALFLayout.py deleted file mode 100644 index 7dc968c3ea..0000000000 --- a/helios/pipeViewer/scripts/alf_gen/ALFLayout.py +++ /dev/null @@ -1,665 +0,0 @@ -'''ALFLayout Generation Tool - - This python library is used to generate an ALF view used with Argos - pipeout viewer by using the locations file from a generated pipeout - of a simulator. - - To use Argos, you do not need to use this library to generate ALF - views; the Argos tool is its own editor. However, to generate - really complicated views, this library/utility comes in handy. - - To view the documentation on this library, run the following - command: - - pydoc ALFLayout - -An example usage can be found in the CoreExample in -map/sparta/example/CoreModel/gen_layouts.py - -''' -import sys, re -import pdb -import math - -def sort_list_canotical(input_list): - def atoi(text): - return int(text) if text.isdigit() else text - def natural_keys(text): - return [ atoi(c) for c in re.split('(\d+)',text) ] - input_list.sort(key=natural_keys) - -# Internal utility function -def generate_mini_caption_name(disp_name_map, location_rexp, matched_locs, mini_cnt): - caption_name = "unknown" - range_str = f'{len(matched_locs)-1}-{len(matched_locs) - mini_cnt}' - if len(disp_name_map) == 0: - caption_name = f"{loc.split('.')[-1]}[{range_str}]" - else: - _, n_rep = re.subn(location_rexp, disp_name_map[0], matched_locs[0]) - if n_rep == 0: - caption_name = f'{disp_name_map[0]}[{range_str}]' - else: - caption_name = disp_name_map[0].replace('\\1', f'{range_str}') - - return caption_name - -# Internal utility function -def generate_caption_name(disp_name_map, location_rexp, loc, loc_idx): - caption_name = "unknown" - - if len(disp_name_map) == 0: - caption_name = loc.split('.')[-1] - return caption_name - - if len(disp_name_map) == 1 or loc_idx >= len(disp_name_map): - caption_name, n_rep = re.subn(location_rexp, disp_name_map[0], loc) - if n_rep == 0: - print(f"WARNING! Could not replace {disp_name_map[0]} in string {loc}. Using {disp_name_map[0]} as is") - caption_name = loc.split('.')[-1] - else: - caption_name = re.sub(location_rexp, disp_name_map[loc_idx], loc) - - return caption_name - -# Internal utility class -class Content: - def __init__(self, content, color, dimensions, position, loc='', t_offset=None): - self._content = content - self._loc = loc - self._color = color - self._dimensions = dimensions - self._position = position - self._t_offset = t_offset - - def output_to_alf(self, alf_file, spacing=''): - print(f'''{spacing}- Content: {self._content} -{spacing} color: ({','.join([str(x) for x in self._color])}) -{spacing} dimensions: ({','.join([str(x) for x in self._dimensions])}) -{spacing} position: ({','.join([str(x) for x in self._position])})''', file=alf_file) - if len(self._loc) > 0: - print(f'{spacing} LocationString: {self._loc}', file=alf_file) - if self._t_offset is not None: - print(f'{spacing} t_offset: {self._t_offset}', file=alf_file) - - -class ALFLayout: - r'''ALFLayout class is the top-level class to create a new ALF file and - "draw" components to be shown on an Argos view. Use starts - with creating an ALFLayout object and populating it with - components (usually aligned with a location.dat file from a - pipeout). - - Example usage: - - my_layout = ALFLayout(start_time = -10, # Start back in time 10 cycles - num_cycles = num_cycles, - clock_scale = 500, - location_file = 'my_pipeout_location.dat', - alf_file = 'my_layout.alf') - - sl_grp = layout.createScheduleLineGroup(default_color=[192,192,192], - include_detail_column = True, - margins=ALFLayout.Margin(top = 2, left = 10)) - - # We have these locations in the locations.dat file: - # top.cpu.alu0.pipe0 - # top.cpu.alu0.pipe1 - # top.cpu.alu0.pipe2 - # top.cpu.alu1.pipe0 - # top.cpu.alu1.pipe1 - # top.cpu.alu1.pipe2 - sl_grp.addScheduleLine('.*alu([0-1]+).*', [r'ALU \1 Stage RR', - r'ALU \1 Stage EX', - r'ALU \1 Stage WB'], space=True) - - # Add a graphic that shows the cycles - sl_grp.addCycleLegend(ALFLayout.CycleLegend(location = 'cycle', interval = 5)) - ''' - - # Open and parse the location file grabbing the location strings. - def __parse_location_file(self, location_file : str): - try: - loc_file = open(location_file, 'r') - except: - print("ERROR: Issues with location file: ", location_file) - exit(1) - - # Skip the first line - loc_file.readline() - - self._loc_strs = [] - for line in loc_file: - (_, loc_str, _) = line.split(',') - # Weed out the named variables with [] -- they are dups - if '[' not in loc_str: - self._loc_strs.append(loc_str) - # Remove dups - self._loc_strs = list(set(self._loc_strs)) - - class Spacing: - def __init__(self, - height : int, - height_spacing : int, - melem_height : int, - melem_spacing : int, - caption_width : int, - onechar_width : int = 9): - self._height = height - self._height_spacing = height_spacing - self._melem_height = int(melem_height) - self._melem_spacing = int(melem_spacing) - self._caption_width = caption_width - self._onechar_width = onechar_width - - @property - def height_spacing(self): - return self._height_spacing - - @property - def melem_height(self): - return self._melem_height - - @property - def height(self): - return self._height - - @property - def caption_width(self): - return self._caption_width - - @property - def onechar_width(self): - return self._onechar_width - - class Margin: - '''Margin top and left starting point for a ScheduleLineGroup or ColumnView''' - def __init__(self, - top = 2, - left = 10): - self._top = top - self._left = left - - @property - def left(self): - return self._left - - @property - def top(self): - return self._top - - class CycleLegend: - def __init__(self, - location, - interval = 5): - self._location = location - self._interval = interval - - @property - def location(self): - return self._location - - @property - def interval(self): - return self._interval - - class Caption(Content): - def __init__(self, name, color, dimensions, position): - Content.__init__(self, - content='caption', - color=color, - dimensions=dimensions, - position=position) - self._name = name - - def output_to_alf(self, alf_file): - Content.output_to_alf(self, alf_file) - print(f' caption: {self._name}', file=alf_file) - - class Cycle(Content): - def __init__(self, loc, color, dimensions, position, t_offset=0): - Content.__init__(self, - content='cycle', - color=color, - dimensions=dimensions, - position=position, - loc=loc, t_offset=t_offset) - - def output_to_alf(self, alf_file): - Content.output_to_alf(self, alf_file) - - ################################################################################ - # Schedule line - class ScheduleLineGroup: - r'''A ScheduleLineGroup allows a user to add ScheduleLine components - that represent a location in a pipeout. Intended usage: - - sl_group = my_layout.createScheduleLineGroup(default_color=[192,192,192], - include_detail_column = True, - margins=ALFLayout.Margin(top = 2, left = 10)) - sl_group.addScheduleLine('.*alu([0-9]+).*', [r'alu[\1]']) - ''' - class ScheduleLine(Content): - def __init__(self, - loc_str : str, - content : str = 'auto_color_annotation', - color : list = [192,192,192], - dimen : list = [0,0], - pos : list = [0,0], - t_offset: int = 0): - Content.__init__(self, - content=content, - color=color, - dimensions=dimen, - position=pos, - loc=loc_str, - t_offset=t_offset) - - def output_to_alf(self, alf_file): - Content.output_to_alf(self, alf_file, spacing=' ') - print(f''' type: schedule_line''', file=alf_file) - - def __init__(self, - clock_scale : int, - num_cycles : int, - locations : list, - def_color : list, - spacing, - margins, - include_detail_column : bool, - time_offset : int, - content_width : int): - self._num_cycles = num_cycles - self._locations = locations - self._def_color = def_color - self._spacing = spacing - self._margins = margins - self._pos = [self._margins.left + self._spacing.caption_width - 1, - self._margins.top] - self._pixel_offset = -(time_offset) * (self._spacing.onechar_width - 1) - self._time_scale = clock_scale/(self._spacing.onechar_width - 1) - self._schedule_lines = [] - self._detailed_schedule_lines = [] - self._t_offset = time_offset - self._dimen = [self._spacing.onechar_width*self._num_cycles, 0] - self._captions = [] - self._horz_lines = [] - self._line_pos = self._pos.copy() - self._caption_pos = [self._margins.left, self._pos[1]] - self._detail_column = include_detail_column - self._content_width = content_width - - def addScheduleLine(self, - location_name : str, - disp_name_map : list, - **kwargs): - '''Add schedule lines to the layout (within a group). The location - name is allowed to hit multiple locations. If so, a line - will be added for each location found. - - Parameters: - - location_name - Regexp of the location to add - disp_name_map - List/map of names to location - - Other keywords: - color - List of RGB: [0,0,0] - reverse - Reverse the locations found (default is N -> 0) - nomunge - The value found the location raw -- do not colorize - space - Add a dark line after all the locations have been added - - ''' - location_rexp = re.compile(location_name) - matched_locs = list(filter(location_rexp.findall, self._locations)) - - if len(matched_locs) == 0: - print("ERROR: Could not find locations matching pattern: ", location_name) - exit(1) - sort_list_canotical(matched_locs) - - mini_perc = 0.0 - reg_perc = 100.0 - if 'mini_split' in kwargs: - mini_perc, reg_perc = kwargs['mini_split'] - - mini_cnt = round(len(matched_locs) * mini_perc/100) - reg_cnt = round(len(matched_locs) * reg_perc/100) - - assert (mini_cnt + reg_cnt) == len(matched_locs) - - color = self._def_color - if 'color' in kwargs: - color = kwargs['color'] - assert isinstance(color, list), "Color must be a list of 3 colors" - - reverse = True - if 'reverse' in kwargs: - reverse = kwargs['reverse'] - - content_width = self._content_width - - # Create the mini layout - if mini_cnt != 0: - melement_height_total = 0 - mini_anno = matched_locs[reg_cnt:] - if reverse: - mini_anno.reverse() - for loc in mini_anno: - self._schedule_lines.append( - self.ScheduleLine(loc, - 'auto_color_anno_notext', - color, - [self._dimen[0], self._spacing.melem_height], - self._line_pos.copy(), - self._t_offset)) - if self._detail_column: - # Add the mini layout to the right - self._detailed_schedule_lines.append (Content(content='auto_color_anno_notext', - loc=loc, color=color, - dimensions=[content_width, - self._spacing.melem_height], - position = [ - self._line_pos[0] + - self._dimen[0] + - self._spacing.caption_width + 20, - self._line_pos[1]], - t_offset = 0)) - - melement_height_total += self._spacing.melem_height - self._line_pos[1] += self._spacing.melem_height - - caption_name = generate_mini_caption_name(disp_name_map, - location_rexp, - matched_locs, - mini_cnt) - - self._captions.append(ALFLayout.Caption(caption_name, color, - [self._spacing.caption_width, - self._spacing.height_spacing], - self._caption_pos.copy())) - - if self._detail_column: - self._captions.append(ALFLayout.Caption(caption_name, color, - [self._spacing.caption_width, - self._spacing.height_spacing], - position=[ - self._line_pos[0] + - self._dimen[0] + 20, self._caption_pos[1]+1])) - - self._caption_pos[1] += int(round(melement_height_total)) - - content = "auto_color_annotation" - if 'nomunge' in kwargs: - content = "auto_color_anno_nomunge" - - loc_idx = 0 - # Create the normal sized items - anno = matched_locs[0:reg_cnt] - if reverse: - anno.reverse() - for loc in anno: - caption_name = generate_caption_name(disp_name_map, - location_rexp, - loc, - loc_idx) - loc_idx += 1 - - self._schedule_lines.append( - self.ScheduleLine(loc, - content, - color, - [self._dimen[0], self._spacing.height], - self._line_pos.copy(), - self._t_offset)) - - self._captions.append(ALFLayout.Caption(caption_name, - color, - [self._spacing.caption_width, - self._spacing.height], - self._caption_pos.copy())) - - if self._detail_column: - self._detailed_schedule_lines.append (Content(content=content, - loc=loc, - color=color, - dimensions=[content_width, - self._spacing.height], - position = [ - self._line_pos[0] + - self._dimen[0] + - self._spacing.caption_width + 20, - self._line_pos[1]], - t_offset = 0)) - self._captions.append(ALFLayout.Caption(caption_name, color, - [self._spacing.caption_width, - self._spacing.height_spacing], - position=[ - self._line_pos[0] + - self._dimen[0] + 20, self._caption_pos[1]+1])) - - self._caption_pos[1] += self._spacing.height - self._line_pos[1] += self._spacing.height - - # Create a horizontal separator - if 'space' in kwargs and kwargs['space']: - self._horz_lines.append(ALFLayout.Caption("", - [0,0,0], # black - [self._dimen[0]+self._spacing.caption_width, 1], - self._caption_pos.copy())) - # Increment the schedule y dimension - self._dimen[1] += self._spacing.height_spacing * len(matched_locs) - - def addCycleLegend(self, cycle_legend): - cycle_line_pos = self._caption_pos.copy() - # Add the cycle keyword - self._captions.append(ALFLayout.Caption('C=1 Cycle', self._def_color, - [self._spacing.caption_width, - self._spacing.height], - cycle_line_pos.copy())) - cycle_word_size = 50 - - # Move X - cycle_line_pos[0] += cycle_word_size - - # Add the current cycle count - self._captions.append(ALFLayout.Cycle(loc=cycle_legend.location, - color=self._def_color, - dimensions=[self._spacing.caption_width - cycle_word_size, - self._spacing.height], - position=cycle_line_pos.copy())) - - # Add cycle markers - cycle_line_pos[0] += self._spacing.caption_width - cycle_word_size - - time_marker_width = 8 * cycle_legend.interval - - for time_marker in range(self._t_offset, self._num_cycles, 5): - self._captions.append(ALFLayout.Caption(f'C=1 {time_marker}', - self._def_color, - [time_marker_width, - self._spacing.height], - cycle_line_pos.copy())) - cycle_line_pos[0] += time_marker_width - - - def output_to_alf(self, alf_file): - assert self._dimen[1] != 0, "Looks like a ScheduleLine was created, but nothing added" - print(f'''- type: schedule - color: ({','.join([str(x) for x in self._def_color])}) - dimensions: ({','.join([str(x) for x in self._dimen])}) - position: ({','.join([str(x) for x in self._pos])}) - pixel_offset: {self._pixel_offset} - time_scale: {self._time_scale} - children: ''', file=alf_file) - for line in self._schedule_lines: - line.output_to_alf(alf_file) - for line in self._detailed_schedule_lines: - line.output_to_alf(alf_file) - for cap in self._captions: - cap.output_to_alf(alf_file) - for horz in self._horz_lines: - horz.output_to_alf(alf_file) - - class ColumnView: - '''A ColumnView has captions to the left and single-cycle view to the right - ''' - def __init__(self, - locations: list, - spacing, - margins, - content_width, - def_color, - time_offset : int): - - self._locations = locations - self._spacing = spacing - self._margins = margins - self._content_width = content_width - self._def_color = def_color - self._t_offset = time_offset - self._pos = [self._margins.left, self._margins.top] - self._captions = [] - self._content = [] - - def addColumn(self, - location_name : str, - disp_name_map : list, - ** kwargs): - location_rexp = re.compile(location_name) - matched_locs = list(filter(location_rexp.findall, self._locations)) - if len(matched_locs) == 0: - print("ERROR: Could not find locations matching pattern: ", location_name) - exit(1) - sort_list_canotical(matched_locs) - matched_locs.reverse() - content = "auto_color_annotation" - if 'nomunge' in kwargs: - content = "auto_color_anno_nomunge" - loc_idx = 0 - for loc in matched_locs: - caption_name = generate_caption_name(disp_name_map, location_rexp, loc, loc_idx) - loc_idx += 1 - - self._captions.append(ALFLayout.Caption(caption_name, - color=self._def_color, - dimensions=[self._spacing.caption_width, - self._spacing.height], - position=self._pos.copy())) - self._content.append(Content(content=content, - loc=loc, - color=self._def_color, - dimensions=[self._content_width, - self._spacing.height], - position = [ - self._pos[0] + - self._spacing.caption_width, - self._pos[1]], - t_offset = 0)) - - self._pos[1] += self._spacing.height - - def output_to_alf(self, alf_file): - for content in self._content: - content.output_to_alf(alf_file) - for caption in self._captions: - caption.output_to_alf(alf_file) - - ################################################################################ - # ALFLayout construction - def __init__(self, - start_time : int, - num_cycles : int, - clock_scale : int, - location_file : str, - alf_file : str): - ''' - Create an ALF Layout file (.alf) used by argos (-l option). - - There are two parts to this generator: - 1. Creating schedule lines (multiple cycle views) - 2. Creating a column view (single cycle view in a column format) - - Construction arguments: - start_time - When to start showing time (can be negative) - num_cycles - Number of cycles to view when adding schedule lines - clock_scale - Used to setup the Argos timescale - location_file - The location file generated by a simulator when using the -z option - alf_file - The final ALF file - ''' - self._all_good = False - self.__parse_location_file(location_file) - self._all_good = True - self._start_time = start_time - self._num_cycles = num_cycles - self._clock_scale = clock_scale - self._spacing = None - self._schedule_line = None - self._column_view = None - - # Variable that contains the state of object being placed in - # the layout (to prevent overlaps) - self._positional_state = [0, 0] - - # Print the first line expected in all ALFs - self._alf_file = open(alf_file, 'w') - print('---', file=self._alf_file) - - def __del__(self): - '''Destroy the Layout, writing all results to the given ALF file - ''' - if self._all_good: - if self._schedule_line: - self._schedule_line.output_to_alf(self._alf_file) - if self._column_view: - self._column_view.output_to_alf(self._alf_file) - print('...', file=self._alf_file) - - ################################################################################ - # ALFLayout API - def setSpacing(self, spacing): - '''For this layout, set the basic spacing that should be used. See - Spacing class for more documentation. - ''' - self._spacing = spacing - - def count(self, location): - '''Count the number of matches for the given location (regexpression). - This is handy when unsure how many units are in the given - location and the need to iterate over them is required. A - capture group is required to make the query very unique. - - num_alu = my_layout.count(r'.*alu([0-9]).*`) - - ''' - regexp = re.compile(location) - loc_matches = list(filter(regexp.findall, self._loc_strs)) - uniq_names = set() - for match in loc_matches: - uniq_names.add(re.sub(regexp, r'\1', match)) - return len(uniq_names) - - def createScheduleLineGroup(self, default_color, include_detail_column, content_width, margins): - '''Create a ScheduleLineGroup that ScheduleLines can be added. See - ScheduleLineGroup for more documentation. - ''' - self._schedule_line = self.ScheduleLineGroup(clock_scale = self._clock_scale, - num_cycles = self._num_cycles, - locations = self._loc_strs, - def_color = default_color, - spacing = self._spacing, - margins = margins, - include_detail_column = include_detail_column, - time_offset = self._start_time, - content_width = content_width) - return self._schedule_line - - def createColumnView(self, margins, content_width, default_color=[192,192,192]): - '''Create a ColumnView -- a single cycle view with added components. - See ColumnView for more documentation. - ''' - self._column_view = self.ColumnView(locations = self._loc_strs, - spacing = self._spacing, - margins = margins, - def_color = default_color, - time_offset=0, - content_width = content_width) - return self._column_view diff --git a/helios/pipeViewer/scripts/color_nodes_by_access.py b/helios/pipeViewer/scripts/color_nodes_by_access.py deleted file mode 100644 index 81fb235080..0000000000 --- a/helios/pipeViewer/scripts/color_nodes_by_access.py +++ /dev/null @@ -1,20 +0,0 @@ -#sample script to be executed by pipeViewer Console -from model.node_element import NodeElement -highest_access_count = 0 -for element in context.GetElements(): - if isinstance(element, NodeElement): - split = element.GetProperty('data').split(':') - if len(split) < 2: - continue # either Start or End - accesses = int(split[1]) - if accesses > highest_access_count: - highest_access_count = accesses - -for element in context.GetElements(): - if isinstance(element, NodeElement): - split = element.GetProperty('data').split(':') - if len(split) < 2: - continue # either Start or End - accesses = int(split[1]) - color_val = int((1 - accesses/(highest_access_count*1.0))*255) - context.dbhandle.database.AddMetadata(element.GetProperty('name'), {'color':(color_val, 255, 255)}) diff --git a/helios/pipeViewer/setup.py b/helios/pipeViewer/setup.py deleted file mode 100644 index cd01234327..0000000000 --- a/helios/pipeViewer/setup.py +++ /dev/null @@ -1,68 +0,0 @@ -import setuptools -from setuptools import find_packages -from Cython.Build import cythonize -from Cython.Distutils import build_ext - -compile_args = ['--std=c++17'] # Required for SPARTA -# override strict flags -compile_args += [ - '-Wno-cast-qual', - '-Wno-deprecated-declarations', - '-Wno-strict-aliasing', - '-Wall', - '-Wpedantic' -] - -# Wrappers for the parts written in Cython -transaction_db = setuptools.Extension( - 'pipe_view.transactiondb', - language='c++', - sources=['pipe_view/transactiondb/src/transactiondb.pyx'], - libraries=["sparta", "simdb", "hdf5", "sqlite3"], - pyrex_gdb = True, - extra_compile_args = compile_args, -) -core = setuptools.Extension( - 'pipe_view.core', - language='c++', - sources=['pipe_view/core/src/core.pyx'], - libraries=["sparta", "simdb", "hdf5", "sqlite3"], - pyrex_gdb = True, - extra_compile_args = compile_args, -) -logsearch = setuptools.Extension( - 'pipe_view.logsearch', - language='c++', - sources=['pipe_view/logsearch/src/logsearch.pyx', 'pipe_view/logsearch/src/log_search.cpp'], - libraries=["sparta", "simdb", "hdf5", "sqlite3"], - pyrex_gdb = True, - extra_compile_args = compile_args, -) -ext_modules = [transaction_db, core, logsearch] - - -py_packages = ["pipe_view", "pipe_view.misc", "pipe_view.gui", - "pipe_view.gui.dialogs", "pipe_view.gui.widgets", - "pipe_view.model", ] - -setuptools.setup( - name='pipe_view', - packages=py_packages, - ext_modules = cythonize(ext_modules, language_level=3, - include_path=['pipe_view/core/src'], # to find common.pxd - ), - entry_points = { - 'gui_scripts': ['argos=pipe_view.argos:main'], - }, - package_data={'pipe_view': ['core/src/common.pxd', 'transactiondb/src/common.pxd', 'resources/*.png', 'stubs/*.pyi']}, - include_package_data=True, - setup_requires=[ - 'cython', - 'wxPython', - ], - install_requires=[ - 'wxPython', - ], - tests_require=['pytest'], - zip_safe=False, - ) diff --git a/helios/pipeViewer/stubs/wx/__init__.pyi b/helios/pipeViewer/stubs/wx/__init__.pyi deleted file mode 100644 index 6cc60927b2..0000000000 --- a/helios/pipeViewer/stubs/wx/__init__.pyi +++ /dev/null @@ -1,1589 +0,0 @@ -# wxPython doesn't (as of Dec. 2022) provide type hints in a format that mypy can use. -# However, this is the one library that we need to type-check the most - passing the wrong -# types (for example, passing floats when ints are expected) can cause all kinds of hard to -# debug rendering bugs. So, this file provides stub definitions for all of the wxPython -# classes, functions, and values that we use in Argos. - -from __future__ import annotations -from types import TracebackType -from typing import Any, Callable, Iterator, List, Optional, Sequence, Tuple, Type, Union, overload - -class Object: ... - -class Trackable: ... - -class EvtHandler(Object, Trackable): - def Bind(self, - event: PyEventBinder, - handler: Callable, - source: Optional[Object] = None, - id: int = ID_ANY, - id2: int = ID_ANY) -> None: ... - -class EventFilter: ... -class AppConsole(EvtHandler, EventFilter): ... - -class App(AppConsole): - def __init__(self, - redirect: bool = False, - filename: Optional[str] = None, - useBestVisual: bool = False, - clearSigInt: bool = True) -> None: ... - def MainLoop(self) -> None: ... - -def NewId() -> int: ... - -BOTH: int -HORIZONTAL: int -VERTICAL: int -CENTRE: int -ALIGN_CENTER: int -ALIGN_CENTER_HORIZONTAL: int -ALIGN_CENTER_VERTICAL: int -ALIGN_BOTTOM: int -ALIGN_LEFT: int -ALIGN_RIGHT: int -ALIGN_TOP: int -BOTTOM: int -LEFT: int -RIGHT: int -TOP: int -ALL: int -EXPAND: int -ID_ANY: int -ID_NONE: int -ID_CANCEL: int -ID_OK: int -ID_FIND: int -ID_EXIT: int -ID_DELETE: int -ID_SAVE: int -ID_BACKWARD: int -ID_NO: int -DEFAULT_DIALOG_STYLE: int -MAXIMIZE_BOX: int -BORDER_NONE: int -BORDER_SIMPLE: int -BORDER_SUNKEN: int -RESIZE_BORDER: int -SUNKEN_BORDER: int -CAPTION: int -CLOSE_BOX: int -CLIP_CHILDREN: int -STAY_ON_TOP: int -SYSTEM_MENU: int -TAB_TRAVERSAL: int -NO_FULL_REPAINT_ON_RESIZE: int -SHAPED: int - -OK: int -CANCEL: int -YES_NO: int -YES_DEFAULT: int -ICON_QUESTION: int - -wxEVT_NULL: int - -class PyEventBinder: - @property - def typeId(self) -> int: ... - -EVT_BUTTON: PyEventBinder -EVT_SPINCTRL: PyEventBinder -EVT_CLOSE: PyEventBinder -EVT_MENU: PyEventBinder -EVT_TEXT: PyEventBinder -EVT_TEXT_ENTER: PyEventBinder -EVT_LIST_ITEM_ACTIVATED: PyEventBinder -EVT_LIST_ITEM_RIGHT_CLICK: PyEventBinder -EVT_KEY_UP: PyEventBinder -EVT_KEY_DOWN: PyEventBinder -EVT_LEFT_DOWN: PyEventBinder -EVT_CHECKBOX: PyEventBinder -EVT_ENTER_WINDOW: PyEventBinder -EVT_SET_FOCUS: PyEventBinder -EVT_SIZE: PyEventBinder -EVT_TIMER: PyEventBinder -EVT_MOTION: PyEventBinder -EVT_CHOICE: PyEventBinder -EVT_KILL_FOCUS: PyEventBinder -EVT_TREE_ITEM_EXPANDING: PyEventBinder -EVT_TREE_ITEM_EXPANDED: PyEventBinder -EVT_TREE_SEL_CHANGED: PyEventBinder -EVT_TREE_KEY_DOWN: PyEventBinder -EVT_ERASE_BACKGROUND: PyEventBinder -EVT_PAINT: PyEventBinder -EVT_LEFT_DCLICK: PyEventBinder -EVT_LEFT_UP: PyEventBinder -EVT_RIGHT_DOWN: PyEventBinder -EVT_RIGHT_UP: PyEventBinder -EVT_MOUSEWHEEL: PyEventBinder -EVT_SCROLLWIN: PyEventBinder -EVT_RADIOBUTTON: PyEventBinder -EVT_ACTIVATE: PyEventBinder -EVT_CHILD_FOCUS: PyEventBinder -EVT_SLIDER: PyEventBinder -EVT_CHAR: PyEventBinder -EVT_COMBOBOX: PyEventBinder -EVT_TOOL: PyEventBinder - -KeyCode = int -WXK_F5: KeyCode -WXK_UP: KeyCode -WXK_DOWN: KeyCode -WXK_RIGHT: KeyCode -WXK_LEFT: KeyCode -WXK_PAGEUP: KeyCode -WXK_PAGEDOWN: KeyCode -WXK_HOME: KeyCode -WXK_END: KeyCode -WXK_SHIFT: KeyCode -WXK_CONTROL: KeyCode -WXK_ESCAPE: KeyCode -WXK_RETURN: KeyCode -WXK_ENTER: KeyCode -WXK_DELETE: KeyCode - -MOD_CONTROL: int - -TIMER_CONTINUOUS: int - -class Timer(EvtHandler): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, owner: EvtHandler, id: int = -1) -> None: ... - def Start(self, milliseconds: int = -1, oneShot: int = TIMER_CONTINUOUS) -> bool: ... - def Stop(self) -> None: ... - def IsRunning(self) -> bool: ... - -class Size: - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, width: int, height: int) -> None: ... - @overload - def __getitem__(self, idx: int) -> int: ... - @overload - def __getitem__(self, idx: slice) -> Sequence[int]: ... - def __iter__(self) -> Iterator[int]: ... - def GetWidth(self) -> int: ... - def GetHeight(self) -> int: ... - def Get(self) -> Tuple[int, int]: ... - -SizeUnion = Union[Tuple[int, int], Size] - -class Event: - def GetId(self) -> int: ... - def Skip(self, skip: bool = True) -> None: ... - def GetEventObject(self) -> Object: ... - -class PaintEvent(Event): ... - -class CommandEvent(Event): - def GetString(self) -> str: ... - -class PyCommandEvent: - def __init__(self, eventType: int = wxEVT_NULL, id: int = 0) -> None: ... - -class NotifyEvent(CommandEvent): ... - -class MenuEvent(Event): ... - -class FocusEvent(Event): - def GetWindow(self) -> Window: ... - -class TimerEvent(Event): ... - -class SizeEvent(Event): ... - -class ActivateEvent(Event): ... - -class ScrollEvent(CommandEvent): ... - -class KeyboardState: - def AltDown(self) -> bool: ... - def ControlDown(self) -> bool: ... - def ShiftDown(self) -> bool: ... - def GetModifiers(self) -> int: ... - -class KeyEvent(Event, KeyboardState): - def GetKeyCode(self) -> int: ... - -class MouseState(KeyboardState): - def GetPosition(self) -> Point: ... - def LeftIsDown(self) -> bool: ... - -class MouseEvent(Event, MouseState): - def GetWheelDelta(self) -> int: ... - def GetWheelRotation(self) -> int: ... - -ALPHA_OPAQUE: int - -class Colour: - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, red: int, green: int, blue: int, alpha: int = ALPHA_OPAQUE) -> None: ... - @overload - def __init__(self, colRGB: int) -> None: ... - @overload - def __init__(self, colour: Colour) -> None: ... - def Get(self, includeAlpha: bool = True) -> Tuple[int, ...]: ... - @overload - def Set(self, red: int, green: int, blue: int, alpha: int = ALPHA_OPAQUE) -> None: ... - @overload - def Set(self, rgb: int) -> None: ... - @overload - def Set(self, str: str) -> bool: ... - @overload - def ChangeLightness(self, ialpha: int) -> Colour: ... - @overload - def ChangeLightness(self, r: int, g: int, b: int, ialpha: int) -> Tuple[int, int, int]: ... - -BLACK: Colour -WHITE: Colour -RED: Colour - -ColourUnion = Union[Tuple[int, int, int], List[int], str, Colour] - -ImageResizeQuality = int -IMAGE_QUALITY_NORMAL: ImageResizeQuality -IMAGE_QUALITY_HIGH: ImageResizeQuality - -class StreamBase: ... -class InputStream(StreamBase): ... - -IMAGE_OPTION_CUR_HOTSPOT_X: str -IMAGE_OPTION_CUR_HOTSPOT_Y: str - -class Image(Object): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, width: int, height: int, clear: bool = True) -> None: ... - @overload - def __init__(self, sz: SizeUnion, clear: bool = True) -> None: ... - @overload - def __init__(self, name: str, type: BitmapType = BITMAP_TYPE_ANY, index: int = -1) -> None: ... - @overload - def __init__(self, name: str, mimetype: str, index: int = -1) -> None: ... - @overload - def __init__(self, stream: InputStream, type: BitmapType = BITMAP_TYPE_ANY, index: int = -1) -> None: ... - @overload - def __init__(self, width: int, height: int, data: bytes) -> None: ... - @overload - def __init__(self, width: int, height: int, data: bytes, alpha: bytes) -> None: ... - @overload - def __init__(self, sz: SizeUnion, data: bytes) -> None: ... - @overload - def __init__(self, sz: SizeUnion, data: bytes, alpha: bytes) -> None: ... - def Scale(self, width: int, height: int, quality: ImageResizeQuality = IMAGE_QUALITY_NORMAL) -> Image: ... - def SetOption(self, name: str, value: Union[int, str]) -> None: ... - -StockCursor = int -CURSOR_WAIT: StockCursor -CURSOR_SIZING: StockCursor -CURSOR_SIZENS: StockCursor -CURSOR_SIZEWE: StockCursor -CURSOR_SIZENESW: StockCursor -CURSOR_SIZENWSE: StockCursor -HOURGLASS_CURSOR: StockCursor - - -class Cursor(GDIObject): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - cursorName: str, - type: BitmapType = BITMAP_TYPE_ANY, - hotSpotX: int = 0, - hotSpotY: int = 0) -> None: ... - @overload - def __init__(self, cursorId: StockCursor) -> None: ... - @overload - def __init__(self, image: Image) -> None: ... - @overload - def __init__(self, cursor: Cursor) -> None: ... - -class BusyCursor: - def __init__(self, cursor: Union[Cursor, StockCursor] = HOURGLASS_CURSOR) -> None: ... - def __enter__(self) -> Cursor: ... - def __exit__(self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType]) -> None: ... - -class RealPoint: ... - -class Point: - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, x: int, y: int) -> None: ... - @overload - def __init__(self, pt: RealPoint) -> None: ... - @overload - def __getitem__(self, idx: int) -> int: ... - @overload - def __getitem__(self, idx: slice) -> Sequence[int]: ... - def __iter__(self) -> Iterator[int]: ... - def Get(self) -> Tuple[int, int]: ... - -PointUnion = Union[Tuple[int, int], Point] - -class ToolTip(Object): ... - -SIZE_AUTO: int -SIZE_USE_EXISTING: int -SIZE_ALLOW_MINUS_ONE: int - -BackgroundStyle = int -BG_STYLE_PAINT: BackgroundStyle - -class Window(EvtHandler): - def AcceptsFocus(self) -> bool: ... - def CaptureMouse(self) -> None: ... - def Center(self, dir: int = BOTH) -> None: ... - @overload - def ClientToScreen(self, x: int, y: int) -> Tuple[int, int]: ... - @overload - def ClientToScreen(self, pt: PointUnion) -> Point: ... - def Close(self, force: bool = False) -> bool: ... - def Destroy(self) -> None: ... - def DestroyLater(self) -> None: ... - def Disable(self) -> bool: ... - def Enable(self, enable: bool = True) -> None: ... - def Fit(self) -> None: ... - def GetBackgroundColour(self) -> Colour: ... - def GetBestSize(self) -> Size: ... - def GetChildren(self) -> List[Window]: ... - def GetClientSize(self) -> Size: ... - def GetCursor(self) -> Cursor: ... - def GetEventHandler(self) -> EvtHandler: ... - def GetFont(self) -> Font: ... - def GetId(self) -> int: ... - def GetMinSize(self) -> Size: ... - def GetParent(self) -> Window: ... - def GetPosition(self) -> Point: ... - def GetRect(self) -> Rect: ... - def GetScreenPosition(self) -> Point: ... - def GetScrollPos(self, orientation: int) -> int: ... - def GetSize(self) -> Size: ... - def GetTextExtent(self, string: str) -> Size: ... - def GetUpdateRegion(self) -> Region: ... - def HasCapture(self) -> bool: ... - def Hide(self) -> bool: ... - def IsShown(self) -> bool: ... - def Layout(self) -> bool: ... - @overload - def Move(self, x: int, y: int, flags: int = SIZE_USE_EXISTING) -> None: ... - @overload - def Move(self, pt: PointUnion, flags: int = SIZE_USE_EXISTING) -> None: ... - def PopEventHandler(self, deleteHandler: bool = False) -> EvtHandler: ... - @overload - def PopupMenu(self, menu: Menu, pos: Point = DefaultPosition) -> bool: ... - @overload - def PopupMenu(self, menu: Menu, x: int, y: int) -> bool: ... - def PushEventHandler(self, handler: EvtHandler) -> None: ... - def Raise(elf) -> None: ... - def Refresh(self, eraseBackground: bool = True, rect: Optional[RectUnion] = None) -> None: ... - def RefreshRect(self, rect: RectUnion, eraseBackground: bool = True) -> None: ... - def ReleaseMouse(self) -> None: ... - @overload - def ScreenToClient(self, x: int, y: int) -> Tuple[int, int]: ... - @overload - def ScreenToClient(self, pt: Point) -> Point: ... - def SendSizeEvent(self, flags: int = 0) -> None: ... - def SetAcceleratorTable(self, accel: AcceleratorTable) -> None: ... - def SetAutoLayout(self, autoLayout: bool) -> None: ... - def SetBackgroundColour(self, colour: ColourUnion) -> bool: ... - def SetBackgroundStyle(self, style: BackgroundStyle) -> bool: ... - @overload - def SetClientSize(self, width: int, height: int) -> None: ... - @overload - def SetClientSize(self, size: SizeUnion) -> None: ... - @overload - def SetClientSize(self, rect: RectUnion) -> None: ... - def SetCursor(self, cursor: Cursor) -> bool: ... - def SetFocus(self) -> None: ... - def SetFont(self, font: Font) -> None: ... - def SetForegroundColour(self, colour: ColourUnion) -> bool: ... - def SetMinSize(self, size: SizeUnion) -> None: ... - def SetPosition(self, pt: PointUnion) -> None: ... - def SetRect(self, rect: RectUnion) -> None: ... - @overload - def SetSize(self, x: int, y: int, width: int, height: int, sizeFlags: int = SIZE_AUTO) -> None: ... - @overload - def SetSize(self, size: SizeUnion) -> None: ... - @overload - def SetSize(self, rect: RectUnion) -> None: ... - @overload - def SetSize(self, width: int, height: int) -> None: ... - def SetSizer(self, sizer: Sizer, deleteOld: bool = True) -> None: ... - def SetSizerAndFit(self, sizer: Sizer, deleteOld: bool = True) -> None: ... - @overload - def SetToolTip(self, tipString: str) -> None: ... - @overload - def SetToolTip(self, tip: ToolTip) -> None: ... - def Show(self, show: bool = True) -> bool: ... - def Update(self) -> None: ... - -class ScrollWinEvent(Event): - def GetOrientation(self) -> int: ... - -class Scrolled: - @overload - def CalcScrolledPosition(self, x: int, y: int) -> Point: ... - @overload - def CalcScrolledPosition(self, pt: PointUnion) -> Point: ... - def GetViewStart(self) -> Tuple[int, int]: ... - def CalcScrollInc(self, evt: ScrollWinEvent) -> int: ... - def GetScrollPixelsPerUnit(self) -> Tuple[int, int]: ... - def SetScrollbars(self, - pixelsPerUnitX: int, - pixelsPerUnitY: int, - noUnitsX: int, - noUnitsY: int, - xPos: int = 0, - yPos: int = 0, - noRefresh: bool = False) -> None: ... - -ScrolledWindowStyle: int - -class ScrolledWindow(Scrolled, Window): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = ScrolledWindowStyle, - name: str = PanelNameStr) -> None: ... - -class SizerFlags: ... - -class PyUserData: ... - -class SizerItem(Object): ... - -class Rect: - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, x: int, y: int, width: int, height: int) -> None: ... - @overload - def __init__(self, pos: PointUnion, size: SizeUnion) -> None: ... - @overload - def __init__(self, size: SizeUnion) -> None: ... - @overload - def __init__(self, topLeft: PointUnion, bottomRight: PointUnion) -> None: ... - def __getitem__(self, idx: int) -> int: ... - def GetTopLeft(self) -> Point: ... - def GetBottomRight(self) -> Point: ... - def SetPosition(self, pos: PointUnion) -> None: ... - def GetHeight(self) -> int: ... - def GetWidth(self) -> int: ... - def SetHeight(self, height: int) -> None: ... - def SetWidth(self, width: int) -> None: ... - @overload - def Inflate(self, dx: int, dy: int) -> Rect: ... - @overload - def Inflate(self, diff: Size) -> Rect: ... - @overload - def Inflate(self, diff: int) -> Rect: ... - @property - def x(self) -> int: ... - @property - def y(self) -> int: ... - @property - def width(self) -> int: ... - @property - def height(self) -> int: ... - @property - def Size(self) -> Size: ... - @property - def Position(self) -> Point: ... - -RectUnion = Union[Rect, Tuple[int, int, int, int]] - -class Region(GDIObject): - def GetBox(self) -> Rect: ... - -BitmapType = int -BITMAP_TYPE_ANY: BitmapType -BITMAP_TYPE_PNG: BitmapType -BITMAP_SCREEN_DEPTH: int - -class GDIObject(Object): ... - -class Bitmap(GDIObject): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, bitmap: Bitmap) -> None: ... - @overload - def __init__(self, bits: bytes, width: int, height: int, depth: int) -> None: ... - @overload - def __init__(self, width: int, height: int, depth: int = BITMAP_SCREEN_DEPTH) -> None: ... - @overload - def __init__(self, sz: SizeUnion, depth: int = BITMAP_SCREEN_DEPTH) -> None: ... - @overload - def __init__(self, width: int, height: int, dc: DC) -> None: ... - @overload - def __init__(self, name: str, type: BitmapType = BITMAP_TYPE_ANY) -> None: ... - @overload - def __init__(self, img: Image, depth: int = BITMAP_SCREEN_DEPTH) -> None: ... - @overload - def __init__(self, img: Image, dc: DC) -> None: ... - @overload - def __init__(self, listOfBytes: list[bytes]) -> None: ... - def GetWidth(self) -> int: ... - def GetHeight(self) -> int: ... - def IsOk(self) -> bool: ... - -BrushStyle = int -BRUSHSTYLE_SOLID: BrushStyle -SOLID: BrushStyle -TRANSPARENT: BrushStyle - -class Brush(GDIObject): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, colour: ColourUnion, style: BrushStyle = BRUSHSTYLE_SOLID) -> None: ... - @overload - def __init__(self, stippleBitmap: Bitmap) -> None: ... - @overload - def __init__(self, brush: Brush) -> None: ... - def GetColour(self) -> Colour: ... - @overload - def SetColour(self, colour: Union[str, ColourUnion]) -> None: ... - @overload - def SetColour(self, red: int, green: int, blue: int) -> None: ... - -PenStyle = int -PENSTYLE_SOLID: PenStyle -LONG_DASH: PenStyle - -class PenInfo: ... - -class Pen(GDIObject): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, info: PenInfo) -> None: ... - @overload - def __init__(self, colour: ColourUnion, width: int = 1, style: PenStyle = PENSTYLE_SOLID) -> None: ... - @overload - def __init__(self, pen: Pen) -> None: ... - -DefaultCoord: int - -DefaultPosition: Point - -DefaultSize: Size - -class Sizer(Object): - @overload - def Add(self, window: Window, flags: SizerFlags) -> None: ... - @overload - def Add(self, window: Window, proportion: int = 0, flag: int = 0, border: int = 0, userData: Optional[PyUserData] = None) -> None: ... - @overload - def Add(self, sizer: Sizer, flags: SizerFlags) -> None: ... - @overload - def Add(self, sizer: Sizer, proportion: int = 0, flag: int = 0, border: int = 0, userData: Optional[PyUserData] = None) -> None: ... - @overload - def Add(self, width: int, height: int, proportion: int = 0, flag: int = 0, border: int = 0, userData: Optional[PyUserData] = None) -> None: ... - @overload - def Add(self, width: int, height: int, flags: SizerFlags) -> None: ... - @overload - def Add(self, item: SizerItem) -> None: ... - @overload - def Add(self, - size: SizeUnion, - proportion: int = 0, - flag: int = 0, - border: int = 0, - userData: Optional[PyUserData] = None) -> None: ... - @overload - def Add(self, size: SizeUnion, flags: SizerFlags) -> None: ... - def SetSizeHints(self, window: Window) -> None: ... - def Fit(self, window: Window) -> Size: ... - @overload - def Hide(self, window: Window, recursive: bool = False) -> bool: ... - @overload - def Hide(self, sizer: Sizer, recursive: bool = False) -> bool: ... - @overload - def Hide(self, index: int) -> bool: ... - @overload - def Show(self, window: Window, show: bool = True, recursive: bool = False) -> bool: ... - @overload - def Show(self, sizer: Sizer, show: bool = True, recursive: bool = False) -> bool: ... - @overload - def Show(self, index: int, show: bool = True) -> bool: ... - @overload - def Detach(self, window: Window) -> bool: ... - @overload - def Detach(self, sizer: Sizer) -> bool: ... - @overload - def Detach(self, index: int) -> bool: ... - @overload - def Insert(self, index: int, window: Window, flags: SizerFlags) -> SizerItem: ... - @overload - def Insert(self, - index: int, - window: Window, - proportion: int = 0, - flag: int = 0, - border: int = 0, - userData: Optional[PyUserData] = None) -> SizerItem: ... - @overload - def Insert(self, index: int, sizer: Sizer, flags: SizerFlags) -> SizerItem: ... - @overload - def Insert(self, - index: int, - sizer: Sizer, - proportion: int = 0, - flag: int = 0, - border: int = 0, - userData: Optional[PyUserData] = None) -> SizerItem: ... - @overload - def Insert(self, - index: int, - width: int, - height: int, - proportion: int = 0, - flag: int = 0, - border: int = 0, - userData: Optional[PyUserData] = None) -> SizerItem: ... - @overload - def Insert(self, - index: int, - width: int, - height: int, - flags: SizerFlags) -> SizerItem: ... - @overload - def Insert(self, - index: int, - item: SizerItem) -> SizerItem: ... - @overload - def Insert(self, - index: int, - size: Size, - proportion: int = 0, - flag: int = 0, - border: int = 0, - userData: Optional[PyUserData] = None) -> SizerItem: ... - @overload - def Insert(self, - index: int, - size: Size, - flags: SizerFlags) -> SizerItem: ... - @overload - def SetMinSize(self, size: SizeUnion) -> None: ... - @overload - def SetMinSize(self, width: int, height: int) -> None: ... - @overload - def GetItem(self, window: Window, recursive: bool = False) -> SizerItem: ... - @overload - def GetItem(self, sizer: Sizer, recursive: bool = False) -> SizerItem: ... - @overload - def GetItem(self, index: int) -> SizerItem: ... - -class BoxSizer(Sizer): - def __init__(self, orient: int = HORIZONTAL) -> None: ... - -class StaticBoxSizer(Sizer): - @overload - def __init__(self, - box: StaticBox, - orient: int = HORIZONTAL) -> None: ... - @overload - def __init__(self, - orient: int, - parent: Window, - label: str = '') -> None: ... - -class GridSizer(Sizer): - ... - -class FlexGridSizer(GridSizer): - @overload - def __init__(self, cols: int, vgap: int, hgap: int) -> None: ... - @overload - def __init__(self, cols: int, gap: SizeUnion = Size(0, 0)) -> None: ... - @overload - def __init__(self, rows: int, cols: int, vgap: int, hgap: int) -> None: ... - @overload - def __init__(self, rows: int, cols: int, gap: SizeUnion = Size(0, 0)) -> None: ... - def AddGrowableCol(self, idx: int, proportion: int = 0) -> None: ... - def AddGrowableRow(self, idx: int, proportion: int = 0) -> None: ... - -class NonOwnedWindow(Window): ... - -class PopupWindow(NonOwnedWindow): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, parent: Window, flags: int = BORDER_NONE) -> None: ... - -class TopLevelWindow(NonOwnedWindow): - def Iconize(self, iconize: bool = True) -> None: ... - def SetTitle(self, title: str) -> None: ... - -PanelNameStr: str - -class Panel(Window): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = TAB_TRAVERSAL, - name: str = PanelNameStr) -> None: ... - -ACCEL_ALT: int -ACCEL_CTRL: int -ACCEL_NORMAL: int -ACCEL_SHIFT: int - -class MenuItem(Object): - def Enable(self, enable: bool = True) -> None: ... - def GetId(self) -> int: ... - def Check(self, check: bool = True) -> None: ... - def IsChecked(self) -> bool: ... - def SetItemLabel(self, label: str) -> None: ... - -ItemKind = int -ITEM_CHECK: ItemKind -ITEM_NORMAL: ItemKind -ITEM_RADIO: ItemKind - -class Menu(EvtHandler): - @overload - def Append(self, id: int, item: str = '', helpString: str = '', kind: ItemKind = ITEM_NORMAL) -> MenuItem: ... - @overload - def Append(self, menuItem: MenuItem) -> MenuItem: ... - def AppendSeparator(self) -> MenuItem: ... - def AppendSubMenu(self, submenu: Menu, text: str, help: str = '') -> MenuItem: ... - def Check(self, id: int, check: bool) -> None: ... - def IsChecked(self, id: int) -> bool: ... - def GetMenuItems(self) -> List[MenuItem]: ... - -class MenuBar(Window): - def Append(self, menu: Menu, title: str) -> None: ... - -StatusBarNameStr: str -STB_DEFAULT_STYLE: int - -class StatusBar(Control): ... - -FrameNameStr: str - -TB_FLAT: int -TB_HORIZONTAL: int - -class ToolBarToolBase(Object): - def GetId(self) -> int: ... - -class ToolBar(Control): - def AddControl(self, control: Control, label: str = '') -> ToolBarToolBase: ... - def AddSeparator(self) -> ToolBarToolBase: ... - @overload - def AddTool(self, tool: ToolBarToolBase) -> ToolBarToolBase: ... - @overload - def AddTool(self, - toolid: int, - label: str, - bitmap: Union[Bitmap, BitmapBundle], - shortHelp: str = '', - kind: ItemKind = ITEM_NORMAL) -> ToolBarToolBase: ... - @overload - def AddTool(self, - toolid: int, - label: str, - bitmap: Union[Bitmap, BitmapBundle], - bmpDisabled: Union[Bitmap, BitmapBundle], - kind: ItemKind = ITEM_NORMAL, - shortHelp: str = '', - longHelp: str = '', - clientData: Any = None) -> ToolBarToolBase: ... - def EnableTool(self, - toolId: int, - enable: bool) -> None: ... - def Realize(self) -> bool: ... - -class Frame(TopLevelWindow): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Optional[Window], - id: int = ID_ANY, - title: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = DEFAULT_DIALOG_STYLE, - name: str = FrameNameStr) -> None: ... - def SetMenuBar(self, menuBar: MenuBar) -> None: ... - def CreateStatusBar(self, number: int = 1, style: int = STB_DEFAULT_STYLE, id: int = 0, name: str = StatusBarNameStr) -> None: ... - def GetStatusBar(self) -> StatusBar: ... - def SetStatusText(self, text: str, number: int = 0) -> None: ... - def SetToolBar(self, toolBar: ToolBar) -> None: ... - -DialogNameStr: str - -class Dialog(TopLevelWindow): - def __enter__(self) -> Dialog: ... - def __exit__(self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType]) -> None: ... - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Optional[Window], - id: int = ID_ANY, - title: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = DEFAULT_DIALOG_STYLE, - name: str = DialogNameStr) -> None: ... - def CreateButtonSizer(self, flags: int) -> Sizer: ... - def GetReturnCode(self) -> int: ... - def EndModal(self, retCode: int) -> None: ... - def ShowModal(self) -> int: ... - def ShowWindowModal(self) -> None: ... - def Centre(self, direction: int = BOTH) -> None: ... - -CHOICEDLG_STYLE: int - -class SingleChoiceDialog(Dialog): - def __init__(self, - parent: Optional[Window], - message: str, - caption: str, - choices: List[str], - style: int = CHOICEDLG_STYLE, - pos: PointUnion = DefaultPosition) -> None: ... - def GetStringSelection(self) -> str: ... - -FileSelectorPromptStr: str -FileSelectorDefaultWildcardStr: str -FileDialogNameStr: str - -FD_DEFAULT_STYLE: int -FD_OPEN: int -FD_CHANGE_DIR: int -FD_SAVE: int -FD_OVERWRITE_PROMPT: int - -class FileDialog(Dialog): - def __init__(self, - parent: Window, - message: str = FileSelectorPromptStr, - defaultDir: str = '', - defaultFile: str = '', - wildcard: str = FileSelectorDefaultWildcardStr, - style: int = FD_DEFAULT_STYLE, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - name: str = FileDialogNameStr) -> None: ... - def ShowModal(self) -> int: ... - def GetPath(self) -> str: ... - def GetFilename(self) -> str: ... - -class GenericProgressDialog(Dialog): ... - -PD_APP_MODAL: int -PD_AUTO_HIDE: int -PD_CAN_ABORT: int -PD_REMAINING_TIME: int - -class ProgressDialog(GenericProgressDialog): - def __init__(self, - title: str, - message: str, - maximum: int = 100, - parent: Optional[Window] = None, - style: int = PD_APP_MODAL | PD_AUTO_HIDE) -> None: ... - def Update(self, value: int, newmsg: str = '') -> Tuple[bool, bool]: ... # type: ignore[override] - -class MessageDialog(Dialog): - def __init__(self, - parent: Window, - message: str, - caption: str = MessageBoxCaptionStr, - style: int = OK | CENTRE, - pos: PointUnion = DefaultPosition) -> None: ... - -ControlNameStr: str - -class Control(Window): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - validator: Validator = DefaultValidator, - name: str = ControlNameStr) -> None: ... - @overload - def GetSizeFromTextSize(self, xlen: int, ylen: int = -1) -> Size: ... - @overload - def GetSizeFromTextSize(self, tsize: SizeUnion) -> Size: ... - def SetLabel(self, label: str) -> None: ... - -StaticTextNameStr: str - -class StaticText(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - label: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - name: str = StaticTextNameStr) -> None: ... - def Wrap(self, width: int) -> None: ... - -class SpinEvent(NotifyEvent): ... - -class SpinCtrl(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - value: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - min: int = 0, - max: int = 100, - initial: int = 0, - name: str = 'wxSpinCtrl') -> None: ... - def GetValue(self) -> int: ... - @overload - def SetValue(self, text: str) -> None: ... - @overload - def SetValue(self, value: int) -> None: ... - def SetRange(self, minVal: int, maxVal: int) -> None: ... - -class Validator(EvtHandler): ... - -DefaultValidator: Validator - -ClientData = int - -class ItemContainerImmutable: - def SetStringSelection(self, string: str) -> bool: ... - def GetStringSelection(self) -> str: ... - -class ItemContainer(ItemContainerImmutable): - @overload - def Append(self, item: str) -> None: ... - @overload - def Append(self, item: str, clientData: ClientData) -> None: ... - @overload - def Append(self, items: List[str]) -> None: ... - def AppendItems(self, items: List[str]) -> None: ... - def Clear(self) -> None: ... - def GetItems(self) -> List[str]: ... - -CB_DROPDOWN: int -CB_READONLY: int -TE_LEFT: int -TE_PROCESS_ENTER: int -TE_READONLY: int - -class TextEntry: - def GetValue(self) -> str: ... - def SetValue(self, value: str) -> None: ... - def SetEditable(self, editable: bool) -> None: ... - def Clear(self) -> None: ... - -ComboBoxNameStr: str - -class ComboBox(Control, ItemContainer, TextEntry): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - value: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - choices: List[str] = [], - style: int = 0, - validator: Validator = DefaultValidator, - name: str = ComboBoxNameStr) -> None: ... - def GetValue(self) -> str: ... - def SetValue(self, text: str) -> None: ... - @overload - def SetSelection(self, from_: int, to_: int) -> None: ... - @overload - def SetSelection(self, n: int) -> None: ... - def GetSelection(self) -> int: ... - def GetCurrentSelection(self) -> int: ... - -class ComboCtrl(Control, TextEntry): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - value: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - validator: Validator = DefaultValidator, - name: str = ComboBoxNameStr) -> None: ... - def SetPopupControl(self, popup: ComboPopup) -> None: ... - def SetPopupMinWidth(self, width: int) -> None: ... - def SetText(self, value: str) -> None: ... - def GetTextCtrl(self) -> TextCtrl: ... - def GetPopupWindow(self) -> Window: ... - -TextCtrlNameStr: str - -class TextCtrl(Control, TextEntry): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - value: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - validator: Validator = DefaultValidator, - name: str = TextCtrlNameStr) -> None: ... - -class AnyButton(Control): ... - -ButtonNameStr: str - -class Button(AnyButton): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - label: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - validator: Validator = DefaultValidator, - name: str = ButtonNameStr) -> None: ... - def GetLabel(self) -> str: ... - -StaticBoxNameStr: str - -class StaticBox(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - label: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - name: str = StaticBoxNameStr) -> None: ... - -ListCtrlNameStr: str -LC_ICON: int -LC_REPORT: int -LC_SORT_ASCENDING: int - -LIST_FORMAT_LEFT: int -LIST_AUTOSIZE: int - -class ListItem(Object): ... - -class ListEvent(NotifyEvent): - def GetIndex(self) -> int: ... - -class ListCtrl(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = LC_ICON, - validator: Validator = DefaultValidator, - name: str = ListCtrlNameStr) -> None: ... - def ClearAll(self) -> None: ... - @overload - def InsertColumn(self, col: int, info: ListItem) -> int: ... - @overload - def InsertColumn(self, col: int, heading: str, format: int = LIST_FORMAT_LEFT, width: int = LIST_AUTOSIZE) -> int: ... - def SetColumnWidth(self, col: int, width: int) -> bool: ... - def SetItem(self, index: int, column: int, label: str, imageId: int = -1) -> bool: ... - def SetItemBackgroundColour(self, item: int, col: ColourUnion) -> None: ... - @overload - def InsertItem(self, info: ListItem) -> int: ... - @overload - def InsertItem(self, index: int, label: str) -> int: ... - @overload - def InsertItem(self, index: int, imageIndex: int) -> int: ... - @overload - def InsertItem(self, index: int, label: str, imageIndex: int) -> int: ... - def DeleteItem(self, item: int) -> bool: ... - -RadioButtonNameStr: str - -class RadioButton(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - label: str = '', - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = LC_ICON, - validator: Validator = DefaultValidator, - name: str = RadioButtonNameStr) -> None: ... - def GetValue(self) -> bool: ... - - -class BitmapBundle: ... - -NullBitmap: BitmapBundle - -StaticBitmapNameStr: str - -class StaticBitmap(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - bitmap: Union[Bitmap, BitmapBundle] = NullBitmap, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = 0, - name: str = StaticBitmapNameStr) -> None: ... - -CheckBoxNameStr: str - -class CheckBox(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - label: str = '', - pos: Point = DefaultPosition, - size: Size = DefaultSize, - style: int = 0, - validator: Validator = DefaultValidator, - name: str = CheckBoxNameStr) -> None: ... - def SetValue(self, state: bool) -> None: ... - def GetValue(self) -> bool: ... - -ChoiceNameStr: str - -class Choice(Control, ItemContainer): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - choices: List[str] = [], - style: int = 0, - validator: Validator = DefaultValidator, - name: str = ChoiceNameStr) -> None: ... - def GetCurrentSelection(self) -> int: ... - def GetString(self, n: int) -> str: ... - def SetSelection(self, n: int) -> None: ... - -StaticLineNameStr: str - -class StaticLine(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: Point = DefaultPosition, - size: Size = DefaultSize, - style: int = 0, - name: str = StaticLineNameStr) -> None: ... - -class ComboPopup: - def __init__(self) -> None: ... - def GetControl(self) -> Window: ... - def GetComboCtrl(self) -> ComboCtrl: ... - def Dismiss(self) -> None: ... - -SliderNameStr: str -SL_HORIZONTAL: int - -class Slider(Control): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - value: int = 0, - minValue: int = 0, - maxValue: int = 100, - pos: Point = DefaultPosition, - size: Size = DefaultSize, - style: int = SL_HORIZONTAL, - validator: Validator = DefaultValidator, - name: str = SliderNameStr) -> None: ... - def SetRange(self, minValue: int, maxValue: int) -> None: ... - def GetValue(self) -> int: ... - def SetValue(self, value: int) -> None: ... - -class TreeItemId: - def IsOk(self) -> bool: ... - -class WithImages: ... - -TreeCtrlNameStr: str - -TR_DEFAULT_STYLE: int -TR_HIDE_ROOT: int -TR_HAS_BUTTONS: int -TR_SINGLE: int -TR_FULL_ROW_HIGHLIGHT: int -TR_MULTIPLE: int - -class TreeEvent(NotifyEvent): - def GetKeyEvent(self) -> KeyEvent: ... - def GetItem(self) -> TreeItemId: ... - -class TreeCtrl(Control, WithImages): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = TR_DEFAULT_STYLE, - validator: Validator = DefaultValidator, - name: str = TreeCtrlNameStr) -> None: ... - def DeleteChildren(self, item: TreeItemId) -> None: ... - def AppendItem(self, - parent: TreeItemId, - text: str, - image: int = -1, - selImage: int = -1, - data: Optional[Any] = None) -> TreeItemId: ... - def SetItemHasChildren(self, item: TreeItemId, hasChildren: bool = True) -> None: ... - def SetQuickBestSize(self, quickBestSize: bool) -> None: ... - def AddRoot(self, - text: str, - image: int = -1, - selImage: int = -1, - data: Optional[Any] = None) -> TreeItemId: ... - def GetItemText(self, item: TreeItemId) -> str: ... - def GetSelection(self) -> TreeItemId: ... - def Expand(self, item: TreeItemId) -> None: ... - def SelectItem(self, item: TreeItemId, select: bool = True) -> None: ... - -RasterOperationMode = int -COPY: RasterOperationMode - -class DC(Object): - def GetTextExtent(self, st: str) -> Size: ... - def SetPen(self, pen: Pen) -> None: ... - def SetBrush(self, brush: Brush) -> None: ... - def GetBackground(self) -> Brush: ... - def SetBackground(self, brush: Brush) -> None: ... - def SetBackgroundMode(self, mode: int) -> None: ... - @overload - def DrawBitmap(self, bitmap: Bitmap, x: int, y: int, useMask: bool = False) -> None: ... - @overload - def DrawBitmap(self, bmp: Bitmap, pt: PointUnion, useMask: bool = False) -> None: ... - @overload - def DrawLine(self, x1: int, y1: int, x2: int, y2: int) -> None: ... - @overload - def DrawLine(self, pt1: PointUnion, pt2: PointUnion) -> None: ... - @overload - def DrawRectangle(self, x: int, y: int, width: int, height: int) -> None: ... - @overload - def DrawRectangle(self, pt: PointUnion, sz: SizeUnion) -> None: ... - @overload - def DrawRectangle(self, rect: Rect) -> None: ... - @overload - def DrawText(self, text: str, x: int, y: int) -> None: ... - @overload - def DrawText(self, text: str, pt: PointUnion) -> None: ... - def DrawTextList(self, - textList: List[str], - coords: List[PointUnion], - foregrounds: Optional[Union[ColourUnion, List[ColourUnion]]] = None, - backgrounds: Optional[Union[ColourUnion, List[ColourUnion]]] = None) -> None: ... - @overload - def SetClippingRegion(self, x: int, y: int, width: int, height: int) -> None: ... - @overload - def SetClippingRegion(self, pt: PointUnion, sz: SizeUnion) -> None: ... - @overload - def SetClippingRegion(self, rect: Rect) -> None: ... - def DestroyClippingRegion(self) -> None: ... - def GetFont(self) -> Font: ... - def SetFont(self, font: Font) -> None: ... - def SetLogicalScale(self, x: float, y: float) -> None: ... - def GetSize(self) -> Size: ... - def Clear(self) -> None: ... - def Blit(self, - xdest: int, - ydest: int, - width: int, - height: int, - source: DC, - xsrc: int, - ysrc: int, - logicalFunc: RasterOperationMode = COPY, - useMask: bool = False, - xsrcMask: int = DefaultCoord, - ysrcMask: int = DefaultCoord) -> bool: ... - def StretchBlit(self, - xdest: int, - ydest: int, - dstWidth: int, - dstHeight: int, - source: DC, - xsrc: int, - ysrc: int, - srcWidth: int, - srcHeight: int, - logicalFunc: RasterOperationMode = COPY, - useMask: bool = False, - xsrcMask: int = DefaultCoord, - ysrcMask: int = DefaultCoord) -> bool: ... - -class MemoryDC(DC): - def SelectObject(self, bitmap: Bitmap) -> None: ... - -class WindowDC(DC): ... - -class PrinterDC(DC): ... - -class GraphicsObject(Object): ... - -class GraphicsContext(GraphicsObject): - @overload - @staticmethod - def Create(window: Window) -> GraphicsContext: ... - @overload - @staticmethod - def Create(windowDC: WindowDC) -> GraphicsContext: ... - @overload - @staticmethod - def Create(memoryDC: MemoryDC) -> GraphicsContext: ... - @overload - @staticmethod - def Create(printerDC: PrinterDC) -> GraphicsContext: ... - @overload - @staticmethod - def Create(image: Image) -> GraphicsContext: ... - @overload - @staticmethod - def Create() -> GraphicsContext: ... - @overload - @staticmethod - def Create(autoPaintDC: AutoBufferedPaintDC) -> GraphicsContext: ... - -class GCDC(DC): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, windowDC: WindowDC) -> None: ... - @overload - def __init__(self, memoryDC: MemoryDC) -> None: ... - @overload - def __init__(self, printerDC: PrinterDC) -> None: ... - @overload - def __init__(self, context: GraphicsContext) -> None: ... - -class BufferedDC(MemoryDC): ... -class BufferedPaintDC(BufferedDC): ... -class AutoBufferedPaintDC(BufferedPaintDC): - def __init__(self, window: Window) -> None: ... - -FontFamily = int -FONTFAMILY_MODERN: FontFamily -SWISS: FontFamily - -FontStyle = int -NORMAL: FontStyle - -FontWeight = int -FONTWEIGHT_BOLD: FontWeight -BOLD: FontWeight - -FontEncoding = int -FONTENCODING_DEFAULT: FontEncoding - -class FontInfo: ... - -class NativeFontInfo: ... - -class Font(GDIObject): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, font: Font) -> None: ... - @overload - def __init__(self, fontInfo: FontInfo) -> None: ... - @overload - def __init__(self, - pointSize: int, - family: FontFamily, - style: FontStyle, - weight: FontWeight, - underline: bool = False, - faceName: str = '', - encoding: FontEncoding = FONTENCODING_DEFAULT) -> None: ... - @overload - def __init__(self, - pixelSize: SizeUnion, - family: FontFamily, - style: FontStyle, - weight: FontWeight, - underline: bool = False, - faceName: str = '', - encoding: FontEncoding = FONTENCODING_DEFAULT) -> None: ... - @overload - def __init__(self, nativeInfoString: str) -> None: ... - @overload - def __init__(self, nativeInfo: NativeFontInfo) -> None: ... - def SetWeight(self, weight: FontWeight) -> None: ... - def GetPixelSize(self) -> Size: ... - def GetPointSize(self) -> int: ... - def Underlined(self) -> Font: ... - -class FontEnumerator: - @staticmethod - def IsValidFacename(facename: str) -> bool: ... - -ArtID = int -ART_NEW: ArtID -ART_CLOSE: ArtID -ART_COPY: ArtID -ART_DELETE: ArtID -ART_FILE_SAVE: ArtID -ART_FILE_SAVE_AS: ArtID -ART_FILE_OPEN: ArtID -ART_GO_BACK: ArtID -ART_GO_DOWN: ArtID -ART_GO_FORWARD: ArtID -ART_GO_UP: ArtID -ART_MINUS: ArtID -ART_PLUS: ArtID -ART_REDO: ArtID -ART_TOOLBAR: ArtID -ART_UNDO: ArtID -ART_WARNING: ArtID - -ArtClient = int -ART_MESSAGE_BOX: ArtClient -ART_OTHER: ArtClient - -class ArtProvider(Object): - @staticmethod - def GetBitmap(id: ArtID, client: ArtClient = ART_OTHER, size: SizeUnion = DefaultSize) -> Bitmap: ... - -class DataObject: ... -class DataObjectSimple(DataObject): ... -class TextDataObject(DataObjectSimple): - def SetText(self, strText: str) -> None: ... - -class Clipboard(Object): - def Open(self) -> bool: ... - def SetData(self, data: DataObject) -> bool: ... - def IsOpened(self) -> bool: ... - def Close(self) -> None: ... - -TheClipboard: Clipboard - -class AcceleratorEntry: ... - -class AcceleratorTable(Object): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, entries: Sequence[Union[AcceleratorEntry, Tuple[int, int, int]]]) -> None: ... - -NullAcceleratorTable: AcceleratorTable - -class Display: - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, index: int) -> None: ... - @overload - def __init__(self, window: Window) -> None: ... - @staticmethod - def GetCount() -> int: ... - def IsPrimary(self) -> bool: ... - @property - def Geometry(self) -> Rect: ... - def GetPPI(self) -> Size: ... - -def GetDisplayPPI() -> Size: ... - -OS_MAC_OS: int -OS_MAC_OSX_DARWIN: int -OS_MAC: int -def GetOsVersion(micro: Optional[int] = None) -> Tuple[int, int, int]: ... -def GetMousePosition() -> Point: ... -def SetCursor(cursor: Cursor) -> None: ... -def BeginBusyCursor(cursor: Union[StockCursor, Cursor] = HOURGLASS_CURSOR) -> None: ... -def IsBusy() -> bool: ... -def EndBusyCursor() -> None: ... -def SafeYield(win: Optional[Window] = None, onlyIfNeeded: bool = False) -> bool: ... -def ImageFromBitmap(bitmap: Bitmap) -> Image: ... -def BitmapFromImage(image: Image) -> Bitmap: ... -def PostEvent(dest: EvtHandler, event: Union[PyCommandEvent, Event]) -> None: ... -def CallAfter(callableObj: Callable, *args: Any, **kwargs: Any) -> None: ... - -MessageBoxCaptionStr: str -def MessageBox(message: str, - caption: str = MessageBoxCaptionStr, - style: int = OK | CENTRE, - parent: Optional[Window] = None, - x: int = DefaultCoord, - y: int = DefaultCoord) -> int: ... - -class PyDeadObjectError(Exception): ... diff --git a/helios/pipeViewer/stubs/wx/html.pyi b/helios/pipeViewer/stubs/wx/html.pyi deleted file mode 100644 index 32bfb28fba..0000000000 --- a/helios/pipeViewer/stubs/wx/html.pyi +++ /dev/null @@ -1,21 +0,0 @@ -from wx import Window, Panel, Scrolled, ID_ANY, DefaultPosition, DefaultSize, PointUnion, SizeUnion -from typing import overload - -class HtmlWindowInterface: - ... - -HW_DEFAULT_STYLE: int -HW_SCROLLBAR_AUTO: int - -class HtmlWindow(Scrolled, Panel, HtmlWindowInterface): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, - parent: Window, - id: int = ID_ANY, - pos: PointUnion = DefaultPosition, - size: SizeUnion = DefaultSize, - style: int = HW_DEFAULT_STYLE, - name: str = 'htmlWindow') -> None: ... - def SetPage(self, source: str) -> bool: ... diff --git a/helios/pipeViewer/transactionsearch/CMakeLists.txt b/helios/pipeViewer/transactionsearch/CMakeLists.txt deleted file mode 100644 index 6373d7ecee..0000000000 --- a/helios/pipeViewer/transactionsearch/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -project(TransactionSearch CXX) - -set (CMAKE_CXX_STANDARD 17) -set (CMAKE_CXX_STANDARD_REQUIRED ON) - -add_executable(transactionsearch src/transaction_search.cpp) - -target_include_directories(transactionsearch PRIVATE ${CMAKE_SOURCE_DIR}/pipeViewer/pipe_view) -target_link_libraries(transactionsearch SPARTA::sparta) - -install(TARGETS transactionsearch RUNTIME) diff --git a/helios/pipeViewer/transactionsearch/src/transaction_search.cpp b/helios/pipeViewer/transactionsearch/src/transaction_search.cpp deleted file mode 100644 index d90fcae2cc..0000000000 --- a/helios/pipeViewer/transactionsearch/src/transaction_search.cpp +++ /dev/null @@ -1,411 +0,0 @@ -/* - */ - -#include "transactiondb/src/Reader.hpp" -#include "transactiondb/src/PipelineDataCallback.hpp" -#include -#include -#include -#include -#include - - -namespace sparta { - namespace pipeViewer { - - /*! \brief Base class for search callbacks that holds configuration parameters and common helper methods - */ - class BaseSearchCallback : public PipelineDataCallback { - protected: - static constexpr char RESULT_TAG_ = 'r'; - static constexpr char PROGRESS_TAG_ = 'p'; - static constexpr char INFO_TAG_ = 'i'; - static constexpr char START_DELIMITER_ = ':'; - static constexpr uint64_t NUMBER_OF_PROGRESS_UPDATES_ = 50; - - /*! Invert search */ - const bool invert_search_; - - /*! Location IDs to include in the search */ - std::set locations_; - - /*! For update progress */ - uint64_t last_step_number_ = 0; - uint64_t search_start_ = 0; - uint64_t search_end_ = 0; - uint64_t search_width_ = 0; - double search_update_stride_ = 0; - - /*! Incremented by one every regular expression match */ - uint64_t hits_ = 0; - - uint64_t recs_viewed_ = 0; - uint64_t recs_with_annot_ = 0; - uint64_t recs_with_ins_ = 0; - uint64_t recs_with_mem_ = 0; - uint64_t recs_with_non_null_annot_ = 0; - uint64_t recs_with_pair_ = 0; - - inline void handleProgressOutput_(const uint64_t current_time) { - uint64_t step; - if (search_update_stride_ != 0) { - step = current_time / search_update_stride_; - } else { - // Step is small that the search doesn't really need progress indicators - step = 100000000; - } - if (step > last_step_number_ && current_time > search_start_) { - float fraction = (current_time - search_start_) / static_cast(search_width_); - std::cout << PROGRESS_TAG_ << fraction << std::endl; - last_step_number_ = step; - } - } - - /*! - * \brief Writes a result to stdout in the form: - * ",@\n" - */ - inline void handleResultOutput_(const uint64_t start_time, - const uint64_t end_time, - const uint64_t location_id, - const std::string& str) { - std::cout << RESULT_TAG_ - << start_time << "," << end_time << "@" << location_id - << START_DELIMITER_; - - if(!str.empty()) { - for(auto c: str) { - if(c == '\n' || c == '\r') { - std::cout << "\\n"; - } - else { - std::cout << c; - } - } - } - - std::cout << std::endl; - } - - /*! - * \brief Writes a result to stdout in the form: - * ",@\n" - */ - inline void handleResultOutput_(const annotation_t* annotation) { - handleResultOutput_(annotation->time_Start, - annotation->time_End, - annotation->location_ID, - annotation->annt); - } - - inline void handleResultOutput_(const pair_t* pairt, const std::string& formatted_pair) { - handleResultOutput_(pairt->time_Start, - pairt->time_End, - pairt->location_ID, - formatted_pair); - } - - public: - BaseSearchCallback(const char* invert_search_str, const char* location_str) : - invert_search_(!!strtoull(invert_search_str, nullptr, 10)) - { - std::istringstream ss(location_str); - uint32_t id; - while (ss >> id) { - locations_.insert(id); - if (ss.peek() == ',') { - ss.ignore(); - } - } - } - - void setSearchParams(const uint64_t search_start, const uint64_t search_end) { - search_start_ = search_start; - search_end_ = search_end; - search_width_ = search_end - search_start; - search_update_stride_ = double(search_width_) / NUMBER_OF_PROGRESS_UPDATES_; - } - - void startProgress() const { - std::cout << INFO_TAG_ << "search start: " << search_start_ << std::endl - << INFO_TAG_ << "search locs: (" << locations_.size() << ") ["; - - for (const auto& lid : locations_) { - std::cout << lid << " "; - } - - std::cout << "]" << std::endl - << INFO_TAG_ << "search invert: " << invert_search_ << std::endl; - } - - void finishedProgress() const { - std::cout << PROGRESS_TAG_ << 1 << std::endl //finish off so progress bar doesn't hang - << INFO_TAG_ << "Number of records: " << recs_viewed_ << std::endl - << INFO_TAG_ << "Number of records with annotation: " << recs_with_annot_ << std::endl - << INFO_TAG_ << "Number of records with instruction: " << recs_with_ins_ << std::endl - << INFO_TAG_ << "Number of records with memory: " << recs_with_mem_ << std::endl - << INFO_TAG_ << "Number of records with pair: " << recs_with_pair_ << std::endl - << INFO_TAG_ << "Number of non-null annotations (searched): " << recs_with_non_null_annot_ << std::endl - << INFO_TAG_ << "Number of hits: " << hits_ << std::endl; - } - }; - - /*! \brief Callback that compares annotations to a string - */ - class SearchStringCallback : public BaseSearchCallback { - private: - /*! Stores string query for comparison in callbacks */ - std::string string_query_; - - public: - SearchStringCallback(std::string query, const char* invert_search_str, const char* location_str) : - BaseSearchCallback(invert_search_str, location_str), - string_query_(std::move(query)) - { - } - - virtual void foundAnnotationRecord(const annotation_t* annotation) override { - handleProgressOutput_(annotation->time_Start); - ++recs_viewed_; - ++recs_with_annot_; - - if (annotation->time_Start > search_end_ || annotation->time_End < search_start_) { - return; - } - - if (!locations_.empty() && locations_.count(annotation->location_ID) == 0) { - return; - } - - std::string_view annt_string(annotation->annt); - if (!annt_string.empty()) { - ++recs_with_non_null_annot_; - if (invert_search_) { - // Inverted search for string keys is a FULL-STRING match - if (annt_string != string_query_) { - handleResultOutput_(annotation); - ++hits_; - } - } - else { - size_t position = annt_string.find(string_query_); - if (position != std::string::npos) { - handleResultOutput_(annotation); - ++hits_; - } - } - } - } - - virtual void foundInstRecord(const instruction_t*) override { - ++recs_viewed_; - ++recs_with_ins_; - } - - virtual void foundMemRecord(const memoryoperation_t*) override { - ++recs_viewed_; - ++recs_with_mem_; - } - - virtual void foundPairRecord(const pair_t* pair) override { - handleProgressOutput_(pair->time_Start); - ++recs_viewed_; - ++recs_with_pair_; - - if (pair->time_Start > search_end_ || pair->time_End < search_start_) { - return; - } - - if (!locations_.empty() && locations_.count(pair->location_ID) == 0) { - return; - } - - const auto annt_string = formatPairAsAnnotation(pair); - if (!annt_string.empty()) { - ++recs_with_non_null_annot_; - if (invert_search_) { - // Inverted search for string keys is a FULL-STRING match - if (annt_string != string_query_) { - handleResultOutput_(pair, annt_string); - ++hits_; - } - } - else { - size_t position = annt_string.find(string_query_); - if (position != std::string::npos) { - handleResultOutput_(pair, annt_string); - ++hits_; - } - } - } - } - }; - - /*! \brief Callback that compares annotations to regex - */ - class SearchRegexCallback : public BaseSearchCallback { - private: - /*! Stores regular expression for comparison in callbacks */ - std::regex regular_expression_; - - public: - SearchRegexCallback(const char* regex, const char* invert_search_str, const char* location_str) : - BaseSearchCallback(invert_search_str, location_str), - regular_expression_(regex) - { - } - - virtual void foundAnnotationRecord(const annotation_t* annotation) override { - handleProgressOutput_(annotation->time_Start); - ++recs_viewed_; - ++recs_with_annot_; - if (!locations_.empty() && locations_.count(annotation->location_ID) == 0) { - return; - } - if (annotation->time_Start > search_end_ || annotation->time_End < search_start_) { - return; - } - if (!annotation->annt.empty()) { - ++recs_with_non_null_annot_; - if ((!invert_search_) == std::regex_search(annotation->annt, std::regex(regular_expression_))) { - handleResultOutput_(annotation); - ++hits_; - } - } - } - - virtual void foundInstRecord(const instruction_t*) override { - ++recs_viewed_; - ++recs_with_ins_; - } - - virtual void foundMemRecord(const memoryoperation_t*) override { - ++recs_viewed_; - ++recs_with_mem_; - } - - virtual void foundPairRecord(const pair_t* pair) override { - handleProgressOutput_(pair->time_Start); - ++recs_viewed_; - ++recs_with_pair_; - - if (pair->time_Start > search_end_ || pair->time_End < search_start_) { - return; - } - - if (!locations_.empty() && locations_.count(pair->location_ID) == 0) { - return; - } - - const auto annt_string = formatPairAsAnnotation(pair); - if (!annt_string.empty()) { - ++recs_with_non_null_annot_; - if ((!invert_search_) == std::regex_search(annt_string, std::regex(regular_expression_))) { - handleResultOutput_(pair, annt_string); - ++hits_; - } - } - } - }; - } //namespace pipeViewer -} //namespace sparta - -class ConstructReaderException : public std::exception { - private: - std::string type_; - - public: - explicit ConstructReaderException(const char* type) : - type_("unknown search type ") - { - type_ += type; - } - - const char* what() const noexcept final { - return type_.c_str(); - } -}; - -static sparta::pipeViewer::Reader constructReader(char** argv) { - if (strcmp(argv[2], "string") == 0) { - return sparta::pipeViewer::Reader::construct(argv[1], - argv[3], - argv[4], - argv[7]); - } - - if (strcmp(argv[2], "regex") == 0) { - return sparta::pipeViewer::Reader::construct(argv[1], - argv[3], - argv[4], - argv[7]); - } - - throw ConstructReaderException(argv[2]); -} - -/*! - * \brief Location search main. - * - * argv expected to contain - * \li 1: Database prefix - * \li 2: Search model ("regex" or "string") - * \li 3: Search expression - * \li 4: Invert Search (1=yes, 0=no). On regex, only mismatches will be in the result set. On - * string-match searches where this is 1, a FULL-STRING comparison will be performed and - * annotations which differ from the full query are matched. When 0 for string-match searches - * matches are found when the query string is within an annotation - * \li 5: Search Start tick. -1 implies start of file - * \li 6: Search End tick. -1 implies end of file - * \li 7: Location filter. Comma-delimited list of location IDs. If empty, no filtering is done - */ -int main(int argc, char** argv) { - if (argc != 8) { - std::cout << "Usage: transactionsearch " << std::endl; - return 1; - } - - try { - sparta::pipeViewer::Reader reader = constructReader(argv); - - uint64_t search_start; - uint64_t search_end; - - int64_t tmp = std::strtoll(argv[5], nullptr, 10); - if (tmp < 0) { - search_start = reader.getCycleFirst(); - } - else { - search_start = static_cast(tmp); - } - - tmp = std::strtoll(argv[6], nullptr, 10); - if (tmp < 0) { - search_end = reader.getCycleLast(); - } - else { - search_end = static_cast(tmp); - } - - // Reject negative-length searches. - // Allow 0-length search or negative search if start is past eof-cycle - // because it is common for eof-cycle+1 to be used as search_start and - // end_cycle to be automatically computed as eof-cycle - if (search_start < reader.getCycleLast() && search_end < search_start) { - std::cerr << "negative search range [" << search_start << ", " << search_end << ")" << std::endl; - return 1; - } - - auto& cb = reader.getCallbackAs(); - cb.setSearchParams(search_start, search_end); - - cb.startProgress(); - reader.getWindow(search_start, search_end); - cb.finishedProgress(); - } - catch(const ConstructReaderException& e) { - std::cerr << e.what() << std::endl; - return 1; - } - return 0; -} diff --git a/helios/pipeViewer/wxPythonInclude.py b/helios/pipeViewer/wxPythonInclude.py deleted file mode 100644 index 97203052fb..0000000000 --- a/helios/pipeViewer/wxPythonInclude.py +++ /dev/null @@ -1,12 +0,0 @@ -# Small util to print the include path for wxPython - -import pkgutil -import os.path -from _frozen_importlib_external import SourceFileLoader - -wx_pkg = pkgutil.get_loader('wx') -if wx_pkg is None: - raise RuntimeError("wx python library is not installed!") -assert isinstance(wx_pkg, SourceFileLoader) - -print(os.path.join(os.path.dirname(wx_pkg.get_filename()), 'include')) diff --git a/sparta/test/CMakeLists.txt b/sparta/test/CMakeLists.txt index 89ad1ae9f0..8a393cf578 100644 --- a/sparta/test/CMakeLists.txt +++ b/sparta/test/CMakeLists.txt @@ -44,7 +44,6 @@ add_custom_target (regress_valgrind) add_custom_command (TARGET regress POST_BUILD COMMAND ctest -LE ${VALGRIND_TEST_LABEL} -j${NUM_CORES} --test-action test) add_custom_command (TARGET regress_valgrind POST_BUILD COMMAND ctest -L ${VALGRIND_TEST_LABEL} -j${NUM_CORES} --test-action test) -#add_subdirectory (pipeViewer) add_subdirectory (Array) add_subdirectory (Audience) add_subdirectory (BasicHistogram) From 7ecddf76e7f4fca44b5b331f5e2b3354d56604f0 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Tue, 25 Feb 2025 12:51:09 -0600 Subject: [PATCH 46/49] SimDB / v3 integration --- sparta/simdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparta/simdb b/sparta/simdb index e6c4148178..ce6130c7f5 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit e6c4148178e67430a9f760fd269805f388ea1fe1 +Subproject commit ce6130c7f5c1858a334b9af357bf7db6067a303e From 757ba977c6b764372e12133e8ab716620760a550 Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Sat, 1 Mar 2025 08:09:33 -0600 Subject: [PATCH 47/49] SimDB / v3 integration --- sparta/simdb | 2 +- sparta/sparta/app/AppTriggers.hpp | 5 +++-- sparta/sparta/app/CommandLineSimulator.hpp | 5 +++++ sparta/sparta/collection/PipelineCollector.hpp | 16 +++++++++++++--- sparta/src/CommandLineSimulator.cpp | 12 +++++++++++- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/sparta/simdb b/sparta/simdb index ce6130c7f5..5495314398 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit ce6130c7f5c1858a334b9af357bf7db6067a303e +Subproject commit 5495314398e32a79fd33b3dc9c1a3d9b77186309 diff --git a/sparta/sparta/app/AppTriggers.hpp b/sparta/sparta/app/AppTriggers.hpp index dd6d04686e..44ae89c33c 100644 --- a/sparta/sparta/app/AppTriggers.hpp +++ b/sparta/sparta/app/AppTriggers.hpp @@ -33,14 +33,15 @@ class PipelineTrigger : public trigger::Triggerable const std::set& pipeline_enabled_node_names, uint64_t pipeline_heartbeat, bool multiple_triggers, - sparta::RootTreeNode * rtn) : + sparta::RootTreeNode * rtn, + size_t pipeline_num_compression_threads = 2) : pipeline_collection_path_(pipeline_collection_path), pipeline_enabled_node_names_(pipeline_enabled_node_names), multiple_triggers_(multiple_triggers), root_(rtn) { auto simdb_filename = getCollectionPath_(); - pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, pipeline_heartbeat, rtn)); + pipeline_collector_.reset(new sparta::collection::PipelineCollector(simdb_filename, pipeline_heartbeat, rtn, pipeline_num_compression_threads)); } void go() override diff --git a/sparta/sparta/app/CommandLineSimulator.hpp b/sparta/sparta/app/CommandLineSimulator.hpp index a2eebe7841..8d3c7e5193 100644 --- a/sparta/sparta/app/CommandLineSimulator.hpp +++ b/sparta/sparta/app/CommandLineSimulator.hpp @@ -461,6 +461,11 @@ class CommandLineSimulator */ uint64_t pipeline_heartbeat_ = 10; + /*! + * \brief Maximum number of compression threads to use for pipeline collection. + */ + size_t pipeline_num_compression_threads_ = 2; + //! The names of the nodes to be enabled std::set pipeline_enabled_node_names_; diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index 15a5f7377c..ce6fd66abb 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -80,6 +80,7 @@ namespace collection ev_collect_->setScheduleableClock(clk); ev_collect_->setScheduler(clk->getScheduler()); ev_collect_->setContinuing(false); + scheduler_ = ev_collect_->getScheduler(); } void enable(CollectableTreeNode * ctn) { @@ -98,6 +99,14 @@ namespace collection } void performCollection() { + const auto tick = scheduler_->getCurrentTick(); + performCollectionAt_(tick - 1); + } + + private: + void performCollectionAt_(sparta::Scheduler::Tick tick) { + if (LOG_MINIFICATION) std::cout << "\n\n[simdb verbose] Performing collection at tick " << tick << "\n"; + for(auto & ctn : enabled_ctns_) { if(ctn->isCollected()) { //This is happening on a specific clock and a specific phase. @@ -115,10 +124,10 @@ namespace collection } } - private: EventSet ev_set_; std::unique_ptr ev_collect_; std::set enabled_ctns_; + sparta::Scheduler* scheduler_ = nullptr; }; // A map of the clock pointer and the structures that @@ -133,7 +142,8 @@ namespace collection public: PipelineCollector(const std::string& simdb_filename, const size_t heartbeat, - sparta::TreeNode * root) + sparta::TreeNode * root, + size_t num_compression_threads = 2) : db_mgr_(std::make_unique(simdb_filename, true)) { sparta_assert(root->isFinalized() == true, @@ -152,7 +162,7 @@ namespace collection // DatabaseManager adds automatically to support this feature. simdb::Schema schema; db_mgr_->createDatabaseFromSchema(schema); - db_mgr_->enableCollection(heartbeat); + db_mgr_->enableCollection(heartbeat, num_compression_threads); // Initialize the clock/collectable map std::function addClks; diff --git a/sparta/src/CommandLineSimulator.cpp b/sparta/src/CommandLineSimulator.cpp index 6a126c4eb2..ed91152edc 100644 --- a/sparta/src/CommandLineSimulator.cpp +++ b/sparta/src/CommandLineSimulator.cpp @@ -376,6 +376,11 @@ CommandLineSimulator::CommandLineSimulator(const std::string& usage, ("heartbeat", named_value("HEARTBEAT", &pipeline_heartbeat_)->default_value(pipeline_heartbeat_), heartbeat_doc.str().c_str()) + ("pipeline-num-compression-threads", + named_value("PIPELINE_MAX_ZLIB_THREADS", &pipeline_num_compression_threads_)->default_value(pipeline_num_compression_threads_), + "The number of threads to use for compressing data before writing to the database. ") + ("log-pipeline-minification", + "Enable logging of minification statistics for pipeline collection") ; std::stringstream arch_search_dirs_str; @@ -1772,6 +1777,10 @@ bool CommandLineSimulator::parse(int argc, if (!fs_path.has_extension() || fs_path.extension() != ".db") { simdb_filename += ".db"; } + + if (vm_.count("log-pipeline-minification") > 0) { + simdb::CollectionPointBase::enableMinificationLogging(); + } } //pevents @@ -2031,7 +2040,8 @@ void CommandLineSimulator::populateSimulation_(Simulation* sim) pipeline_enabled_node_names_, pipeline_heartbeat_, multiple_triggers, - sim->getRoot())); + sim->getRoot(), + pipeline_num_compression_threads_)); } // Finalize the pevent controller now that the tree is built. From 82ae5f412cb379586cfcadb1dfda1982338dae8f Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Tue, 4 Mar 2025 11:11:21 -0600 Subject: [PATCH 48/49] SimDB / v3 integration --- .../CoreModel/src/LoadStoreInstInfo.hpp | 10 +++- .../example/CoreModel/src/MemAccessInfo.hpp | 10 +++- sparta/simdb | 2 +- sparta/sparta/collection/Collectable.hpp | 15 +++++- .../sparta/collection/CollectableTreeNode.hpp | 5 ++ .../sparta/collection/IterableCollector.hpp | 24 ++++++++- .../sparta/collection/PipelineCollector.hpp | 49 +++++++++++++++---- 7 files changed, 99 insertions(+), 16 deletions(-) diff --git a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp index 1bc6b646aa..0084d96d48 100644 --- a/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp +++ b/sparta/example/CoreModel/src/LoadStoreInstInfo.hpp @@ -163,6 +163,11 @@ inline void defineStructSchema(StructSchema("DID"); schema.addEnum("rank"); schema.addEnum("state"); + + //TODO cnyce + //StructSchema mem_access_info_schema; + //defineStructSchema(mem_access_info_schema); + //schema.stealFieldsFrom(mem_access_info_schema); } template <> @@ -174,8 +179,9 @@ inline void writeStructFields( serializer->writeField(inst->getPriority()); serializer->writeField(inst->getState()); - core_example::MemoryAccessInfo* mem_access_info = inst->getMemoryAccessInfoPtr().get(); - StructSerializer::getInstance()->writeStruct(mem_access_info, serializer->getBuffer()); + //TODO cnyce + //core_example::MemoryAccessInfo* mem_access_info = inst->getMemoryAccessInfoPtr().get(); + //StructSerializer::getInstance()->writeStruct(mem_access_info, serializer->getBuffer()); } } // namespace simdb diff --git a/sparta/example/CoreModel/src/MemAccessInfo.hpp b/sparta/example/CoreModel/src/MemAccessInfo.hpp index c30fbed459..985dcb5024 100644 --- a/sparta/example/CoreModel/src/MemAccessInfo.hpp +++ b/sparta/example/CoreModel/src/MemAccessInfo.hpp @@ -191,6 +191,11 @@ inline void defineStructSchema(StructSchema("mmu"); schema.addEnum("cache"); + + //TODO cnyce + //StructSchema ex_inst_schema; + //defineStructSchema(ex_inst_schema); + //schema.stealFieldsFrom(ex_inst_schema); } template <> @@ -203,8 +208,9 @@ inline void writeStructFields( serializer->writeField(inst->getMMUState()); serializer->writeField(inst->getCacheState()); - core_example::ExampleInst* ex_inst = inst->getInstPtr().get(); - StructSerializer::getInstance()->writeStruct(ex_inst, serializer->getBuffer()); + //TODO cnyce + //core_example::ExampleInst* ex_inst = inst->getInstPtr().get(); + //StructSerializer::getInstance()->writeStruct(ex_inst, serializer->getBuffer()); } } // namespace simdb diff --git a/sparta/simdb b/sparta/simdb index 5495314398..0101510b22 160000 --- a/sparta/simdb +++ b/sparta/simdb @@ -1 +1 @@ -Subproject commit 5495314398e32a79fd33b3dc9c1a3d9b77186309 +Subproject commit 0101510b22924723ff20c3e6df9272914c3c0b54 diff --git a/sparta/sparta/collection/Collectable.hpp b/sparta/sparta/collection/Collectable.hpp index 9af665b616..c6c9dda3e5 100644 --- a/sparta/sparta/collection/Collectable.hpp +++ b/sparta/sparta/collection/Collectable.hpp @@ -43,7 +43,7 @@ namespace sparta{ */ template - class Collectable : public CollectableTreeNode + class Collectable : public CollectableTreeNode, public simdb::TickReader { public: /** @@ -104,6 +104,14 @@ namespace sparta{ //! Virtual destructor -- does nothing virtual ~Collectable() {} + uint64_t getTick() const override { + return getClock()->getScheduler()->getCurrentTick() - 1; + } + + uint16_t getElemId() const override { + return simdb_collectable_->getElemId(); + } + //! Explicitly/manually collect a value for this collectable, ignoring //! what the Collectable is currently pointing to. //! Here we pass the actual object of the collectable type we are collecting. @@ -227,6 +235,11 @@ namespace sparta{ void configCollectable(simdb::CollectionMgr *mgr) override final { using value_type = MetaStruct::remove_any_pointer_t; simdb_collectable_ = mgr->createCollectable(getLocation(), getClock()->getName()); + simdb_collectable_->setTickReader(*this); + } + + simdb::CollectionPointBase* getSimDbCollectable() const override { + return simdb_collectable_.get(); } protected: diff --git a/sparta/sparta/collection/CollectableTreeNode.hpp b/sparta/sparta/collection/CollectableTreeNode.hpp index 3aee7c184c..661884ba73 100644 --- a/sparta/sparta/collection/CollectableTreeNode.hpp +++ b/sparta/sparta/collection/CollectableTreeNode.hpp @@ -15,6 +15,7 @@ namespace simdb { class DatabaseManager; class CollectionMgr; + class CollectionPointBase; } namespace sparta{ @@ -73,6 +74,8 @@ namespace collection */ virtual void configCollectable(simdb::CollectionMgr *) = 0; + virtual uint16_t getElemId() const = 0; + /** * \brief Method that tells this treenode that is now running * collection. @@ -116,6 +119,8 @@ namespace collection //! does not get closed out. virtual void closeRecord(const bool & simulation_ending = false) { (void) simulation_ending; } + virtual simdb::CollectionPointBase* getSimDbCollectable() const = 0; + protected: /** diff --git a/sparta/sparta/collection/IterableCollector.hpp b/sparta/sparta/collection/IterableCollector.hpp index 2510ffa59c..9cafb5a9bd 100644 --- a/sparta/sparta/collection/IterableCollector.hpp +++ b/sparta/sparta/collection/IterableCollector.hpp @@ -47,7 +47,7 @@ namespace collection { * class will output a warning message (only once). */ template -class IterableCollector : public CollectableTreeNode +class IterableCollector : public CollectableTreeNode, public simdb::TickReader { public: typedef typename IterableType::size_type size_type; @@ -196,6 +196,14 @@ class IterableCollector : public CollectableTreeNode setManualCollection(); } + uint64_t getTick() const override { + return getClock()->getScheduler()->getCurrentTick() - 1; + } + + uint16_t getElemId() const override { + return simdb_collectable_->getElemId(); + } + //! \brief Do not perform any automatic collection //! The SchedulingPhase is ignored void setManualCollection() { @@ -258,6 +266,12 @@ class IterableCollector : public CollectableTreeNode getLocation(), getClock()->getName(), expected_capacity_); + + simdb_collectable_->setTickReader(*this); + } + + simdb::CollectionPointBase* getSimDbCollectable() const override { + return simdb_collectable_.get(); } private: @@ -280,6 +294,14 @@ class IterableCollector : public CollectableTreeNode { // Nothing to do here } + + uint16_t getElemId() const override { + return 0; + } + + simdb::CollectionPointBase* getSimDbCollectable() const override { + return nullptr; + } }; //! Virtual method called by CollectableTreeNode when collection diff --git a/sparta/sparta/collection/PipelineCollector.hpp b/sparta/sparta/collection/PipelineCollector.hpp index ce6fd66abb..1eeecd51dc 100644 --- a/sparta/sparta/collection/PipelineCollector.hpp +++ b/sparta/sparta/collection/PipelineCollector.hpp @@ -105,9 +105,8 @@ namespace collection private: void performCollectionAt_(sparta::Scheduler::Tick tick) { - if (LOG_MINIFICATION) std::cout << "\n\n[simdb verbose] Performing collection at tick " << tick << "\n"; - - for(auto & ctn : enabled_ctns_) { + (void)tick; + for(auto ctn : enabled_ctns_) { if(ctn->isCollected()) { //This is happening on a specific clock and a specific phase. //We need to honor the collectable value at this very time, @@ -231,6 +230,24 @@ namespace collection */ void destroy() { + if (!auto_collected_paths_.empty()) { + db_mgr_->safeTransaction([&](){ + // TODO cnyce - 1st class API for UPDATE + std::ostringstream oss; + oss << "UPDATE CollectableTreeNodes SET AutoCollected = 1 WHERE Location IN ("; + for (size_t idx = 0; idx < auto_collected_paths_.size(); ++idx) { + oss << "'" << auto_collected_paths_[idx] << "'"; + if (idx != auto_collected_paths_.size() - 1) { + oss << ","; + } + } + oss << ")"; + + db_mgr_->executeSQL(oss.str()); + return true; + }); + } + db_mgr_->postSim(); if(collection_active_) { @@ -352,7 +369,7 @@ namespace collection auto ccm_pair = clock_ctn_map_.find(ctn->getClock()); sparta_assert(ccm_pair != clock_ctn_map_.end()); ccm_pair->second[static_cast(collection_phase)]->enable(ctn); - addToAutoSweep(ctn); + addToAutoSweep_(ctn, true); } /** @@ -378,11 +395,7 @@ namespace collection void addToAutoSweep(CollectableTreeNode * ctn) { - auto& sweep = sweepers_[ctn->getClock()]; - if (!sweep) { - sweep.reset(new ClockDomainSweeper(db_mgr_->getCollectionMgr(), ctn->getClock())); - } - sweep->enable(ctn); + addToAutoSweep_(ctn, false); } void removeFromAutoSweep(CollectableTreeNode * ctn) @@ -452,6 +465,21 @@ namespace collection sparta::UniqueEvent ev_sweep_; }; + void addToAutoSweep_(CollectableTreeNode * ctn, bool is_auto_collected) + { + auto& sweep = sweepers_[ctn->getClock()]; + if (!sweep) { + sweep.reset(new ClockDomainSweeper(db_mgr_->getCollectionMgr(), ctn->getClock())); + } + sweep->enable(ctn); + + ctn->getSimDbCollectable()->setAutoCollect(is_auto_collected); + + if (is_auto_collected) { + auto_collected_paths_.push_back(ctn->getLocation()); + } + } + //! The SimDB database std::unique_ptr db_mgr_; @@ -463,6 +491,9 @@ namespace collection //! Actively auto-sweeping nodes std::unordered_map> sweepers_; + + //! All element paths for auto-collected collectables. + std::vector auto_collected_paths_; }; }// namespace collection From 5d2c70c986f7e263025b63e90a76c223642b91ba Mon Sep 17 00:00:00 2001 From: Colby Nyce Date: Tue, 4 Mar 2025 13:53:02 -0600 Subject: [PATCH 49/49] Update submodule to colby-nyce/simdb.git --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 7c58c0f42c..6087d7cfd6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "sparta/simdb"] path = sparta/simdb - url = git@github.com:colbynyce-mips/simdb.git + url = git@github.com:colby-nyce/simdb.git