diff --git a/qa_interfaces/IBenchmark.h b/qa_interfaces/IBenchmark.h new file mode 100644 index 00000000..6650353c --- /dev/null +++ b/qa_interfaces/IBenchmark.h @@ -0,0 +1,103 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2020 Metrological + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Module.h" + +// @stubgen:include + +namespace Thunder { +namespace QualityAssurance { + + // @json 2.0.0 + struct EXTERNAL IBenchmark : virtual public Core::IUnknown { + enum { ID = ID_BENCHMARK }; + + ~IBenchmark() override = default; + + struct RoundTripStats { + uint64_t minNs; + uint64_t avgNs; + uint64_t maxNs; + uint64_t stddevNs; + }; + + struct MemoryStats { + uint64_t residentBefore; // RSS in bytes before benchmark + uint64_t residentAfter; // RSS in bytes after benchmark + uint64_t allocatedBefore; // Allocated in bytes before benchmark + uint64_t allocatedAfter; // Allocated in bytes after benchmark + }; + + + enum FailureReason : uint8_t { + LATENCY_THRESHOLD_EXCEEDED, + MEMORY_THRESHOLD_EXCEEDED, + LATENCY_AND_MEMORY_THRESHOLD_EXCEEDED + }; + + struct BenchmarkResult { + string apiName; + uint32_t iterations; + + RoundTripStats roundTrip; + MemoryStats memory; + + bool passed; + Core::OptionalType failureReason; + }; + + typedef RPC::IIteratorType IBenchmarkResultIterator; + + // @brief Run the benchmark + // @param iterations: Denotes the number of iterations the benchmark should run + // @retval ERROR_NONE Benchmark completed and all thresholds passed + // @retval ERROR_GENERAL Benchmark completed but one or more thresholds exceeded + virtual Core::hresult Trigger(const uint32_t iterations) = 0; + + // @brief Collect the results of the most recent benchmark run + // @param report: Iterator over the per-method benchmark results + virtual Core::hresult CollectData(IBenchmarkResultIterator*& report /* @out */) const = 0; + + // @property + // @brief Maximum allowed deviation in avg latency compared to first-run baseline, in millipercent (1000 = 1%, 0 = no latency check) + virtual Core::hresult LatencyThreshold(const uint32_t maxLatencyDeviationPct) = 0; + virtual Core::hresult LatencyThreshold(uint32_t& maxLatencyDeviationPct /* @out */) const = 0; + + // @property + // @brief Maximum allowed RSS growth in bytes per method (0 = no memory check) + virtual Core::hresult MemoryThreshold(const uint64_t maxMemoryGrowthBytes) = 0; + virtual Core::hresult MemoryThreshold(uint64_t& maxMemoryGrowthBytes /* @out */) const = 0; + + // @event + struct EXTERNAL INotification : virtual public Core::IUnknown { + enum { ID = ID_BENCHMARK_NOTIFICATION }; + + ~INotification() override = default; + + virtual void PerformanceCheckCompleted() = 0; + }; + + virtual Core::hresult Register(IBenchmark::INotification* sink) = 0; + virtual Core::hresult Unregister(IBenchmark::INotification* sink) = 0; + }; + +} +} diff --git a/qa_interfaces/IBenchmarkPayload.h b/qa_interfaces/IBenchmarkPayload.h new file mode 100644 index 00000000..7babdfa0 --- /dev/null +++ b/qa_interfaces/IBenchmarkPayload.h @@ -0,0 +1,99 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2020 Metrological + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include "Module.h" + +// @stubgen:include + +namespace Thunder { + namespace QualityAssurance { + + struct EXTERNAL IBenchmarkPayload : virtual public Core::IUnknown + { + enum { ID = ID_BENCHMARK_PAYLOAD }; + + ~IBenchmarkPayload() override = default; + + struct SampleData + { + uint32_t id; + uint32_t value; + string name; + }; + + enum PayloadType : uint8_t + { + PAYLOAD_UNKNOWN, + PAYLOAD_SMALL, + PAYLOAD_MEDIUM, + PAYLOAD_LARGE + }; + + typedef RPC::IIteratorType IPayloadTypeIterator; + + // @brief Retrieve supported payload types via an iterator (exercises COM-RPC iterator serialization) + virtual Core::hresult GetPayloadTypes(IPayloadTypeIterator*& types /* @out */) const = 0; + + virtual Core::hresult SendUint16(const uint16_t value) = 0; + + virtual Core::hresult SendUint32(const uint32_t value) = 0; + + virtual Core::hresult SendUint64(const uint64_t value) = 0; + + virtual Core::hresult SendBool(const bool value) = 0; + + virtual Core::hresult SendFloat(const float value) = 0; + + virtual Core::hresult SendDouble(const double value) = 0; + + virtual Core::hresult SendString(const string& value) = 0; + + virtual Core::hresult SendSampleData(const SampleData& data) = 0; + + virtual Core::hresult SendNoPayload() = 0; + + virtual Core::hresult SendBuffer(const uint16_t bufferSize, const uint8_t buffer[] /* @length:bufferSize @in */) = 0; + + virtual Core::hresult SendUint32Array(const std::vector& data /* @restrict:1..1024 */) = 0; + + virtual Core::hresult SendReceiveUint16(const uint16_t input, uint16_t &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveUint32(const uint32_t input, uint32_t &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveUint64(const uint64_t input, uint64_t &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveBool(const bool input, bool &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveFloat(const float input, float &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveDouble(const double input, double &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveString(const string &input, string &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveSampleData(const SampleData &input, SampleData &output /* @out */) const = 0; + + virtual Core::hresult SendReceiveBuffer(uint16_t &bufferSize /* @inout */, uint8_t buffer[] /* @length:bufferSize @inout */) const = 0; + + virtual Core::hresult SendReceiveUint32Array(const std::vector& input /* @restrict:1..1024 */, std::vector& output /* @out @restrict:1..1024 */) const = 0; + + virtual Core::hresult Add(const uint32_t a, const uint32_t b, uint32_t &result /* @out */) const = 0; + }; +} +} diff --git a/qa_interfaces/QAIds.h b/qa_interfaces/QAIds.h index 08173dc0..d91d89e2 100644 --- a/qa_interfaces/QAIds.h +++ b/qa_interfaces/QAIds.h @@ -74,8 +74,14 @@ namespace QualityAssurance { ID_TESTKEEP = ID_TESTTEXTOPTIONS + 4, ID_TESTKEEP_NOTIFICATION = ID_TESTTEXTOPTIONS + 5, ID_TESTCUSTOM = ID_TESTTEXTOPTIONS + 6, - ID_TESTCUSTOM_NOTIFICATION = ID_TESTTEXTOPTIONS + 7 + ID_TESTCUSTOM_NOTIFICATION = ID_TESTTEXTOPTIONS + 7, + ID_BENCHMARK = RPC::IDS::ID_EXTERNAL_QA_INTERFACE_OFFSET + 0x040, + ID_BENCHMARK_NOTIFICATION = ID_BENCHMARK + 1, + ID_BENCHMARK_RESULT_ITERATOR = ID_BENCHMARK + 2, + + ID_BENCHMARK_PAYLOAD = RPC::IDS::ID_EXTERNAL_QA_INTERFACE_OFFSET + 0x050, + ID_BENCHMARK_PAYLOADTYPE_ITERATOR = ID_BENCHMARK_PAYLOAD + 1, }; } }