Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,16 @@ This project adheres to the [Contributor Covenant Code of Conduct](https://www.c
### Prerequisites

- **Python 3.10+**
- **pip** (usually bundled with Python)
- **Make**
- **Git**

### One-time setup

1. Fork the repository on GitHub, then clone your fork and go into the project directory:

```bash
git clone https://github.com/HH-MWB/timerun.git
git clone https://github.com/YOUR_USERNAME/timerun.git
cd timerun
```

Expand Down Expand Up @@ -112,7 +114,6 @@ timerun/
│ ├── environment.py # Optional: hooks (before/after scenario, etc.)
│ └── steps/ # Step definitions (flat; all .py files loaded)
│ ├── __init__.py
│ ├── utils.py # Shared constants and helpers (no step decorators)
│ ├── common_steps.py # Shared steps used by multiple features
│ └── *_steps.py # Feature-specific step files
├── pyproject.toml # Project metadata and config
Expand All @@ -128,7 +129,7 @@ timerun/
```

- **`timerun.py`** — The only library module; keep it a single file by design.
- **`features/`** — All executable specs; no separate unit test directory. Layout follows [behave](https://behave.readthedocs.io/) convention: step definitions live under `features/steps/` (flat; subdirectories are not searched). Shared logic lives in `features/steps/utils.py`; shared steps (e.g. metadata, wall-time buffer, exception propagation) in `common_steps.py`. Run behave from the project root so `from features.steps.utils import ...` works.
- **`features/`** — All executable specs; no separate unit test directory. Layout follows [behave](https://behave.readthedocs.io/) convention: step definitions live under `features/steps/` (flat; subdirectories are not searched). Shared steps (e.g. metadata, wall-time buffer, exception propagation) live in `common_steps.py`. Run behave from the project root.

## Pull Request Process

Expand Down
2 changes: 1 addition & 1 deletion docs/recipes/analyze-results.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

Two common ways to get a list of measurements:

1. **From a decorated function** — use the timer’s `measurements` deque when you run the same function many times (e.g. benchmarks or repeated calls):
1. **From a decorated function** — use the wrapped callable’s `measurements` deque when you run the same function many times (e.g. benchmarks or repeated calls):

```python
from timerun import Timer
Expand Down
5 changes: 2 additions & 3 deletions features/block_timing.feature
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,11 @@ Feature: Block timing
Then the measurement's wall time duration is within the configured buffer of 10,000,000 nanoseconds
And the measurement's CPU time duration is within the configured buffer of 0 nanoseconds

Scenario: CPU-bound block with `with` yields wall and CPU time close together
Scenario: CPU-bound block with `with` yields meaningful wall and CPU time
Given a CPU-bound operation that runs for around 10,000,000 nanoseconds
When I measure the operation using `with`
Then the measurement's wall time duration is within the configured buffer of 10,000,000 nanoseconds
Then the measurement's wall time duration is at least 10,000,000 nanoseconds
And the measurement's CPU time duration is within the configured buffer of 10,000,000 nanoseconds
And the measurement's CPU time is close to wall time

# --- One Timer, multiple blocks or threads ---

Expand Down
85 changes: 30 additions & 55 deletions features/steps/block_timing_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,13 @@
from __future__ import annotations

import asyncio
import time
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING

from behave import given, then, when

import timerun
from features.steps.utils import (
BUFFER_NS,
assert_metadata_key_equals,
assert_wall_time_within_buffer,
sleep_wall_at_least,
spin_wall_at_least,
)

if TYPE_CHECKING:
from behave.runner import Context
Expand Down Expand Up @@ -72,11 +66,13 @@ def step_given_callback_records_invocations(
callback_kind: str,
) -> None:
"""Store list and callback that records the measurement passed to it."""
# List and callback that appends each measurement.
invocations: list[timerun.Measurement] = []

def record_invocation(m: timerun.Measurement) -> None:
invocations.append(m)

# Store on context for When/Then.
setattr(context, f"{callback_kind}_invocations", invocations)
setattr(context, f"{callback_kind}_callback", record_invocation)

Expand All @@ -88,10 +84,16 @@ def record_invocation(m: timerun.Measurement) -> None:
def step_measure_operation_using_with(context: Context) -> None:
"""Measure with Timer(); sleep or spin per operation_kind."""
with timerun.Timer() as context.measurement:
if getattr(context, "operation_kind", "blocking") == "CPU-bound":
spin_wall_at_least(context.operation_duration_ns)
if context.operation_kind == "CPU-bound":
# Spin until duration elapsed.
start = time.process_time_ns()
while (
time.process_time_ns() - start < context.operation_duration_ns
):
pass
else:
sleep_wall_at_least(context.operation_duration_ns)
# Sleep for duration.
time.sleep(context.operation_duration_ns / 1e9)


@when("I measure the async operation using `async with`")
Expand Down Expand Up @@ -121,7 +123,7 @@ def step_measure_blocks_from_threads(
# Worker: enter timer, sleep, return measurement.
def run() -> timerun.Measurement:
with cm as m:
sleep_wall_at_least(context.thread_sleep_ns)
time.sleep(context.thread_sleep_ns / 1e9)
return m

# Run thread_count workers and collect measurements.
Expand All @@ -135,11 +137,13 @@ def step_measure_two_sequential_blocks(context: Context) -> None:
"""Measure two sequential blocks."""
cm = timerun.Timer()

# First block.
with cm as context.first_measurement:
sleep_wall_at_least(context.first_block_ns)
time.sleep(context.first_block_ns / 1e9)

# Second block.
with cm as context.second_measurement:
sleep_wall_at_least(context.second_block_ns)
time.sleep(context.second_block_ns / 1e9)


@when("I measure nested blocks with the same Timer instance")
Expand All @@ -148,10 +152,10 @@ def step_measure_nested_blocks(context: Context) -> None:
cm = timerun.Timer()

with cm as context.outer_measurement:
sleep_wall_at_least(context.outer_block_ns)
time.sleep(context.outer_block_ns / 1e9)

with cm as context.inner_measurement:
sleep_wall_at_least(context.inner_block_ns)
time.sleep(context.inner_block_ns / 1e9)


@when("I measure a code block with that metadata")
Expand Down Expand Up @@ -216,36 +220,22 @@ def step_exit_without_enter(context: Context) -> None:
# --- Then ---


@then("the measurement's CPU time is close to wall time")
def step_cpu_close_to_wall(context: Context) -> None:
"""Assert CPU close to wall time."""
# Required context validation.
assert context.measurement.wall_time is not None
assert context.measurement.cpu_time is not None

# Duration in [wall - BUFFER_NS, wall].
wall = context.measurement.wall_time.duration
cpu = context.measurement.cpu_time.duration
min_cpu = max(0, wall - BUFFER_NS)
assert min_cpu <= cpu <= wall, (
f"CPU {cpu} not in [wall-BUFFER_NS, wall] = [{min_cpu}, {wall}]"
)


@then(
"the {which} measurement's wall time duration is within the configured "
"buffer of {expected_ns:n} nanoseconds",
"the measurement's wall time duration is at least {expected_ns:n} "
"nanoseconds",
)
def step_which_measurement_wall_within_buffer(
def step_measurement_wall_time_at_least(
context: Context,
which: str,
expected_ns: int,
) -> None:
"""Assert which measurement wall time in buffer."""
assert_wall_time_within_buffer(
getattr(context, f"{which}_measurement"),
expected_ns,
BUFFER_NS,
"""Assert wall time >= expected; used for CPU-bound scenario."""
# Require wall_time and get duration.
assert context.measurement.wall_time is not None
duration = context.measurement.wall_time.duration

# Duration must be at least expected.
assert duration >= expected_ns, (
f"wall time {duration} < expected {expected_ns}"
)


Expand All @@ -265,21 +255,6 @@ def step_outer_wall_at_least_inner(context: Context) -> None:
assert outer_d >= inner_d, f"outer {outer_d} < inner {inner_d}"


@then('the {which} measurement\'s metadata key "{key}" is "{value}"')
def step_measurement_metadata_key(
context: Context,
which: str,
key: str,
value: str,
) -> None:
"""Assert which measurement metadata[key] is value."""
assert_metadata_key_equals(
getattr(context, f"{which}_measurement"),
key,
value,
)


@then('the second measurement\'s metadata does not contain key "{key}"')
def step_second_measurement_metadata_no_key(
context: Context,
Expand Down
69 changes: 53 additions & 16 deletions features/steps/common_steps.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Shared step definitions used by multiple features.
"""Step definitions shared by more than one feature.

Steps here use consistent wording and semantics across features
(exception assertions, error messages, measurement metadata, wall time buffer).
Rule: a step belongs here only if its Gherkin text is used in multiple feature
files (e.g. block_timing.feature and function_timing.feature). Steps used by
a single feature live in that feature's step file (e.g. block_timing_steps.py).
"""

from __future__ import annotations
Expand All @@ -11,16 +12,21 @@

from behave import given, then

from features.steps.utils import (
BUFFER_NS,
CPU_LOWER_SLACK_NS,
assert_metadata_key_equals,
assert_wall_time_within_buffer,
)

if TYPE_CHECKING:
from behave.runner import Context

# Buffer: expected_ns <= duration <= expected_ns + BUFFER_NS (10 ms).
BUFFER_NS = 10_000_000

# --- Helpers ---


def _measurement(context: Context, which: str | None) -> object:
"""Resolve measurement from context."""
if which is None:
return context.measurement
return getattr(context, f"{which}_measurement")


# --- Given ---

Expand Down Expand Up @@ -49,24 +55,49 @@ def step_exception_raised(context: Context, exception_type: str) -> None:
@then('the error message is "{message}"')
def step_error_message_is(context: Context, message: str) -> None:
"""Assert exception message."""
# Required: an exception was stored by the When step.
assert hasattr(context, "exception"), "Expected an exception to be raised"

# Message must match.
assert str(context.exception) == message


@then("an exception was propagated to the caller")
def step_exception_propagated(context: Context) -> None:
"""Assert ValueError was caught."""
# Required: an exception was stored.
assert hasattr(context, "exception")

# Must be ValueError.
assert isinstance(context.exception, ValueError)


@then(
"the measurement's wall time duration is within the configured buffer of "
"{expected_ns:n} nanoseconds",
)
def step_wall_time_within_buffer(context: Context, expected_ns: int) -> None:
"""Assert wall time in buffer."""
assert_wall_time_within_buffer(context.measurement, expected_ns, BUFFER_NS)
@then(
"the {which} measurement's wall time duration is within the configured "
"buffer of {expected_ns:n} nanoseconds",
)
def step_wall_time_within_buffer(
context: Context,
expected_ns: int,
which: str | None = None,
) -> None:
"""Assert wall time in buffer (default or first/second/outer/inner)."""
measurement = _measurement(context, which)

# Resolve duration and buffer bound.
assert measurement.wall_time is not None
duration = measurement.wall_time.duration
max_ns = expected_ns + BUFFER_NS

# Duration must lie in [expected_ns, expected_ns + buffer].
assert expected_ns <= duration <= max_ns, (
f"wall time {duration} not in [{expected_ns}, {max_ns}] "
f"(buffer={BUFFER_NS})"
)


@then(
Expand All @@ -75,20 +106,26 @@ def step_wall_time_within_buffer(context: Context, expected_ns: int) -> None:
)
def step_cpu_time_within_buffer(context: Context, expected_ns: int) -> None:
"""Assert CPU time in buffer."""
# Resolve duration and buffer bounds.
assert context.measurement.cpu_time is not None

# Duration must lie in [min_ns, max_ns].
duration = context.measurement.cpu_time.duration
min_ns = max(0, expected_ns - CPU_LOWER_SLACK_NS)
min_ns = max(0, expected_ns - 1_000_000)
max_ns = expected_ns + BUFFER_NS
assert min_ns <= duration <= max_ns, (
f"CPU time {duration} not in [{min_ns}, {max_ns}] (buffer={BUFFER_NS})"
)


@then('the measurement\'s metadata key "{key}" is "{value}"')
@then('the {which} measurement\'s metadata key "{key}" is "{value}"')
def step_measurement_metadata_key_is(
context: Context,
key: str,
value: str,
which: str | None = None,
) -> None:
"""Assert metadata[key] is value."""
assert_metadata_key_equals(context.measurement, key, value)
"""Assert metadata[key] is value (default or first/second/outer/inner)."""
measurement = _measurement(context, which)
assert measurement.metadata[key] == value
Loading