Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions docs/profiling.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Continuous profiling

The SDK can periodically capture Go runtime profiles and ship them to Traceway,
where they power flame graphs and allocation views alongside your traces and
metrics.

Profiling is **opt-in** and runs in a single background goroutine. When enabled,
each cycle captures:

- a **CPU profile** (`runtime/pprof` `StartCPUProfile` / `StopCPUProfile`) over a
~30s window, and
- a **heap profile** (`pprof.Lookup("heap")`),

then POSTs each one as a separate request to `<server>/api/profiles/ingest`.

## Enabling

```go
err := traceway.Init(
connectionString,
traceway.WithProfiling("checkout-api"),
)
```

`WithProfiling(serviceName)` enables profiling. `serviceName` is the application
name the profiles are grouped under in the dashboard. If you pass an empty
string, it defaults to the running binary's name (`filepath.Base(os.Args[0])`).

### Interval

```go
traceway.WithProfiling("checkout-api"),
traceway.WithProfilingInterval(2 * time.Minute),
```

`WithProfilingInterval` controls how often a cycle runs. The default is **60s**.
The CPU window is **30s**, automatically capped to `interval / 2` so a short
interval can never be overrun by an in-flight CPU capture. The first profile is
captured one interval after `Init` (not at startup).

## What gets sent

Each profile is uploaded as its own request, reusing the token and server from
the connection string:

| Part | Value |
| --- | --- |
| Method / path | `POST <scheme>://<host>/api/profiles/ingest` |
| `Authorization` | `Bearer <project_token>` |
| Query `service` | the app name from `WithProfiling` |
| Query `serverName` | the host (same value as `WithServerName`, defaults to the hostname) |
| Query `appVersion` | the value from `WithVersion` |
| `Content-Type` | `application/octet-stream` |
| Body | the raw pprof bytes |

The ingest URL is derived from the connection string's report URL by keeping its
scheme and host and replacing the path with `/api/profiles/ingest`.

pprof output is already gzip-compressed, so the body is sent **as-is** — no extra
`Content-Encoding: gzip`.

## Notes

- A heap snapshot is taken **without** forcing a `runtime.GC()` first, to avoid
injecting GC latency spikes into the host application.
- Profiling never affects your application's behavior on failure. A capture or
upload error skips that one profile for the cycle; the other profile is still
attempted. Errors are silent unless `WithDebug(true)` is set.
- If another part of the process already holds the CPU profiler,
`StartCPUProfile` fails for that cycle; the heap profile is unaffected.

## Middleware

`WithProfiling` lives on the core `traceway` package. Apps that initialize via
`traceway.Init` directly (for example, worker/task processes using `MeasureTask`)
can enable it today. A pass-through option for the HTTP middleware wrappers
(`tracewayhttp`, `tracewaygin`, `tracewayfiber`, `tracewaychi`,
`tracewayfasthttp`) is a follow-up.
191 changes: 191 additions & 0 deletions profiling.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package traceway

import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime/pprof"
"time"
)

const (
profileIngestPath = "/api/profiles/ingest"
defaultProfilingInterval = 60 * time.Second
defaultCPUProfileWindow = 30 * time.Second
profileUploadTimeout = 30 * time.Second
)

func WithProfiling(serviceName string) func(*TracewayOptions) {
return func(s *TracewayOptions) {
s.profilingEnabled = true
if serviceName == "" {
serviceName = filepath.Base(os.Args[0])
}
s.profilingService = serviceName
}
}

func WithProfilingInterval(d time.Duration) func(*TracewayOptions) {
return func(s *TracewayOptions) {
s.profilingInterval = d
}
}

func profileIngestURL(reportURL string) (string, error) {
u, err := url.Parse(reportURL)
if err != nil {
return "", err
}
if u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("traceway: cannot derive profile ingest URL from %q", reportURL)
}
return u.Scheme + "://" + u.Host + profileIngestPath, nil
}

func captureCPUProfile(d time.Duration) ([]byte, error) {
var buf bytes.Buffer
if err := pprof.StartCPUProfile(&buf); err != nil {
return nil, err
}
time.Sleep(d)
pprof.StopCPUProfile()
return buf.Bytes(), nil
}

func captureHeapProfile() ([]byte, error) {
prof := pprof.Lookup("heap")
if prof == nil {
return nil, fmt.Errorf("traceway: heap profile unavailable")
}
var buf bytes.Buffer
if err := prof.WriteTo(&buf, 0); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

type profiler struct {
url string
token string
service string
serverName string
appVersion string
interval time.Duration
cpuWindow time.Duration
client *http.Client
debug bool
}

func (p *profiler) uploadProfile(body []byte) error {
req, err := http.NewRequest(http.MethodPost, p.url, bytes.NewReader(body))
if err != nil {
return err
}

q := req.URL.Query()
q.Set("service", p.service)
q.Set("serverName", p.serverName)
q.Set("appVersion", p.appVersion)
req.URL.RawQuery = q.Encode()

req.Header.Set("Authorization", "Bearer "+p.token)
req.Header.Set("Content-Type", "application/octet-stream")

client := p.client
if client == nil {
client = http.DefaultClient
}

resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("traceway: profile ingest returned status %d", resp.StatusCode)
}
return nil
}

func (p *profiler) collectOnce() {
window := p.cpuWindow
if window <= 0 {
window = defaultCPUProfileWindow
}

if cpu, err := captureCPUProfile(window); err != nil {
p.logError("cpu profile capture failed", err)
} else if err := p.uploadProfile(cpu); err != nil {
p.logError("cpu profile upload failed", err)
}

if heap, err := captureHeapProfile(); err != nil {
p.logError("heap profile capture failed", err)
} else if err := p.uploadProfile(heap); err != nil {
p.logError("heap profile upload failed", err)
}
}

func (p *profiler) safeCollectOnce() {
defer func() {
if r := recover(); r != nil {
p.logError("profiling cycle panicked", fmt.Errorf("%v", r))
}
}()
p.collectOnce()
}

func (p *profiler) run() {
ticker := time.NewTicker(p.interval)
defer ticker.Stop()

for range ticker.C {
p.safeCollectOnce()
}
}

func (p *profiler) logError(msg string, err error) {
if p.debug {
log.Printf("Traceway: %s: %v", msg, err)
}
}

func startProfiler(reportURL, token string, opts *TracewayOptions) error {
ingestURL, err := profileIngestURL(reportURL)
if err != nil {
return err
}

interval := opts.profilingInterval
if interval <= 0 {
interval = defaultProfilingInterval
}

cpuWindow := defaultCPUProfileWindow
if half := interval / 2; half < cpuWindow {
cpuWindow = half
}

p := &profiler{
url: ingestURL,
token: token,
service: opts.profilingService,
serverName: opts.serverName,
appVersion: opts.version,
interval: interval,
cpuWindow: cpuWindow,
client: &http.Client{Timeout: profileUploadTimeout},
debug: opts.debug,
}

go p.run()

return nil
}
Loading