Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 66 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,55 @@ LAND ?= true
WATCH ?= true
QUEUE ?= demo-queue
STRATEGY ?= SQUASH_REBASE
GATEWAY_ADDR ?= localhost:8081
# Where the client looks for the gateway. Left empty, every target below finds
# the running stack's published port for itself — Compose picks a fresh one on
# every start, and a number copied out of a previous run's output is the most
# common reason a demo command cannot connect. Set it to reach a gateway this
# Makefile did not start.
GATEWAY_ADDR ?=

# Resolves $(GATEWAY_ADDR), or the local stack's port when it is unset, into
# $$addr for the recipe that includes it. Not a $(shell ...) assignment: that
# would run at parse time, shelling out to Docker on every `make help`.
define resolve_gateway_addr
addr="$(GATEWAY_ADDR)"; \
port=$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | head -1 | sed 's/.*://'); \
if [ -z "$$addr" ]; then \
if [ -z "$$port" ]; then \
echo "No gateway found: '$(SUBMITQUEUE_LOCAL_PROJECT)' is not running." >&2; \
echo "Start it with 'make local-submitqueue-start', or name one with GATEWAY_ADDR=host:port." >&2; \
exit 2; \
fi; \
addr="localhost:$$port"; \
elif [ "$(origin GATEWAY_ADDR)" = "environment" ] && [ -n "$$port" ] && [ "$$addr" != "localhost:$$port" ]; then \
echo "Note: GATEWAY_ADDR=$$addr is exported in your shell, so that is what will be used." >&2; \
echo " The stack running here is on localhost:$$port — 'unset GATEWAY_ADDR' to use it." >&2; \
fi
endef

# Resolves which provider to create changes for into $$provider, preferring the
# one the running stack was started with.
#
# The two have to agree. A change minted for one provider is meaningless to a
# stack wired to another: fake changes point at no repository, so a stack
# running the git merger rejects every one of them as a commit it cannot find,
# and fifty requests fail identically for a reason that is nowhere in the error.
# The stack knows which provider it has — it is mounted at /etc/submitqueue —
# so a run that was not told otherwise asks it rather than guessing.
define resolve_provider
provider="$(PROVIDER)"; \
mounted=$$(docker inspect $(SUBMITQUEUE_LOCAL_PROJECT)-orchestrator-service-1 \
--format '{{range .Mounts}}{{if eq .Destination "/etc/submitqueue"}}{{.Source}}{{end}}{{end}}' 2>/dev/null); \
if [ -n "$$mounted" ]; then \
running=$$(basename "$$mounted"); \
if [ "$(origin PROVIDER)" = "file" ]; then \
provider="$$running"; \
elif [ "$$provider" != "$$running" ]; then \
echo "Note: creating $$provider changes, but the running stack is '$$running'." >&2; \
echo " They have to match — a $$provider change is not something a '$$running' stack can land." >&2; \
fi; \
fi
endef

# Fails if git working tree is dirty. Usage: $(call assert_clean,fix command)
define assert_clean
Expand Down Expand Up @@ -197,16 +245,17 @@ clean-proto: ## Clean generated proto files
@echo "Proto clean complete!"

demo-requests: ## Create N changes, enqueue each as it is created, and watch (PROVIDER=fake|git|github COUNT=3 FOLDERS=0 FILES=3 CONCURRENCY=5)
@$(BAZEL) run //service/submitqueue/demo/requests -- \
-provider $(PROVIDER) \
@set -e; $(resolve_gateway_addr); $(resolve_provider); \
$(BAZEL) run //service/submitqueue/demo/requests -- \
-provider $$provider \
-repo $(DEMO_REPO) \
-sandbox-dir $(SQ_GIT_SANDBOX_DIR) \
-count $(COUNT) \
-folders $(FOLDERS) \
-files $(FILES) \
-concurrency $(CONCURRENCY) \
-stacked=$(STACKED) \
-addr $(GATEWAY_ADDR) \
-addr $$addr \
-queue $(QUEUE) \
-strategy $(STRATEGY) \
-land=$(LAND) -watch=$(WATCH)
Expand Down Expand Up @@ -262,25 +311,29 @@ land: ## Land a change or a stack (PR=<url>, PRS="<url> <url>", or URI=<change-u
echo " opts: QUEUE=$(QUEUE) STRATEGY=$(STRATEGY) GATEWAY_ADDR=$(GATEWAY_ADDR)"; \
exit 2; \
fi
@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $(GATEWAY_ADDR) land \
@set -e; $(resolve_gateway_addr); \
$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $$addr land \
-queue $(QUEUE) \
-strategy $(STRATEGY) \
$(if $(PR),-pr $(PR)) $(foreach p,$(PRS),-pr $(p)) \
$(if $(URI),-uri $(URI)) $(foreach u,$(URIS),-uri $(u))

land-status: ## Read a landed request's status (SQID=... [QUEUE=demo-queue])
@if [ -z "$(SQID)" ]; then echo "Usage: make land-status SQID=demo-queue/1 [QUEUE=demo-queue]"; exit 2; fi
@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $(GATEWAY_ADDR) status -queue $(QUEUE) -sqid $(SQID)
@set -e; $(resolve_gateway_addr); \
$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $$addr status -queue $(QUEUE) -sqid $(SQID)

land-list: ## Show a queue's recent requests as a table (QUEUE=demo-queue SINCE=1h LIMIT=50)
@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $(GATEWAY_ADDR) list -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)
@set -e; $(resolve_gateway_addr); \
$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $$addr list -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)

land-watch: ## Follow a queue's requests until they settle (QUEUE=demo-queue SINCE=15m LIMIT=50)
@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $(GATEWAY_ADDR) watch -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)
@set -e; $(resolve_gateway_addr); \
$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
-addr $$addr watch -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)

license-fix: ## Add missing license headers to source files
@$(BAZEL) run //tool/linter/licenseheader -- --fix
Expand Down Expand Up @@ -470,7 +523,7 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|
fi
@echo ""
@echo "Generate traffic with:"
@echo " make demo-requests GATEWAY_ADDR=localhost:<gateway port>"
@echo " make demo-requests"

local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps data and PROVIDER=git's sandbox)
@echo "Stopping SubmitQueue services..."
Expand Down
9 changes: 3 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,14 @@ Cross-domain Go code (errors, metrics, consumer framework, HTTP helpers, shared

## Quick Start

Put traffic through the queue and watch it land. Requires Docker and Docker Compose, and nothing else — no repository, no account, no token. See [Development Setup](doc/howto/DEVELOPMENT.md) for full prerequisites.
Put traffic through the queue and watch it land. Requires Docker and Docker Compose, and nothing else — no repository, no account, no token.

```bash
# Start the full stack (Gateway + Orchestrator + Runway + MySQL)
make local-submitqueue-start

# Compose publishes a random host port; the line above prints it, as does this
make local-submitqueue-ps
export GATEWAY_ADDR=localhost:<gateway port>

# Create changes, enqueue each as it is created, and watch them settle
# Create changes, enqueue each as it is created, and watch them settle.
# It finds the running stack's port and provider itself — nothing to copy.
make demo-requests

# Stop services
Expand Down
10 changes: 3 additions & 7 deletions doc/howto/DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,11 @@ docker ps
# 2. Start the full stack
make local-submitqueue-start

# 3. Read the gateway's port (Compose publishes a random one)
make local-submitqueue-ps
export GATEWAY_ADDR=localhost:<gateway port>

# 4. Create changes, enqueue them, and watch them land
# 3. Create changes, enqueue them, and watch them land
make demo-requests

# 5. Stop services
make local-stop
# 4. Stop services
make local-submitqueue-stop
```

[QUICKSTART.md](QUICKSTART.md) walks through the same run in detail, and on to `PROVIDER=git`, which lands real commits into a repository on disk — still with no credential.
Expand Down
32 changes: 18 additions & 14 deletions doc/howto/QUICKSTART.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,7 @@ Compose publishes each service on a **random** host port so several stacks can r
Gateway gRPC port: 58537
```

Export it, because every command below needs it:

```bash
export GATEWAY_ADDR=localhost:58537
```

Leaving it unset does not fall back to anything useful — the client's default is `localhost:8081`, the `go run` port rather than the compose one.
You do not have to note it down. Every command below finds the running stack's port for itself, which matters because Compose picks a fresh one on every start — a number copied from an earlier run is the most common reason a demo command cannot connect. Set `GATEWAY_ADDR=host:port` only to reach a gateway this Makefile did not start.

## Put traffic through it

Expand Down Expand Up @@ -118,6 +112,17 @@ Eight builds means the batch was speculating down eight paths at once, and `wait

`land-watch` fixes its set when it starts and exits non-zero if any request in that set finishes anywhere other than `landed`, which makes it usable from a script. A request accepted after the watch begins is not picked up: a watch that grew as the queue did would never finish.

Watching more requests than the window holds takes over the screen while it runs, the way `top` does, so the table can be scrolled rather than trimmed:

| Key | |
|---|---|
| `↑` `↓` or `k` `j` | one row |
| `PgUp` `PgDn` or `Space` | one screen |
| `g` `G` | first row, last row |
| `q` | stop watching |

The view follows the end of the table by default, so new rows and new stages appear without touching it. Scrolling up holds your place; scrolling back to the bottom starts following again. The screen you had is restored on exit and the finished table is printed into it whole, so nothing is lost with the view — and when output is redirected, none of this happens at all and the run stays a plain log.

A listing of a busy queue is mostly `speculating` rows, since that is where a request spends most of its active life — waiting on the build its batch was admitted for.

Under the hood these are `client list` and `client watch`, which take a queue and reach any gateway:
Expand Down Expand Up @@ -178,13 +183,14 @@ Gateway gRPC port: 55295
Merge target: /tmp/sq-sandbox/sandbox.git
```

Then the same command as before, with the same `PROVIDER`:
Then the same command as before, unchanged:

```bash
export GATEWAY_ADDR=localhost:55295
PROVIDER=git make demo-requests
make demo-requests
```

`demo-requests` creates changes for whichever provider the running stack was started with, so there is nothing to repeat and nothing to keep in sync. The two must agree — a fake change points at no repository, so a stack running the git merger rejects every one of them as a commit it cannot find — and rather than leaving that to memory, a run with no `PROVIDER` of its own asks the stack which one it has. Passing one that disagrees still works, and says so before it starts.

Now `demo-requests` pushes real branches with real commits, and landing them is a real cherry-pick and push. Look at the repository itself:

```bash
Expand All @@ -200,13 +206,11 @@ b5d86d6 seed the sandbox

The commits are there, and they are not the ones that were pushed: `SQUASH_REBASE` replays each change onto the target rather than merging it, which is why the queue can keep the trunk linear.

**`PROVIDER` has to match on both commands.** It selects what the stack merges with *and* what `demo-requests` creates; pointing fake changes at a stack wired to git means asking the merger to fetch a ref that was never pushed.

One property worth seeing, because it is the thing a submit queue exists for. A stack lands as a single push, so no reader ever observes it half-applied:

```bash
git -C /tmp/sq-sandbox/sandbox.git reflog show refs/heads/main | wc -l
PROVIDER=git make demo-requests COUNT=3 STACKED=true
make demo-requests COUNT=3 STACKED=true
git -C /tmp/sq-sandbox/sandbox.git reflog show refs/heads/main | wc -l
```

Expand Down Expand Up @@ -257,7 +261,7 @@ PROVIDER=github make local-submitqueue-start

The token is required rather than defaulted: a stack that silently falls back to the fake integrations reports changes as landed without having gone near the provider, which is a much worse way to find out.

From here everything is as before — `PROVIDER=github make demo-requests` opens real pull requests, enqueues them and watches them land.
From here everything is as before — `make demo-requests` opens real pull requests, enqueues them and watches them land, having picked up from the running stack that this one is GitHub.

### Land a pull request

Expand Down
10 changes: 10 additions & 0 deletions service/submitqueue/demo/requests/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,11 +211,21 @@ func run(ctx context.Context, cfg config) error {
return nil
}

// A large run has more changes than a window has lines, so the wait happens
// in a full-screen view the reader can scroll. Restored before Conclude, so
// the final table lands in the scrollback and not on a screen that is about
// to be handed back.
stop, quit := t.Interact(ctx)
defer stop()

select {
case <-ctx.Done():
stop()
return ctx.Err()
case <-quit:
case <-t.Settled():
}
stop()
return t.Conclude()
}

Expand Down
10 changes: 10 additions & 0 deletions service/submitqueue/gateway/client/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,13 +282,23 @@ func runWatch(ctx context.Context, sq *client.Client, args []string) error {
t.Seal()
t.Note("watching %d request(s) in %s", len(rows), *queue)

// A watch of a busy queue holds more requests than a window does, so it runs
// as a full-screen view the reader can scroll. Restored before Conclude, so
// the final table lands in the scrollback rather than disappearing with the
// screen it was drawn on.
stop, quit := t.Interact(ctx)
defer stop()

go t.Poll(ctx, sq.Gateway(), *queue)

select {
case <-ctx.Done():
stop()
return ctx.Err()
case <-quit:
case <-t.Settled():
}
stop()
return t.Conclude()
}

Expand Down
2 changes: 2 additions & 0 deletions submitqueue/client/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go_library(
"conn.go",
"land.go",
"query.go",
"tui.go",
"view.go",
"watch.go",
],
Expand All @@ -28,6 +29,7 @@ go_test(
srcs = [
"conn_test.go",
"query_test.go",
"tui_test.go",
"view_test.go",
],
embed = [":go_default_library"],
Expand Down
Loading
Loading