[WIP] Testology!

.github/workflows/nix.yml

@@ -35,9 +35,16 @@ jobs:
 
       - uses: cachix/install-nix-action@v27
         with:
+          # Authenticate Nix's github: flake fetches. Without a token,
+          # resolving the unpinned `nixos-unstable` branch to a commit
+          # hits api.github.com unauthenticated, and from shared Actions
+          # runner IPs that gets throttled (HTTP 429), failing the build.
+          # GITHUB_TOKEN raises the rate limit enough to avoid that.
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
           extra_nix_config: |
             experimental-features = nix-command flakes
             accept-flake-config = true
+            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
 
       # Free shared Nix cache; speeds up consecutive runs from cold
       # to warm by populating /nix/store from previous runs of this

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
       - 'v*'
 
 env:
-  GO_VERSION: '1.26.1'
+  GO_VERSION: '1.26.4'
   MUSL_RELEASE: 'v1.3.29'
   MODULE: github.com/skycoin/skywire
 

.github/workflows/test-ipv6.yml

@@ -1,6 +1,13 @@
-# Phase 5 of the #1525 IPv6 plan: a CI lane that exercises the
-# dual-stack dmsg-server + Happy Eyeballs dialer path end-to-end on
-# the docker bridge configured for both IPv4 and IPv6.
+# Phase 5 of the #1525 IPv6 plan: a CI lane that BUILDS and RUNS a
+# dual-stack dmsg deployment on a docker bridge configured for both
+# IPv4 and IPv6, then asserts, at the wire level, that dmsg works over
+# IPv6 — the crown jewel being a full dmsg Noise handshake completed
+# straight to the server's IPv6 endpoint.
+#
+# This replaces the earlier compose-syntax-only check: the images are
+# built from source via a self-contained Dockerfile (no docker_build.sh
+# / image_tag / base_image prerequisite), the stack is brought up, and
+# docker/dmsg/scripts/assert-ipv6-e2e.sh runs the v6 assertions.
 #
 # Runs in addition to the existing Test lane in test.yml; the v4-only
 # path there continues to validate the backward-compat case (visor
@@ -9,17 +16,21 @@
 #
 # Trigger: PRs that touch the IPv6 surface, or anything under
 # pkg/dmsg / pkg/address-resolver / pkg/transport/network / the
-# v6-aware compose + workflow files themselves. Avoids waking this
-# lane for changes that can't affect v6 (docs, CI for other lanes).
+# v6-aware compose + config + workflow files themselves. Avoids waking
+# this lane for changes that can't affect v6 (docs, CI for other lanes).
 
 on:
   pull_request:
     paths:
       - 'pkg/dmsg/**'
       - 'pkg/address-resolver/**'
       - 'pkg/transport/network/**'
-      - 'docker/dmsg/docker-compose.e2e-v6.yml'
+      - 'pkg/services/dmsgsrv/**'
+      - 'pkg/services/dmsgdisc/**'
+      - 'cmd/dmsg/**'
+      - 'docker/dmsg/**'
       - 'docker/config/dmsg-server-v6.json'
+      - 'docker/config/services-dmsg-v6.json'
       - '.github/workflows/test-ipv6.yml'
   workflow_dispatch:
 
@@ -28,23 +39,16 @@ name: Test IPv6 dual-stack
 jobs:
   ipv6-e2e:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           fetch-tags: true
 
-      - uses: actions/setup-go@v6
-        with:
-          go-version: '1.26.1'
-          cache: true
-          cache-dependency-path: go.sum
-
       # Docker has IPv6 support compiled in but daemon-level IPv6 is
-      # off by default on ubuntu-latest runners. Enable it before
-      # `docker compose config` parses + validates the v6 bridge spec
-      # (and before the full e2e follow-up tries to actually allocate
-      # v6 addresses on the bridge).
+      # off by default on ubuntu-latest runners. Enable it before the
+      # compose stack tries to allocate v6 addresses on the bridge.
       - name: Enable Docker IPv6
         run: |
           set -eux
@@ -58,42 +62,44 @@ jobs:
           sudo systemctl restart docker
           docker info | grep -i ipv6 || true
 
-      # First-stage validation: parse the compose file, resolve
-      # dockerfile + config-file paths, confirm the v6 subnet syntax
-      # is well-formed. Cheap, fast, and catches the most common
-      # regression (broken context-relative dockerfile paths) without
-      # the full build cost. The actual build + run + wire-level
-      # assertion follows in a separate step that's allowed to
-      # short-circuit until the docker_build.sh integration lands
-      # (see "deferred to follow-up" below).
-      - name: Validate dual-stack compose syntax
+      # Build the self-contained image once (skywire + dmsgprobe from
+      # source). Both compose services reference this tag, so the
+      # subsequent `up` needs no --build and won't rebuild.
+      - name: Build dmsg dual-stack image
+        run: |
+          set -eux
+          docker build \
+            -f docker/dmsg/images/dmsg-e2e/Dockerfile \
+            -t skywire-dmsg-e2e:local .
+
+      - name: Validate dual-stack compose config
         working-directory: docker/dmsg
         run: |
           set -eux
           docker compose -f docker-compose.e2e-v6.yml config > /tmp/compose-rendered.yml
           # Sanity: rendered output mentions both address families.
           grep -q '172.21.0.4' /tmp/compose-rendered.yml
           grep -q 'fd00:dead:beef::4' /tmp/compose-rendered.yml
-          # Both dockerfile context paths must resolve to real files
-          # on disk (compose resolves them lazily at build-time, so
-          # the parser doesn't catch this for us).
-          test -f ../images/dmsg-discovery/Dockerfile
-          test -f ../images/dmsg-server/Dockerfile
-          test -f ./images/dmsg-client/Dockerfile
-          # The v6 server config must exist where the volume mount
-          # expects it.
-          test -f ../config/dmsg-server-v6.json
+          # The v6 server config must exist and set the v6 endpoint.
           grep -q '"public_address_v6"' ../config/dmsg-server-v6.json
 
-      # Full e2e build + run is deferred — the existing dmsg-server +
-      # dmsg-discovery images are built via docker/docker_build.sh
-      # with multi-stage build args (image_tag + base_image) that a
-      # plain `docker compose up --build` can't supply. The lane
-      # plumbing to invoke docker_build.sh as a prerequisite + then
-      # `compose up` against the locally-tagged images is a focused
-      # follow-up. Tracking via the #1525 thread.
-      - name: Note follow-up scope
+      - name: Bring up the dual-stack dmsg stack
+        run: |
+          set -eux
+          docker compose -f docker/dmsg/docker-compose.e2e-v6.yml up -d --wait
+
+      - name: Assert dmsg works over IPv6 (wire-level)
+        run: |
+          set -eux
+          bash docker/dmsg/scripts/assert-ipv6-e2e.sh
+
+      - name: Dump logs on failure
+        if: failure()
+        run: |
+          docker compose -f docker/dmsg/docker-compose.e2e-v6.yml ps || true
+          docker compose -f docker/dmsg/docker-compose.e2e-v6.yml logs --tail=200 || true
+
+      - name: Tear down
+        if: always()
         run: |
-          echo "Compose syntax + paths validated. Full v6 e2e build+run lane is a"
-          echo "follow-up — needs docker_build.sh integration to supply image_tag"
-          echo "and base_image build args. See #1525 for tracking."
+          docker compose -f docker/dmsg/docker-compose.e2e-v6.yml down --remove-orphans -v || true

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-go@v6
         with:
-          go-version: '1.26.1'
+          go-version: '1.26.4'
           cache: true
           cache-dependency-path: go.sum
 
@@ -33,8 +33,27 @@ jobs:
       - name: Lint Shell Scripts
         run: make lint-shell
 
+      # golangci-lint already ran above (pinned, cached). check-ci skips the
+      # redundant second golangci-lint run but keeps go vet + config-gen smoke
+      # + the test suite.
       - name: Check Format and Run Tests
-        run: make check
+        run: make check-ci
+
+  # e2e runs as its own job so it executes in parallel with the linux check
+  # job above instead of serially after it — cutting overall wall-clock.
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.4'
+          cache: true
+          cache-dependency-path: go.sum
 
       - name: Setup SSH Key, Build and Run E2E
         shell: bash
@@ -60,7 +79,7 @@ jobs:
 
       - uses: actions/setup-go@v6
         with:
-          go-version: '1.26.1'
+          go-version: '1.26.4'
           cache: true
           cache-dependency-path: go.sum
 
@@ -83,7 +102,7 @@ jobs:
 
       - uses: actions/setup-go@v6
         with:
-          go-version: '1.26.1'
+          go-version: '1.26.4'
           cache: true
           cache-dependency-path: go.sum
 
@@ -100,3 +119,109 @@ jobs:
           $env:GO111MODULE='on'
           make check-windows
         shell: powershell
+
+  # Native (no-Docker) client-side e2e on macOS: runs a small skywire deployment
+  # + two visors as host processes on 127.0.0.1 and exercises the CLIENT runtime
+  # (visor, hypervisor, skysocks-client, vpn-client) — the OS-specific code the
+  # Linux Docker `e2e` job can't reach. See internal/nativee2e. The test binary is
+  # compiled as the runner user, then run under sudo so vpn-client/vpn-server can
+  # create TUN devices and set up pf NAT (macOS vpn-server: os_server_darwin.go).
+  e2e-darwin:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.4'
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Build binaries + compile native e2e
+        run: |
+          set -e
+          mkdir -p ./_nbin
+          go build -mod=vendor -o ./_nbin/skywire ./cmd/skywire
+          for a in skychat skysocks skysocks-client vpn-server vpn-client; do
+            go build -mod=vendor -o ./_nbin/$a ./cmd/apps/$a
+          done
+          go test -c -mod=vendor -tags client_e2e -o ./nativee2e.test ./internal/nativee2e
+
+      - name: Run native client e2e (root for vpn TUN + pf)
+        run: sudo env SKYWIRE_NATIVEE2E_BIN="$PWD/_nbin" ./nativee2e.test -test.v -test.timeout=25m
+
+  # Native client-side e2e on Windows: visor + hypervisor + skysocks-client +
+  # vpn-client/vpn-server (WinTUN + WinNAT, os_server_windows.go). The runner is
+  # admin, so TUN creation + NAT work; wintun.dll is provisioned next to the
+  # binary since it isn't embedded in the wintun Go binding.
+  e2e-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.4'
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Build binaries + provision WinTUN driver
+        shell: pwsh
+        run: |
+          $env:GO111MODULE='on'
+          New-Item -ItemType Directory -Force -Path ./_nbin | Out-Null
+          go build -mod=vendor -o ./_nbin/skywire.exe ./cmd/skywire
+          foreach ($a in 'skychat','skysocks','skysocks-client','vpn-server','vpn-client') {
+            go build -mod=vendor -o ./_nbin/$a.exe ./cmd/apps/$a
+          }
+          # WinTUN driver DLL is NOT embedded in the golang.zx2c4.com/wintun
+          # binding (it LoadLibraryEx's wintun.dll from the app dir / System32).
+          # The visor runs vpn-client/vpn-server in-process, so the DLL must sit
+          # next to skywire.exe (= _nbin) for TUN creation to work. Zip layout:
+          # wintun/bin/amd64/wintun.dll. Try the skywire mirror first, then upstream.
+          $urls = @('https://freeshell.de/~mrpalide/wintun-0.14.1.zip','https://deb.skywire.dev/wintun-0.14.1.zip','https://www.wintun.net/builds/wintun-0.14.1.zip')
+          $ok = $false
+          foreach ($u in $urls) {
+            try { Invoke-WebRequest -Uri $u -OutFile wintun.zip -ErrorAction Stop; $ok = $true; Write-Host "downloaded wintun from $u"; break }
+            catch { Write-Host "wintun download failed from ${u}: $_" }
+          }
+          if (-not $ok) { throw 'could not download wintun.dll from any source' }
+          Expand-Archive -Path wintun.zip -DestinationPath wintun-extracted -Force
+          Copy-Item wintun-extracted/wintun/bin/amd64/wintun.dll ./_nbin/wintun.dll
+          Write-Host "wintun.dll present:" (Test-Path ./_nbin/wintun.dll)
+
+      - name: Run native client e2e (admin runner → vpn TUN + NAT)
+        shell: pwsh
+        run: |
+          $env:GO111MODULE='on'
+          $env:SKYWIRE_NATIVEE2E_BIN = "$PWD/_nbin"
+          go test -mod=vendor -tags client_e2e -v -timeout=25m ./internal/nativee2e/
+
+  # Redis-backed store tests. redis_store_test.go / redis_test.go live behind
+  # //go:build !no_ci because they need a real Redis (redis://localhost:6379),
+  # so the tag-free unit lanes (which run with -tags no_ci) skip them — leaving
+  # them running in NO job. This lane provides a Redis service and runs those
+  # store packages WITHOUT no_ci so the redis-backed tests actually execute.
+  redis-store:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.4'
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Run redis-backed store tests
+        run: go test -mod=vendor -timeout=5m ./pkg/address-resolver/store/... ./pkg/dmsg/discovery/store/...

.golangci.yml

@@ -55,7 +55,8 @@ linters:
   exclusions:
     generated: lax
     paths:
-      - third_party$
+      - vendor/
+      - third_party/
       - cmd/skywire-cli/commands/gotop
       - builtin$
       - examples$

Makefile

@@ -149,6 +149,11 @@ check-help: ## Cursory check of the help menus
 	go run . --help
 	@echo "compilation successful"
 
+vet: ## Run go vet (the non-golangci-lint half of 'lint')
+	CGO_ENABLED=0 ${OPTS} go vet -mod=vendor -tags 'withoutsystray withoutgotop' ./...
+
+check-ci: vet check-cg check-help test ## CI check: like 'check' but skips golangci-lint (CI runs it as a separate pinned, cached step). Keeps go vet + config-gen smoke + tests.
+
 check-windows: lint-windows test-windows ## Run linters and tests on windows image
 
 build: clean build-merged ## Install dependencies, build apps and binaries. `go build` with ${OPTS}
@@ -316,7 +321,6 @@ lint-shell:
 	find ./docker -type f -iname '*.sh' -print0 | xargs -0 -I {} bash -c "$$(command -v ./shellcheck || command -v shellcheck) -e SC2086 \"{}\""
 
 test: ## Run tests
-	-go clean -testcache &>/dev/null
 	${OPTS} go test ${TEST_OPTS} ./internal/... ./pkg/... ./cmd/...
 	${OPTS} go test ${TEST_OPTS}
 
@@ -556,9 +560,9 @@ e2e-run: ## E2E. Start e2e environment and wait for all health checks to pass
 	@# uptime-tracker + its postgres are gone (uptime is integrated
 	@# into the discovery services).
 	bash -c "DOCKER_TAG=e2e docker compose up -d --wait redis"
-	bash -c "DOCKER_TAG=e2e docker compose up -d --wait deployment-services"
-	bash -c "DOCKER_TAG=e2e docker compose up -d --wait visor-b"
-	bash -c "DOCKER_TAG=e2e docker compose up -d --wait visor-a visor-c"
+	bash -c "DOCKER_TAG=e2e docker compose up -d --wait deployment-services || { echo '=== deployment-services unhealthy — logs: ==='; docker compose logs --tail=150 deployment-services; exit 1; }"
+	bash -c "DOCKER_TAG=e2e docker compose up -d --wait visor-b || { echo '=== visor-b unhealthy — deployment-services (dmsg-server side) + visor-b logs: ==='; docker compose logs --tail=200 deployment-services; docker compose logs --tail=120 visor-b; exit 1; }"
+	bash -c "DOCKER_TAG=e2e docker compose up -d --wait visor-a visor-c || { echo '=== visor-a/visor-c unhealthy — logs: ==='; docker compose logs --tail=150 visor-a visor-c; exit 1; }"
 	bash -c "DOCKER_TAG=e2e docker compose ps"
 
 e2e-logs: