diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..88f33140 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +@lucaspin diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..7cf705b4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +on: [push, pull_request] +name: Test +jobs: + unit-testing: + runs-on: windows-latest + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.23.x + - name: Check out repository code + uses: actions/checkout@v2 + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest + - name: Test + run: gotestsum --format short-verbose --packages="./..." -- -p 1 diff --git a/.gitignore b/.gitignore index 570a6ba3..7fe7de22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /build /log agent +.vagrant/ diff --git a/.goreleaser.yml b/.goreleaser.yml index 0e55ebd6..de3fd1bf 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,9 +1,11 @@ +version: 2 project_name: agent before: hooks: - go get ./... builds: -- env: +- id: non-windows-build + env: - CGO_ENABLED=0 ldflags: - -s -w -X main.VERSION={{.Tag}} @@ -13,15 +15,49 @@ builds: goarch: - 386 - amd64 + - arm + - arm64 + - ppc64le +- id: windows-build + env: + - CGO_ENABLED=0 + ldflags: + - -s -w -X main.VERSION={{.Tag}} + goos: + - windows + goarch: + - 386 + - amd64 + - arm archives: - - id: agent - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' - replacements: - darwin: Darwin - linux: Linux - 386: i386 - amd64: x86_64 + - id: non-windows-archive + builds: + - non-windows-build + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + files: + - README.md + - install.sh + + - id: windows-archive + builds: + - windows-build + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + files: + - README.md + - install.ps1 checksum: name_template: '{{ .ProjectName }}_checksums.txt' @@ -35,10 +71,13 @@ changelog: - Merge branch brews: - - github: + - repository: owner: semaphoreci name: homebrew-tap - folder: Formula + commit_author: + name: release-bot-agent + email: contact+release-bot-agent@renderedtext.com + directory: Formula homepage: https://semaphoreci.com description: Semaphore 2.0 agent. test: | diff --git a/.semaphore/release.yml b/.semaphore/release.yml index 0154c61d..69c9f68b 100644 --- a/.semaphore/release.yml +++ b/.semaphore/release.yml @@ -2,8 +2,8 @@ version: "v1.0" name: Release agent: machine: - type: e1-standard-4 - os_image: ubuntu1804 + type: f1-standard-2 + os_image: ubuntu2404 blocks: - name: "Release" task: @@ -11,14 +11,19 @@ blocks: - name: GO111MODULE value: "on" secrets: - - name: sem-robot-ghtoken + - name: github-release-bot-agent + - name: dockerhub-write prologue: commands: - - sem-version go 1.13 + - sem-version go 1.23.8 - "export GOPATH=~/go" - "export PATH=/home/semaphore/go/bin:$PATH" - checkout jobs: - - name: Sem Agent + - name: Release new version commands: - - curl -sL https://git.io/goreleaser | bash -s -- --rm-dist + - export GITHUB_TOKEN=$ACCESS_TOKEN + - curl -sL https://git.io/goreleaser | bash -s -- --clean + - echo $DOCKERHUB_PASSWORD | docker login -u $DOCKERHUB_USERNAME --password-stdin + - make docker.build + - make docker.push diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fff7af9c..01b93609 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -2,11 +2,14 @@ version: v1.0 name: Agent agent: machine: - type: e1-standard-2 - os_image: ubuntu1804 + type: f1-standard-2 + os_image: ubuntu2404 + +execution_time_limit: + minutes: 10 blocks: - - name: "Tests" + - name: "Lint" dependencies: [] task: env_vars: @@ -15,18 +18,43 @@ blocks: prologue: commands: - - sem-version go 1.13 + - sem-version go 1.24.12 - checkout - - go version - - go get - - go build jobs: - - name: Unit Tests + - name: Lint commands: - - make test + - go install github.com/mgechev/revive@latest + - make lint + + - name: "Security checks" + dependencies: [] + task: + secrets: + - name: security-toolbox-shared-read-access + prologue: + commands: + - checkout + - mv ~/.ssh/security-toolbox ~/.ssh/id_rsa + - sudo chmod 600 ~/.ssh/id_rsa + - sem-version go 1.24.12 + jobs: + - name: Check dependencies + commands: + - make check.deps + - name: Check code + commands: + - make check.static + - name: Check docker + commands: + - make docker.build + - make check.docker + epilogue: + always: + commands: + - 'if [ -f results.xml ]; then test-results publish --name="Security checks" results.xml; fi' - - name: "Shell Executor E2E tests" + - name: "Tests" dependencies: [] task: env_vars: @@ -35,45 +63,29 @@ blocks: prologue: commands: - - sem-version go 1.13 + - sem-version go 1.24.12 - checkout - go version - go get - go build - epilogue: - commands: - - docker exec -ti agent cat /tmp/agent_log - jobs: - - name: Shell + - name: Unit Tests commands: - - "make e2e TEST=shell/$TEST" - matrix: - - env_var: TEST - values: - - command_aliases - - env_vars - - failed_job - - job_stopping - - file_injection - - file_injection_broken_file_mode - - stty_restoration - - epilogue_on_pass - - epilogue_on_fail - - ssh_jump_points - - unicode - - unknown_command - - broken_unicode - - killing_root_bash - - set_e - - set_pipefail + - go install gotest.tools/gotestsum@latest + - make test + + epilogue: + always: + commands: + - test-results publish junit-report.xml - name: "Docker Executor E2E" dependencies: [] task: secrets: - - name: aws-ecr-payground + - name: aws-ecr-agent-e2e-secret + - name: hellopuller - name: gcr-test-secret - name: docker-registry-test-secret env_vars: @@ -84,28 +96,36 @@ blocks: prologue: commands: - - sem-version go 1.13 + - sem-version go 1.24.12 - checkout - go version - go get - go build + - mkdir /tmp/agent epilogue: commands: - - docker exec -ti agent cat /tmp/agent_log + - if [ "$TEST_MODE" = "api" ]; then docker exec -ti agent cat /tmp/agent_log; else docker logs e2e_support_agent_1; fi + - if [ "$TEST_MODE" = "api" ]; then echo "No hub"; else docker logs e2e_support_hub_1; fi jobs: - name: Docker commands: - "make e2e TEST=docker/$TEST" matrix: + - env_var: TEST_MODE + values: + - api + - listen - env_var: TEST values: - hello_world - command_aliases - env_vars - failed_job + - job_logs_as_artifact - job_stopping + - job_stopping_on_epilogue - file_injection - file_injection_broken_file_mode - stty_restoration @@ -115,11 +135,12 @@ blocks: - container_env_vars - container_options - dockerhub_private_image - - docker_registry_private_image - docker_private_image_ecr + - docker_private_image_ecr_v2 + - docker_private_image_ecr_account_id + - docker_private_image_ecr_account_id_v2 - docker_private_image_gcr - dockerhub_private_image_bad_creds - - docker_registry_private_image_bad_creds - docker_private_image_ecr_bad_creds - docker_private_image_gcr_bad_creds - ssh_jump_points @@ -131,6 +152,134 @@ blocks: - check_dev_kvm - host_setup_commands - multiple_containers + - compose_v1 + + - name: "Hosted E2E tests" + dependencies: [] + task: + env_vars: + - name: GO111MODULE + value: "on" + + prologue: + commands: + - sem-version go 1.24.12 + - checkout + - go version + - go get + - go build + - mkdir /tmp/agent + + epilogue: + commands: + - docker exec -ti agent cat /tmp/agent_log + + jobs: + - name: Hosted + commands: + - "TEST_MODE=api make e2e TEST=hosted/$TEST" + matrix: + - env_var: TEST + values: + - ssh_jump_points + - job_logs_as_artifact_default + - job_logs_as_artifact_always + - job_logs_as_artifact_compressed + - job_logs_as_artifact_never + - job_logs_as_artifact_not_trimmed + - job_logs_as_artifact_trimmed + + - name: "Self hosted E2E" + dependencies: [] + task: + env_vars: + - name: GO111MODULE + value: "on" + - name: TEST_MODE + value: "listen" + + prologue: + commands: + - sem-version go 1.24.12 + - checkout + - go version + - go get + - go build + - mkdir /tmp/agent + + epilogue: + commands: + - docker logs e2e_support_agent_1 + - docker logs e2e_support_hub_1 + + jobs: + - name: Self hosted + commands: + - "make e2e TEST=self-hosted/$TEST" + matrix: + - env_var: TEST + values: + - docker_compose_host_env_vars + - docker_compose_host_files + - docker_compose_missing_host_files + - docker_compose_fail_on_missing_host_files + + - name: "Kubernetes Executor E2E" + dependencies: [] + task: + secrets: + - name: aws-ecr-agent-e2e-secret + - name: gcr-test-secret + - name: docker-registry-test-secret + env_vars: + - name: GO111MODULE + value: "on" + - name: TEST_MODE + value: "listen" + - name: AWS_REGION + value: "us-east-1" + + prologue: + commands: + - sem-version go 1.24.12 + - curl -sLO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && install minikube-linux-amd64 /tmp/ + - /tmp/minikube-linux-amd64 config set WantUpdateNotification false + - /tmp/minikube-linux-amd64 start --driver=docker + - checkout + - mkdir /tmp/agent + # The docker container uses root, and not semaphore + - grep -rl "/home/semaphore" ~/.kube | xargs sed -i 's/home\/semaphore/root/g' + - grep -rl "/home/semaphore" ~/.minikube | xargs sed -i 's/home\/semaphore/root/g' + + epilogue: + commands: + - docker logs e2e_support_agent_1 + - docker logs e2e_support_hub_1 + + jobs: + - name: Kubernetes executor + commands: + - "make e2e TEST=kubernetes/$TEST" + matrix: + - env_var: TEST + values: + - shell__not-allowed + - shell__default-image + - docker_compose__env-vars + - docker_compose__epilogue + - docker_compose__file-injection + - docker_compose__multiple-containers + - job_logs_as_artifact + - private_image_gcr + - private_image_ecr_no_account_id + - private_image_ecr_with_account_id + +after_pipeline: + task: + jobs: + - name: Submit Reports + commands: + - test-results gen-pipeline-report promotions: - name: Release diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..54eed35c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# Repository Guidelines + +## Onboarding & Architecture Reference +- Start by reading `DOCUMENTATION.md` for a walkthrough of architecture, component ownership, and common triage paths before tackling changes. +- Keep the document handy when planning fixes; it maps package responsibilities, runtime flow, and gotchas you should re-check during reviews. + +## Project Structure & Module Organization +- Root CLI entry point is `main.go`; domain logic lives under `pkg/` (e.g. `executors/`, `listener/`, `kubernetes/`) with packages named for their subsystem. +- Runtime assets such as sample configs reside in `config.example.yaml`, release docs live in `docs/`, and Docker/Vagrant files support local environments; built binaries land in `build/`. +- Tests and fixtures sit in `pkg/*/*_test.go` for unit coverage and Ruby E2E helpers in `test/` (see `test/e2e` and `test/e2e_support`). + +## Build, Test, and Development Commands +- `make build` compiles a Linux CLI at `build/agent`; use `make build.windows` for a Windows artifact or `make docker.build.dev` for a dev image. +- `make test` runs `gotestsum` across `./...` with a JUnit report; run `go test ./pkg/listener -run TestName` to iterate on a single package. +- `make run JOB=` executes the agent locally; `make serve` starts the API shim with a static auth token for manual workflows. +- `make lint` executes `revive` with `lint.toml`; the `check.*` targets pull the security-toolbox for deeper audits when needed. + +## Coding Style & Naming Conventions +- Target Go 1.23; always format with `gofmt` (or `go fmt ./...`) before pushing and organize imports with `goimports`. +- Exported identifiers follow Go’s PascalCase, package-private helpers use camelCase, and test doubles mirror the production type names with `Fake` or `Stub` suffixes. +- Keep configuration structs in `pkg/config` aligned with `config.example.yaml`; when introducing flags, add them to `pkg/server` command wiring. + +## Testing Guidelines +- Co-locate `*_test.go` files with their packages and favor table-driven tests plus `testify` assertions already in use. +- Aim to cover new branches and failure paths, especially around `pkg/executors` and `pkg/listener` which guard remote job execution. +- For smoke verification against remote services, add or extend Ruby scripts in `test/e2e` and run with `make e2e TEST=