diff --git a/.circleci/config.yml b/.circleci/config.yml index 774d9817f..6a2bc5ef8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,32 +1,28 @@ version: 2 + +# https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ defaults: &defaults - machine: - docker_layer_caching: true - working_directory: ~/weaveworks/service + docker: + - image: weaveworks/service-build:master-8245ec7d + working_directory: /go/src/github.com/weaveworks/service + jobs: lint: <<: *defaults steps: - checkout - - run: make RM= lint + - run: make BUILD_IN_CONTAINER=false lint build: <<: *defaults steps: - checkout - - run: sudo docker info >/dev/null 2>&1 || sudo service docker start; - # FIXME: we probably don't need to copy the code and change the working dir - - run: echo -e "export GOPATH=$HOME\nexport SRCDIR=$HOME/src/github.com/weaveworks/service\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV - - run: mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR - - run: | - cd $SRCDIR/build - ../tools/rebuild-image quay.io/weaveworks/build . build.sh Dockerfile - touch .uptodate - - run: cd $SRCDIR; make RM= + - setup_remote_docker + - run: make BUILD_IN_CONTAINER=false SUDO= - run: | set -e set -o pipefail - cd $SRCDIR; if [ $(./tools/image-tag | grep -e '-WIP$') ]; then + if [ $(./tools/image-tag | grep -e '-WIP$') ]; then echo "WIP build; exiting with error to mark as a failure" git --no-pager diff exit 1 @@ -46,15 +42,22 @@ jobs: docker tag ${tagged_image} ${repo_tmp_image} docker push $repo_tmp_image done + - persist_to_workspace: + root: . + paths: + - users/client/mock_client.go + - gcp-service/grpc/gcp-service.pb.go + - users/users.pb.go + - common/billing/grpc/billing.pb.go + - kubectl-service/grpc/kubectl-service.pb.go + test: <<: *defaults steps: - checkout - - run: echo -e "export GOPATH=$HOME\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV - # Link the working directory so that it appears in GOPATH. This is required for cover to map coverage statistics to sources. - - run: mkdir -p ${GOPATH}/src/github.com/weaveworks && ln -s $(pwd) ${GOPATH}/src/github.com/weaveworks/service + - setup_remote_docker - run: go get github.com/mattn/goveralls - - run: COVERDIR=./coverage make RM= test + - run: COVERDIR=./coverage make BUILD_IN_CONTAINER=false test - run: ./tools/cover/gather_coverage.sh ./coverage $CIRCLE_WORKING_DIRECTORY/coverage - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$CIRCLE_WORKING_DIRECTORY/profile.cov -service=circleci || true - run: mkdir /tmp/coverage && cp coverage.* /tmp/coverage @@ -65,6 +68,7 @@ jobs: <<: *defaults steps: - checkout + - setup_remote_docker - &pull-tmp-images run: name: Download docker images build earlier in workflow @@ -84,28 +88,29 @@ jobs: docker tag ${repo_tmp_image} ${image}:latest docker image rm ${repo_tmp_image} done - # FIXME: we probably don't need to copy the code and change the working dir - - run: echo -e "export GOPATH=$HOME\nexport SRCDIR=$HOME/src/github.com/weaveworks/service\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV - - run: mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR + - attach_workspace: + at: . - run: go get github.com/nats-io/gnatsd - run: command: gnatsd background: true - run: | - cd $SRCDIR - make RM= notebooks-integration-test - make RM= users-integration-test - make RM= billing-integration-test - make RM= pubsub-integration-test - make RM= flux-integration-test - make RM= kubectl-service-integration-test - make RM= gcp-service-integration-test - make RM= notification-integration-test + touch users/client/mock_client.go gcp-service/grpc/gcp-service.pb.go users/users.pb.go common/billing/grpc/billing.pb.go kubectl-service/grpc/kubectl-service.pb.go + make touch-uptodate + make BUILD_IN_CONTAINER=false notification-integration-test + make BUILD_IN_CONTAINER=false notebooks-integration-test + make BUILD_IN_CONTAINER=false users-integration-test + make BUILD_IN_CONTAINER=false billing-integration-test + make BUILD_IN_CONTAINER=false pubsub-integration-test + make BUILD_IN_CONTAINER=false flux-integration-test + make BUILD_IN_CONTAINER=false kubectl-service-integration-test + make BUILD_IN_CONTAINER=false gcp-service-integration-test upload: <<: *defaults steps: - checkout + - setup_remote_docker - <<: *pull-tmp-images - run: name: Publish docker images to final repository @@ -119,6 +124,7 @@ jobs: <<: *defaults steps: - checkout + - setup_remote_docker - <<: *pull-tmp-images - run: name: Dry-run of publishing docker images diff --git a/Makefile b/Makefile index 1a8652576..e2ed2f9e3 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ images: $(info $(IMAGE_NAMES)) @echo > /dev/null +touch-uptodate: + touch $(UPTODATE_FILES) + all: $(UPTODATE_FILES) # Generating proto code is automated. @@ -176,105 +179,103 @@ $(EXES) test: build/$(UPTODATE) $(PROTO_GOS) -e TESTDIRS=${TESTDIRS} \ $(IMAGE_PREFIX)/build $@ -billing-integration-test: build/$(UPTODATE) - @mkdir -p $(shell pwd)/.pkg - DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=billing_test' postgres:9.5)"; \ - $(SUDO) docker run $(RM) -ti \ - -v $(shell pwd)/.pkg:/go/pkg \ - -v $(shell pwd):/go/src/github.com/weaveworks/service \ - -v $(shell pwd)/billing-api/db/migrations:/migrations \ - --workdir /go/src/github.com/weaveworks/service \ - --link "$$DB_CONTAINER":billing-db.weave.local \ - $(IMAGE_PREFIX)/build $@; \ - status=$$?; \ - test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \ - exit $$status - -flux-integration-test: build/$(UPTODATE) - @mkdir -p $(shell pwd)/.pkg - NATS_CONTAINER="$$(docker run -d nats)"; \ - POSTGRES_CONTAINER="$$(docker run -d postgres)"; \ - $(SUDO) docker run $(RM) -ti \ - -v $(shell pwd)/.pkg:/go/pkg \ - -v $(shell pwd):/go/src/github.com/weaveworks/service \ - -v $(shell pwd)/billing-api/db/migrations:/migrations \ - --workdir /go/src/github.com/weaveworks/service \ - --link "$$NATS_CONTAINER":nats \ - --link "$$POSTGRES_CONTAINER":postgres \ - $(IMAGE_PREFIX)/build $@; \ - status=$$?; \ - test -n "$(CIRCLECI)" || docker rm -f "$$NATS_CONTAINER" "$$POSTGRES_CONTAINER"; \ - exit $$status - else -$(EXES): build/$(UPTODATE) $(PROTO_GOS) +$(EXES): $(PROTO_GOS) go build $(GO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) -%.pb.go: build/$(UPTODATE) +%.pb.go: protoc -I ./vendor:./$(@D) --gogoslick_out=plugins=grpc:./$(@D) ./$(patsubst %.pb.go,%.proto,$@) -lint: build/$(UPTODATE) +lint: ./tools/lint . -test: build/$(UPTODATE) $(PROTO_GOS) $(MOCK_GOS) +test: $(PROTO_GOS) $(MOCK_GOS) TESTDIRS=${TESTDIRS} ./tools/test -netgo -no-race -$(MOCK_USERS): build/$(UPTODATE) +$(MOCK_USERS): mockgen -destination=$@ github.com/weaveworks/service/users UsersClient \ && sed -i'' s,github.com/weaveworks/service/vendor/,, $@ -$(MOCK_BILLING_DB): build/$(UPTODATE) $(BILLING_DB)/db.go +$(MOCK_BILLING_DB): $(BILLING_DB)/db.go mockgen -destination=$@ github.com/weaveworks/service/$(BILLING_DB) DB -$(MOCK_BILLING_GRPC): build/$(UPTODATE) +$(MOCK_BILLING_GRPC): mockgen -source=$(BILLING_GRPC) -destination=$@ -package=grpc -$(MOCK_COMMON_GCP_PARTNER_CLIENT): build/$(UPTODATE) +$(MOCK_COMMON_GCP_PARTNER_CLIENT): mockgen -destination=$@ github.com/weaveworks/service/common/gcp/partner API \ && sed -i'' s,github.com/weaveworks/service/vendor/,, $@ -$(MOCK_COMMON_GCP_PARTNER_ACCESS): build/$(UPTODATE) +$(MOCK_COMMON_GCP_PARTNER_ACCESS): mockgen -destination=$@ github.com/weaveworks/service/common/gcp/partner Accessor \ && sed -i'' s,github.com/weaveworks/service/vendor/,, $@ -billing-integration-test: build/$(UPTODATE) $(MOCK_GOS) - /bin/bash -c "go test -tags 'netgo integration' -timeout 30s $(BILLING_TEST_DIRS)" +endif + + +# Test and misc stuff +INTEGRATION_TEST_IMAGE=golang:1.9.2-stretch + +billing-integration-test: $(MOCK_GOS) + @mkdir -p $(shell pwd)/.pkg + DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=billing_test' postgres:9.5)"; \ + $(SUDO) docker run $(RM) -ti \ + -v $(shell pwd)/.pkg:/go/pkg \ + -v $(shell pwd):/go/src/github.com/weaveworks/service \ + -v $(shell pwd)/billing-api/db/migrations:/migrations \ + --workdir /go/src/github.com/weaveworks/service \ + --link "$$DB_CONTAINER":billing-db.weave.local \ + $(INTEGRATION_TEST_IMAGE) \ + /bin/bash -c "go test -tags 'netgo integration' -timeout 30s $(BILLING_TEST_DIRS)"; \ + status=$$?; \ + test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \ + exit $$status flux-integration-test: + @mkdir -p $(shell pwd)/.pkg # These packages must currently be tested in series because # otherwise they will all race to run migrations. - /bin/bash -c "go test -tags integration -timeout 30s ./flux-api" - /bin/bash -c "go test -tags integration -timeout 30s ./flux-api/bus/nats" - /bin/bash -c "go test -tags integration -timeout 30s ./flux-api/history/sql" - /bin/bash -c "go test -tags integration -timeout 30s ./flux-api/instance/sql" - -endif - + NATS_CONTAINER="$$(docker run -d nats)"; \ + POSTGRES_CONTAINER="$$(docker run -d postgres)"; \ + $(SUDO) docker run $(RM) -ti \ + -v $(shell pwd)/.pkg:/go/pkg \ + -v $(shell pwd):/go/src/github.com/weaveworks/service \ + -v $(shell pwd)/billing-api/db/migrations:/migrations \ + --workdir /go/src/github.com/weaveworks/service \ + --link "$$NATS_CONTAINER":nats \ + --link "$$POSTGRES_CONTAINER":postgres \ + $(INTEGRATION_TEST_IMAGE) \ + /bin/bash -c "go test -tags integration -timeout 30s ./flux-api && \ + go test -tags integration -timeout 30s ./flux-api/bus/nats &&\ + go test -tags integration -timeout 30s ./flux-api/history/sql &&\ + go test -tags integration -timeout 30s ./flux-api/instance/sql"; \ + status=$$?; \ + test -n "$(CIRCLECI)" || docker rm -f "$$NATS_CONTAINER" "$$POSTGRES_CONTAINER"; \ + exit $$status -# Test and misc stuff -notebooks-integration-test: $(NOTEBOOKS_UPTODATE) +notebooks-integration-test: DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=notebooks_test' postgres:9.5)"; \ docker run $(RM) \ -v $(shell pwd):/go/src/github.com/weaveworks/service \ -v $(shell pwd)/notebooks/db/migrations:/migrations \ --workdir /go/src/github.com/weaveworks/service/notebooks \ --link "$$DB_CONTAINER":configs-db.weave.local \ - golang:1.9.2-stretch \ + $(INTEGRATION_TEST_IMAGE) \ /bin/bash -c "go test -tags integration -timeout 30s ./..."; \ status=$$?; \ test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \ exit $$status -users-integration-test: $(USERS_UPTODATE) $(PROTO_GOS) $(MOCK_GOS) +users-integration-test: $(PROTO_GOS) $(MOCK_GOS) DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=users_test' postgres:9.5)"; \ docker run $(RM) \ -v $(shell pwd):/go/src/github.com/weaveworks/service \ -v $(shell pwd)/users/db/migrations:/migrations \ --workdir /go/src/github.com/weaveworks/service/users \ --link "$$DB_CONTAINER":users-db.weave.local \ - golang:1.9.2-stretch \ + $(INTEGRATION_TEST_IMAGE) \ /bin/bash -c "go test -tags integration -timeout 30s ./..."; \ status=$$?; \ test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \ @@ -286,31 +287,31 @@ pubsub-integration-test: -v $(shell pwd):/go/src/github.com/weaveworks/service \ --net=host -p 127.0.0.1:1337:1337 \ --workdir /go/src/github.com/weaveworks/service/common/gcp/pubsub \ - golang:1.9.2-stretch \ + $(INTEGRATION_TEST_IMAGE) \ /bin/bash -c "RUN_MANUAL_TEST=1 go test -tags integration -timeout 30s ./..."; \ status=$$?; \ test -n "$(CIRCLECI)" || docker rm -f "$$PUBSUB_EMU_CONTAINER"; \ exit $$status -kubectl-service-integration-test: kubectl-service/$(UPTODATE) kubectl-service/grpc/kubectl-service.pb.go +kubectl-service-integration-test: kubectl-service/grpc/kubectl-service.pb.go SVC_CONTAINER="$$(docker run -d -p 4887:4772 -p 8887:80 $(IMAGE_PREFIX)/kubectl-service -dry-run=true)"; \ docker run $(RM) \ -v $(shell pwd):/go/src/github.com/weaveworks/service \ --workdir /go/src/github.com/weaveworks/service/kubectl-service \ --link "$$SVC_CONTAINER":kubectl-service.weave.local \ - golang:1.9.2-stretch \ + $(INTEGRATION_TEST_IMAGE) \ /bin/bash -c "go test -tags integration -timeout 30s ./..."; \ status=$$?; \ test -n "$(CIRCLECI)" || docker rm -f "$$SVC_CONTAINER"; \ exit $$status -gcp-service-integration-test: gcp-service/$(UPTODATE) gcp-service/grpc/gcp-service.pb.go +gcp-service-integration-test: gcp-service/grpc/gcp-service.pb.go SVC_CONTAINER="$$(docker run -d -p 4888:4772 -p 8888:80 $(IMAGE_PREFIX)/gcp-service -dry-run=true)"; \ docker run $(RM) \ -v $(shell pwd):/go/src/github.com/weaveworks/service \ --workdir /go/src/github.com/weaveworks/service/gcp-service \ --link "$$SVC_CONTAINER":gcp-service.weave.local \ - golang:1.9.2-stretch \ + $(INTEGRATION_TEST_IMAGE) \ /bin/bash -c "go test -tags integration -timeout 30s ./..."; \ status=$$?; \ test -n "$(CIRCLECI)" || docker rm -f "$$SVC_CONTAINER"; \