From c06b3ec9eb01d8db9163a2e61239a105d320c5ac Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Wed, 21 Sep 2016 16:50:30 +0900 Subject: [PATCH 01/37] fix gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d80b0b3bdf..97ee0273ac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ vuls .vscode *.txt *.json -*.sqlite3 +*.sqlite3* *.db tags .gitmodules From 093bcb7477b4781e81e36ec7a8bc6bfe79f9e904 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Fri, 23 Sep 2016 21:59:27 +0900 Subject: [PATCH 02/37] Fix error handling in tui --- report/tui.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/report/tui.go b/report/tui.go index 4a41635bc8..bd63026c2c 100644 --- a/report/tui.go +++ b/report/tui.go @@ -44,26 +44,28 @@ func RunTui(jsonDirName string) subcommands.ExitStatus { var err error scanHistory, err = selectScanHistory(jsonDirName) if err != nil { - log.Fatal(err) + log.Errorf("%s", err) return subcommands.ExitFailure } g := gocui.NewGui() if err := g.Init(); err != nil { - log.Panicln(err) + log.Errorf("%s", err) + return subcommands.ExitFailure } defer g.Close() g.SetLayout(layout) if err := keybindings(g); err != nil { - log.Panicln(err) + log.Errorf("%s", err) + return subcommands.ExitFailure } g.SelBgColor = gocui.ColorGreen g.SelFgColor = gocui.ColorBlack g.Cursor = true if err := g.MainLoop(); err != nil && err != gocui.ErrQuit { - log.Panicln(err) + log.Errorf("%s", err) return subcommands.ExitFailure } From 95bf387ecce5b60886d6f5e060956c039446c3e5 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Sat, 24 Sep 2016 20:35:41 +0900 Subject: [PATCH 03/37] Fix a help message of -report-azure-blob option --- commands/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/scan.go b/commands/scan.go index db32993127..da232db048 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -208,7 +208,7 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.reportAzureBlob, "report-azure-blob", false, - "Write report to S3 (container/yyyyMMdd_HHmm/servername.json)", + "Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json)", ) f.StringVar(&p.azureAccount, "azure-account", "", "Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified") f.StringVar(&p.azureKey, "azure-key", "", "Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified") From 2f89a24100a67e7071b0b896231eadc4d710d665 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Sat, 1 Oct 2016 12:38:51 +0900 Subject: [PATCH 04/37] Fix defer cache.DB.close --- scan/serverapi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scan/serverapi.go b/scan/serverapi.go index a26e54493d..2e7b87aefc 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -451,7 +451,7 @@ func Scan() []error { defer func() { if cache.DB != nil { - defer cache.DB.Close() + cache.DB.Close() } }() From c2eab87a3f3af0809d6c0b41e6c84d1a6eb92e25 Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Wed, 21 Sep 2016 11:49:27 +0900 Subject: [PATCH 05/37] fix docker --- .gitignore | 1 + README.ja.md | 1 - setup/docker/README.ja.md | 101 ------------ setup/docker/README.md | 147 ++++++++++-------- setup/docker/base/latest/Dockerfile | 12 ++ setup/docker/base/latest/README.md | 14 ++ setup/docker/conf/.gitkeep | 0 setup/docker/docker-compose.yml | 11 -- setup/docker/dockerfile/Dockerfile | 73 --------- .../dockerfile/scripts/fetch_jvn_all.sh | 6 - .../dockerfile/scripts/fetch_jvn_last2y.sh | 6 - .../dockerfile/scripts/fetch_jvn_latest.sh | 5 - .../dockerfile/scripts/fetch_nvd_all.sh | 6 - .../dockerfile/scripts/fetch_nvd_last2y.sh | 6 - .../dockerfile/scripts/scan_for_vulsrepo.sh | 8 - .../dockerfile/scripts/update_modules.sh | 16 -- .../go-cve-dictionary/latest/Dockerfile | 21 +++ .../docker/go-cve-dictionary/latest/README.md | 66 ++++++++ setup/docker/vuls/latest/Dockerfile | 21 +++ setup/docker/vuls/latest/README.md | 96 ++++++++++++ setup/docker/vulsrepo/latest/Dockerfile | 31 ++++ setup/docker/vulsrepo/latest/README.md | 44 ++++++ 22 files changed, 389 insertions(+), 303 deletions(-) delete mode 100644 setup/docker/README.ja.md create mode 100644 setup/docker/base/latest/Dockerfile create mode 100644 setup/docker/base/latest/README.md delete mode 100644 setup/docker/conf/.gitkeep delete mode 100644 setup/docker/docker-compose.yml delete mode 100644 setup/docker/dockerfile/Dockerfile delete mode 100644 setup/docker/dockerfile/scripts/fetch_jvn_all.sh delete mode 100644 setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh delete mode 100644 setup/docker/dockerfile/scripts/fetch_jvn_latest.sh delete mode 100644 setup/docker/dockerfile/scripts/fetch_nvd_all.sh delete mode 100644 setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh delete mode 100644 setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh delete mode 100644 setup/docker/dockerfile/scripts/update_modules.sh create mode 100644 setup/docker/go-cve-dictionary/latest/Dockerfile create mode 100644 setup/docker/go-cve-dictionary/latest/README.md create mode 100644 setup/docker/vuls/latest/Dockerfile create mode 100644 setup/docker/vuls/latest/README.md create mode 100644 setup/docker/vulsrepo/latest/Dockerfile create mode 100644 setup/docker/vulsrepo/latest/README.md diff --git a/.gitignore b/.gitignore index d80b0b3bdf..39a48a8e6a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ vendor/ log/ results/ *config.toml +!setup/docker/* diff --git a/README.ja.md b/README.ja.md index 8df0f87a7a..1f24a11dc9 100644 --- a/README.ja.md +++ b/README.ja.md @@ -65,7 +65,6 @@ Vulsは上に挙げた手動運用での課題を解決するツールであり Vulsのセットアップは以下の3パターンがある - Dockerコンテナ上にセットアップ -Docker Composeを用いて少ないコマンドでセットアップ可能 see https://github.com/future-architect/vuls/tree/master/setup/docker [日本語README](https://github.com/future-architect/vuls/blob/master/setup/docker/README.ja.md) - Chefでセットアップ diff --git a/setup/docker/README.ja.md b/setup/docker/README.ja.md deleted file mode 100644 index 831060c8d1..0000000000 --- a/setup/docker/README.ja.md +++ /dev/null @@ -1,101 +0,0 @@ -# Vuls on Docker - -## What's Vuls-On-Docker - -- 数個のコマンドを実行するだけでVulsとvulsrepoのセットアップが出来るスクリプト -- Dockerコンテナ上にVulsと[vulsrepo](https://github.com/usiusi360/vulsrepo)をセットアップ可能 -- スキャン結果をvulsrepoでブラウザで分析可能 -- 脆弱性データベースの更新が可能 -- モジュールのアップデートが可能 - -## Setting up your machine - -1. [Install Docker](https://docs.docker.com/engine/installation/) -2. [Install Docker-Compose](https://docs.docker.com/compose/install/) -3. 実行前に以下のコマンドが実行可能なことを確認する - - ``` - $ docker version - $ docker-compose version - ``` - -4. Vulsをgit clone - ``` - mkdir work - cd work - git clone https://github.com/future-architect/vuls.git - cd vuls/setup/docker - ``` - -## Start A Vuls Container - -- 以下のコマンドを実行してコンテナをビルドする - - ``` - $ docker-compose up -d - ``` - -## Setting up Vuls - -1. スキャン対象サーバのSSH秘密鍵を保存(vuls/setup/docker/conf/)する -2. config.toml(vuls/setup/docker/conf/config.toml) を環境に合わせて作成する - - ``` - [servers] - - [servers.172-31-4-82] - host = "172.31.4.82" - user = "ec2-user" - keyPath = "conf/id_rsa" - ``` - -## Fetch Vulnerability database - -- NVDから脆弱性データベースを取得する - ``` - $ docker exec -t vuls scripts/fetch_nvd_all.sh - ``` - -- レポートを日本語化する場合は、JVNから脆弱性データを取得する - ``` - $ docker exec -t vuls scripts/fetch_jvn_all.sh - ``` - -## Scan servers with Vuls-On-Docker - -- スキャンを実行する - - ``` - $ docker exec -t vuls vuls prepare -config=conf/config.toml - $ docker exec -t vuls scripts/scan_for_vulsrepo.sh - ``` - -## See the results in a browser - -``` -http://${Vuls_Host}/vulsrepo/ -``` - -# Update modules - -- vuls, go-cve-dictionary, vulsrepoのモジュールをアップデートする - ``` - $ docker exec -t vuls scripts/update_modules.sh - ``` - -# Update Vulnerability database - -- NVDの過去2年分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_nvd_last2y.sh - ``` - -- JVNの過去1ヶ月分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_jvn_month.sh - ``` - -- JVNの過去1週間分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_jvn_week.sh - ``` diff --git a/setup/docker/README.md b/setup/docker/README.md index 2df2d58d59..5f63583783 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -1,87 +1,106 @@ -# Vuls on Docker +# Vuls Docker components -## What's Vuls-On-Docker +This is the Git repo of the official Docker image for vuls. -- This is a dockernized-Vuls with vulsrepo UI in it. -- It's designed to reduce the cost of installation and the dependencies that vuls requires. -- You can run install and run Vuls on your machine with only a few commands. -- The result can be viewed with a browser +# Supported tags and respective `Dockerfile` links -## Setting up your machine - -1. [Install Docker](https://docs.docker.com/engine/installation/) -2. [Install Docker-Compose](https://docs.docker.com/compose/install/) -3. Make sure that you can run the following commands before you move on. +- go-cve-dictionary + - [`latest` (*go-cve-dictionary:latest Dockerfile*)]() +- vuls + - [`latest` (*vuls:latest Dockerfile*)]() +- vulsrepo + - [`latest` (*vulsrepo:latest Dockerfile*)]() - ``` - $ docker version - $ docker-compose version - ``` - -4. git clone vuls - ``` - mkdir work - cd work - git clone https://github.com/future-architect/vuls.git - cd vuls/setup/docker - ``` +This image version is same as the github repository version. +# How to use this image +1. fetch nvd (vuls/go-cve-dictionary) +1. configuration (vuls/vuls) +1. prepare (vuls/vuls) +1. scan (vuls/vuls) +1. vulsrepo (vuls/vulsrepo) -## Start A Vuls Container +## Step1. Fetch NVD -- Execute the following command to build and run a Vuls Container +```console +$ for i in {2002..2016}; do \ + docker run --rm -it \ + -v $PWD:/vuls vuls/go-cve-dictionary fetchnvd -years $i; \ + done +``` - ``` - $ docker-compose up -d - ``` +## Step2. Configuration -## Setting up Vuls +Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). -1. Locate ssh-keys of target servers in (vuls/setup/docker/conf/) -2. Create and adjust config.toml(vuls/setup/docker/conf/config.toml) to your environment - - ``` - [servers] +```toml +[servers] - [servers.172-31-4-82] - host = "172.31.4.82" - user = "ec2-user" - keyPath = "conf/id_rsa" - ``` +[servers.amazon] +host = "54.249.93.16" +port = "22" +user = "vuls-user" +keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker +``` +  + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + vuls/vuls configtest \ + -config=./config.toml # path to config.toml in docker +``` -## Fetch Vulnerability database +## Step3. Prepare -- Fetch Vulnerability database from NVD - ``` - $ docker exec -t vuls scripts/fetch_nvd_all.sh - ``` +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + vuls/vuls prepare \ + -config=./config.toml # path to config.toml in docker +``` -## Scan servers with Vuls-On-Docker +## Step4. Scan + +```console +$ docker run --rm -it \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v /etc/localtime:/etc/localtime:ro \ + -e "TZ=Asia/Tokyo" \ + vuls/vuls scan \ + -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ + -config=./config.toml \ # path to config.toml in docker + -report-json +``` -- Use the embedded script to scan servers for vulsrepo(or run whatever with docker exec) +## Step5. vulsrepo - ``` - $ docker exec -t vuls vuls prepare -config=conf/config.toml - $ docker exec -t vuls scripts/scan_for_vulsrepo.sh - ``` +```console +$docker run -dt \ + -v $PWD:/vuls \ + -p 80:80 \ + vuls/vulsrepo +``` -## See the results in a browser +# User Feedback -``` -http://${Vuls_Host}/vulsrepo/ -``` +## Documentation + +Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). -# Update modules +## Issues -- update vuls, go-cve-dictionary, vulsrepo - ``` - $ docker exec -t vuls scripts/update_modules.sh - ``` +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). -# Update Vulnerability database +## Contributing -- Fetch Vulnerability database from NVD - ``` - $ docker exec -t vuls scripts/fetch_nvd_last2y.sh - ``` +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/base/latest/Dockerfile b/setup/docker/base/latest/Dockerfile new file mode 100644 index 0000000000..43b4aedd7a --- /dev/null +++ b/setup/docker/base/latest/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.7.1 + +MAINTAINER hikachan sadayuki-matsuno + +# glide install +ENV GLIDE_VERSION 0.12.2 +ENV GLIDE_DOWNLOAD_URL https://github.com/Masterminds/glide/releases/download/v$GLIDE_VERSION/glide-v$GLIDE_VERSION-linux-amd64.tar.gz +RUN curl -fsSL "$GLIDE_DOWNLOAD_URL" -o glide.tar.gz \ + && mkdir /usr/local/glide \ + && tar -C /usr/local/glide -xzf glide.tar.gz \ + && ln -s /usr/local/glide/linux-amd64/glide /usr/local/bin/ \ + && rm glide.tar.gz diff --git a/setup/docker/base/latest/README.md b/setup/docker/base/latest/README.md new file mode 100644 index 0000000000..de795289c2 --- /dev/null +++ b/setup/docker/base/latest/README.md @@ -0,0 +1,14 @@ +# Vuls-Base-Docker + +This is the Git repo of the official Docker image for vuls-base. +See the [Hub page](https://hub.docker.com/r/vuls/vuls-base/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`0.1.0`, `latest` (*vuls-base:0.1.0 Dockerfile*)]() + + +# What is vuls-base? + +Base images of vuls and go-cve-dictionary. + diff --git a/setup/docker/conf/.gitkeep b/setup/docker/conf/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/setup/docker/docker-compose.yml b/setup/docker/docker-compose.yml deleted file mode 100644 index cb749d1d40..0000000000 --- a/setup/docker/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '2' -services: - vuls: - container_name: vuls - build: ./dockerfile - image: vuls-docker:0.1 - volumes: - - ./conf:/opt/vuls/conf - ports: - - "80:80" - diff --git a/setup/docker/dockerfile/Dockerfile b/setup/docker/dockerfile/Dockerfile deleted file mode 100644 index 625658eb51..0000000000 --- a/setup/docker/dockerfile/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM buildpack-deps:jessie-scm - -# golang Install -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - gcc \ - libc6-dev \ - make \ - curl \ - && rm -rf /var/lib/apt/lists/* - -ENV GOLANG_VERSION 1.6.2 -ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz -ENV GOLANG_DOWNLOAD_SHA256 e40c36ae71756198478624ed1bb4ce17597b3c19d243f3f0899bb5740d56212a - -RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ - && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ - && tar -C /usr/local -xzf golang.tar.gz \ - && rm golang.tar.gz - -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - -RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" - -# glide install -ENV GLIDE_VERSION 0.10.2 -ENV GLIDE_DOWNLOAD_URL https://github.com/Masterminds/glide/releases/download/$GLIDE_VERSION/glide-$GLIDE_VERSION-linux-amd64.tar.gz -RUN curl -fsSL "$GLIDE_DOWNLOAD_URL" -o glide.tar.gz \ - && mkdir /usr/local/glide \ - && tar -C /usr/local/glide -xzf glide.tar.gz \ - && ln -s /usr/local/glide/linux-amd64/glide /usr/local/bin/ \ - && rm glide.tar.gz - -#Vuls Install -ENV VULS_ROOT /opt/vuls -RUN mkdir -p /var/log/vuls ${VULS_ROOT}/conf /root/.ssh/ -RUN chmod 700 -R /var/log/vuls $VULS_ROOT -# RUN go get github.com/kotakanbe/go-cve-dictionary -# RUN go get github.com/future-architect/vuls - -RUN go get -v -d github.com/kotakanbe/go-cve-dictionary \ - && cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary \ - && glide install \ - && go install - -RUN go get -v -d github.com/future-architect/vuls \ - && cd $GOPATH/src/github.com/future-architect/vuls \ - && glide install \ - && go install - -# Copy custom Scripts -COPY ./scripts/ ${VULS_ROOT}/scripts -RUN chmod 755 ${VULS_ROOT}/scripts/* - - -#Vulrepo Install -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - apache2 \ - libcgi-pm-perl \ - libjson-perl \ - && rm -rf /var/lib/apt/lists/* \ - && cd /var/www/html/ \ - && git clone https://github.com/usiusi360/vulsrepo \ - && mkdir /var/www/html/vulsrepo/results \ - && cp /var/www/html/vulsrepo/dist/cgi/vulsrepo.conf.sample /etc/apache2/conf-enabled/vulsrepo.conf \ - && a2enmod cgid - -#Home -WORKDIR /opt/vuls -EXPOSE 80 443 -ENTRYPOINT service apache2 start && tail -f /dev/null diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_all.sh b/setup/docker/dockerfile/scripts/fetch_jvn_all.sh deleted file mode 100644 index f738a10b45..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -for i in {2002..2016}; do go-cve-dictionary fetchjvn -years $i; done - diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh b/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh deleted file mode 100644 index 62a0d992d9..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchjvn -last2y - diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh b/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh deleted file mode 100644 index 66a5b36f80..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchjvn -latest diff --git a/setup/docker/dockerfile/scripts/fetch_nvd_all.sh b/setup/docker/dockerfile/scripts/fetch_nvd_all.sh deleted file mode 100644 index fb9060d104..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_nvd_all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i; done - diff --git a/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh b/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh deleted file mode 100644 index 57a2955830..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchnvd -last2y - diff --git a/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh b/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh deleted file mode 100644 index c3c43161c2..0000000000 --- a/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -VULS_CONF=${VULS_ROOT}/conf -APACHE_VULSREPO_ROOT=/var/www/html/vulsrepo -cd $VULS_ROOT -vuls scan -report-json --cve-dictionary-dbpath=${VULS_ROOT}/cve.sqlite3 -config=${VULS_CONF}/config.toml -rm ${APACHE_VULSREPO_ROOT}/results/* -cp ${VULS_ROOT}/results/current/* ${APACHE_VULSREPO_ROOT}/results diff --git a/setup/docker/dockerfile/scripts/update_modules.sh b/setup/docker/dockerfile/scripts/update_modules.sh deleted file mode 100644 index 9cd3b493dd..0000000000 --- a/setup/docker/dockerfile/scripts/update_modules.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -cd $GOPATH/src/github.com/future-architect/vuls -git pull origin master -glide install -go install - - -cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -git pull origin master -glide install -go install - - -cd /var/www/html/vulsrepo -git pull origin master diff --git a/setup/docker/go-cve-dictionary/latest/Dockerfile b/setup/docker/go-cve-dictionary/latest/Dockerfile new file mode 100644 index 0000000000..8a464819d0 --- /dev/null +++ b/setup/docker/go-cve-dictionary/latest/Dockerfile @@ -0,0 +1,21 @@ +FROM vuls/vuls-base:latest + +MAINTAINER hikachan sadayuki-matsuno + +ENV REPOSITORY github.com/kotakanbe/go-cve-dictionary +ENV WORKDIR /vuls +# go-cve-dictionary install +RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ + && cd $GOPATH/src/$REPOSITORY \ + && glide install \ + && go build \ + && mv $GOPATH/src/$REPOSITORY/go-cve-dictionary /usr/local/bin \ + && chmod +x /usr/local/bin/go-cve-dictionary \ + && mkdir -p /var/log/vuls + +VOLUME $WORKDIR +WORKDIR $WORKDIR +ENV PWD $WORKDIR + +ENTRYPOINT ["go-cve-dictionary"] +CMD ["--help"] diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md new file mode 100644 index 0000000000..26c0fd4bd2 --- /dev/null +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -0,0 +1,66 @@ +# go-cve-dictionary + +This is the Git repo of the official Docker image for go-cve-dictionary. +See the [Hub page](https://hub.docker.com/r/vuls/go-cve-dictionary/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*go-cve-dictionary:latest Dockerfile*)]() + +This image version is same as the github repository version. + +# What is go-cve-dictionary? + +This is tool to build a local copy of the NVD (National Vulnerabilities Database) [1] and the Japanese JVN [2], which contain security vulnerabilities according to their CVE identifiers [3] including exhaustive information and a risk score. The local copy is generated in sqlite format, and the tool has a server mode for easy querying. + +[1] https://en.wikipedia.org/wiki/National_Vulnerability_Database +[2] https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures +[3] http://jvndb.jvn.jp/apis/termsofuse.html + +# How to use this image + +## fetchnvd + +```console +$ for i in {2002..2016}; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + vuls/go-cve-dictionary fetchnvd -years $i; \ + done +``` + +## server + +```console +$ docker run -dt \ + --name go-cve-dictionary \ + -v $PWD:/vuls \ + --expose 1323 \ + -p 1323:1323 \ + vuls/go-cve-dictionary server --bind=0.0.0.0 +``` + +Prease refer to [this](https://hub.docker.com/r/vuls/go-cve-dictionary). + +## vuls + +Please refer to [this](https://hub.docker.com/r/vuls/vuls/). + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/vuls/latest/Dockerfile b/setup/docker/vuls/latest/Dockerfile new file mode 100644 index 0000000000..7ca2e8e1cf --- /dev/null +++ b/setup/docker/vuls/latest/Dockerfile @@ -0,0 +1,21 @@ +FROM vuls/vuls-base:latest + +MAINTAINER hikachan sadayuki-matsuno + +ENV REPOSITORY github.com/future-architect/vuls +ENV WORKDIR /vuls +# go-cve-dictionary install +RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ + && cd $GOPATH/src/$REPOSITORY \ + && glide install \ + && go build \ + && mv $GOPATH/src/$REPOSITORY/vuls /usr/local/bin \ + && mkdir -p /var/log/vuls \ + && chmod +x /usr/local/bin/vuls + +VOLUME $WORKDIR +WORKDIR $WORKDIR +ENV PWD $WORKDIR + +ENTRYPOINT ["vuls"] +CMD ["--help"] diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md new file mode 100644 index 0000000000..fd3f8b1b45 --- /dev/null +++ b/setup/docker/vuls/latest/README.md @@ -0,0 +1,96 @@ +# Vuls-Docker + +This is the Git repo of the official Docker image for vuls. +See the [Hub page](https://hub.docker.com/r/vuls/vuls/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*vuls:latest Dockerfile*)]() + +This image version is same as the github repository version. + +# What is Vuls? + +Vuls is the Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. +Please see the [Documentation](https://github.com/future-architect/vuls) + +![logo](https://github.com/future-architect/vuls/blob/master/img/vuls_logo.png?raw=true) + +# How to use this image + +## configtest + +Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). + +```toml +[servers] + +[servers.amazon] +host = "54.249.93.16" +port = "22" +user = "vuls-user" +keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker +``` +  + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + vuls/vuls configtest +``` + + +## prepare + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + vuls/vuls prepare \ + -config=./config.toml # path to config.toml in docker +``` + +## scan + +```console +$ docker run --rm -it \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v /etc/localtime:/etc/localtime:ro \ + vuls/vuls scan \ + -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ + -config=./config.toml \ # path to config.toml in docker + -report-json +``` + +## tui + +```console +$ docker run --rm -it \ + -v $PWD:/vuls \ + vuls/vuls tui +``` + +## vulsrepo + +Prease refer to [this](https://hub.docker.com/r/vuls/vulsrepo/). + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/vulsrepo/latest/Dockerfile b/setup/docker/vulsrepo/latest/Dockerfile new file mode 100644 index 0000000000..8410553837 --- /dev/null +++ b/setup/docker/vulsrepo/latest/Dockerfile @@ -0,0 +1,31 @@ +FROM httpd:2.4 + +MAINTAINER hikachan sadayuki-matsuno +# install packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + vim \ + git \ + libcgi-pm-perl \ + libjson-perl \ + && rm -r /var/lib/apt/lists/* + +# env +ENV HTTPD_PREFIX /usr/local/apache2 + +VOLUME /vuls + +WORKDIR ${HTTPD_PREFIX}/htdocs +RUN git clone https://github.com/usiusi360/vulsrepo.git \ + && echo "LoadModule cgid_module modules/mod_cgid.so" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo "" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo " Options +ExecCGI +FollowSymLinks" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo " AddHandler cgi-script cgi" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo "" >> $HTTPD_PREFIX/conf/httpd.conf \ + && sed -i -e 's/User daemon/#User/g' $HTTPD_PREFIX/conf/httpd.conf \ + && sed -i -e 's/Group daemon/#Group/g' $HTTPD_PREFIX/conf/httpd.conf \ + && ln -snf /vuls/results /usr/local/apache2/htdocs/vulsrepo/results + +EXPOSE 80 +CMD ["httpd-foreground"] diff --git a/setup/docker/vulsrepo/latest/README.md b/setup/docker/vulsrepo/latest/README.md new file mode 100644 index 0000000000..bffb42b694 --- /dev/null +++ b/setup/docker/vulsrepo/latest/README.md @@ -0,0 +1,44 @@ +# VulsRepo-Docker + +This is the Git repo of the official Docker image for vulsrepo. +See the [Hub page](https://hub.docker.com/r/vuls/vulsrepo/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*vulsrepo:latest Dockerfile*)]() + +This image version is same as the github repository version. + +# What is vulsrepo? + +VulsRepo is visualized based on the json report output in [vuls](https://github.com/future-architect/vuls). + +# How to use this image + +## vulsrepo + +```console +$docker run -dt \ + -v $PWD:/vuls \ + -p 80:80 \ + vuls/vulsrepo +``` + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request From 85056aaa0095733b7c8ed676478391365157bd64 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Sat, 1 Oct 2016 17:12:58 +0900 Subject: [PATCH 06/37] Update README.md --- setup/docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/docker/README.md b/setup/docker/README.md index 5f63583783..70361cbf3c 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -73,8 +73,8 @@ $ docker run --rm -it \ -e "TZ=Asia/Tokyo" \ vuls/vuls scan \ -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ - -config=./config.toml \ # path to config.toml in docker - -report-json + -report-json \ + -config=./config.toml # path to config.toml in docker ``` ## Step5. vulsrepo From eaddc7f2ba8648f78e8da66c24af0785a089f524 Mon Sep 17 00:00:00 2001 From: Takayuki Ushida Date: Thu, 6 Oct 2016 21:05:57 +0900 Subject: [PATCH 07/37] fix typo --- README.ja.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.ja.md b/README.ja.md index 1f24a11dc9..ecfc7a81f4 100644 --- a/README.ja.md +++ b/README.ja.md @@ -1028,7 +1028,7 @@ $ go-cve-dictionary fetchnvd -last2y - すべての期間の脆弱性情報を取得(10分未満) ``` - $ for i in {1998..2016}; do ./go-cve-dictionary fetchjvn -years $i; done + $ for i in {1998..2016}; do go-cve-dictionary fetchjvn -years $i; done ``` - 2年分の情報を取得 From 4a07974b540c26220b15441eb6e16852ce22ecb6 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 7 Oct 2016 08:26:36 -0400 Subject: [PATCH 08/37] Fixed bug with parsing update line on CentOS/RHEL --- scan/redhat.go | 2 +- scan/redhat_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scan/redhat.go b/scan/redhat.go index 21411cacdb..71d37668ea 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -407,7 +407,7 @@ func (o *redhat) parseYumCheckUpdateLines(stdout string) (results models.Package func (o *redhat) parseYumCheckUpdateLine(line string) (models.PackageInfo, error) { fields := strings.Fields(line) - if len(fields) != 3 { + if len(fields) < 3 { return models.PackageInfo{}, fmt.Errorf("Unknown format: %s", line) } splitted := strings.Split(fields[0], ".") diff --git a/scan/redhat_test.go b/scan/redhat_test.go index 9bd87f2665..60a5df7cd4 100644 --- a/scan/redhat_test.go +++ b/scan/redhat_test.go @@ -616,6 +616,7 @@ Obsoleting Packages python-libs.i686 2.6.6-64.el6 rhui-REGION-rhel-server-releases python-ordereddict.noarch 1.1-3.el6ev installed bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates +pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 ` r.Packages = []models.PackageInfo{ @@ -644,6 +645,11 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates Version: "1.0", Release: "1", }, + { + Name: "pytalloc", + Version: "2.0.1", + Release: "0", + }, } var tests = []struct { in string @@ -687,6 +693,13 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates NewVersion: "9.3.6", NewRelease: "25.P1.el5_11.8", }, + { + Name: "pytalloc", + Version: "2.0.1", + Release: "0", + NewVersion: "2.0.7", + NewRelease: "2.el6", + }, }, }, } From 26364421e8cbe01f028c26e74560bfbf33eda342 Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Sun, 9 Oct 2016 23:45:31 +0900 Subject: [PATCH 09/37] fix README --- setup/docker/base/latest/README.md | 7 ++++++- setup/docker/go-cve-dictionary/latest/README.md | 9 ++++++--- setup/docker/vuls/latest/README.md | 7 +++++-- setup/docker/vulsrepo/latest/README.md | 7 +++++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/setup/docker/base/latest/README.md b/setup/docker/base/latest/README.md index de795289c2..f28d508858 100644 --- a/setup/docker/base/latest/README.md +++ b/setup/docker/base/latest/README.md @@ -5,7 +5,12 @@ See the [Hub page](https://hub.docker.com/r/vuls/vuls-base/) for the full readme # Supported tags and respective `Dockerfile` links -- [`0.1.0`, `latest` (*vuls-base:0.1.0 Dockerfile*)]() +- [`latest` (*vuls-base:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/base/latest/Dockerfile) + +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. # What is vuls-base? diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md index 26c0fd4bd2..f57f79232e 100644 --- a/setup/docker/go-cve-dictionary/latest/README.md +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -1,13 +1,16 @@ -# go-cve-dictionary +# go-cve-dictionary-Docker This is the Git repo of the official Docker image for go-cve-dictionary. See the [Hub page](https://hub.docker.com/r/vuls/go-cve-dictionary/) for the full readme on how to use the Docker image and for information regarding contributing and issues. # Supported tags and respective `Dockerfile` links -- [`latest` (*go-cve-dictionary:latest Dockerfile*)]() +- [`latest` (*go-cve-dictionary:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/go-cve-dictionary/latest/Dockerfile) -This image version is same as the github repository version. +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. # What is go-cve-dictionary? diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md index fd3f8b1b45..c86a0d0925 100644 --- a/setup/docker/vuls/latest/README.md +++ b/setup/docker/vuls/latest/README.md @@ -5,9 +5,12 @@ See the [Hub page](https://hub.docker.com/r/vuls/vuls/) for the full readme on h # Supported tags and respective `Dockerfile` links -- [`latest` (*vuls:latest Dockerfile*)]() +- [`latest` (*vuls:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/vuls/latest/Dockerfile) -This image version is same as the github repository version. +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. # What is Vuls? diff --git a/setup/docker/vulsrepo/latest/README.md b/setup/docker/vulsrepo/latest/README.md index bffb42b694..71846dab0d 100644 --- a/setup/docker/vulsrepo/latest/README.md +++ b/setup/docker/vulsrepo/latest/README.md @@ -5,9 +5,12 @@ See the [Hub page](https://hub.docker.com/r/vuls/vulsrepo/) for the full readme # Supported tags and respective `Dockerfile` links -- [`latest` (*vulsrepo:latest Dockerfile*)]() +- [`latest` (*vulsrepo:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/vulsrepo/latest/Dockerfile) -This image version is same as the github repository version. +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. # What is vulsrepo? From cfc9e064b9853dc84278eea6ca7b39a4eb4c3aeb Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Fri, 5 Aug 2016 16:34:26 +0900 Subject: [PATCH 10/37] change e-mail package from gomail to net/smtp --- README.ja.md | 4 +-- README.md | 4 +-- commands/discover.go | 2 +- glide.lock | 2 -- glide.yaml | 1 - report/mail.go | 59 ++++++++++++++++++++++++++++---------------- 6 files changed, 43 insertions(+), 29 deletions(-) diff --git a/README.ja.md b/README.ja.md index 1f24a11dc9..2e27d1463f 100644 --- a/README.ja.md +++ b/README.ja.md @@ -374,7 +374,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -455,7 +455,7 @@ host = "172.31.4.82" ``` [mail] smtpAddr = "smtp.gmail.com" - smtpPort = "465" + smtpPort = "587" user = "username" password = "password" from = "from@address.com" diff --git a/README.md b/README.md index d1e5819cdc..1140e04bae 100644 --- a/README.md +++ b/README.md @@ -373,7 +373,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -457,7 +457,7 @@ You can customize your configuration using this template. ``` [mail] smtpAddr = "smtp.gmail.com" - smtpPort = "465" + smtpPort = "587" user = "username" password = "password" from = "from@address.com" diff --git a/commands/discover.go b/commands/discover.go index 9e66c55ebc..92d147cd67 100644 --- a/commands/discover.go +++ b/commands/discover.go @@ -100,7 +100,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" diff --git a/glide.lock b/glide.lock index 442bbc2b35..022ec975d9 100644 --- a/glide.lock +++ b/glide.lock @@ -118,6 +118,4 @@ imports: - unix - name: gopkg.in/alexcesaro/quotedprintable.v3 version: 2caba252f4dc53eaf6b553000885530023f54623 -- name: gopkg.in/gomail.v2 - version: 81ebce5c23dfd25c6c67194b37d3dd3f338c98b1 devImports: [] diff --git a/glide.yaml b/glide.yaml index 015240a5dd..afc3e18e9c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -35,4 +35,3 @@ import: - package: golang.org/x/net subpackages: - context -- package: gopkg.in/gomail.v2 diff --git a/report/mail.go b/report/mail.go index 34cd3febde..caa92adc39 100644 --- a/report/mail.go +++ b/report/mail.go @@ -18,13 +18,14 @@ along with this program. If not, see . package report import ( - "crypto/tls" "fmt" - "strconv" + "net" + "net/mail" + "net/smtp" + "strings" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" - "gopkg.in/gomail.v2" ) // MailWriter send mail @@ -33,37 +34,53 @@ type MailWriter struct{} func (w MailWriter) Write(scanResults []models.ScanResult) (err error) { conf := config.Conf for _, s := range scanResults { - m := gomail.NewMessage() - m.SetHeader("From", conf.Mail.From) - m.SetHeader("To", conf.Mail.To...) - m.SetHeader("Cc", conf.Mail.Cc...) + to := strings.Join(conf.Mail.To[:], ", ") + cc := strings.Join(conf.Mail.Cc[:], ", ") + mailAddresses := append(conf.Mail.To, conf.Mail.Cc...) + if _, err := mail.ParseAddressList(strings.Join(mailAddresses[:], ", ")); err != nil { + return fmt.Errorf("Failed to parse mail addresses: %s", err) + } subject := fmt.Sprintf("%s%s %s", conf.Mail.SubjectPrefix, s.ServerInfo(), s.CveSummary(), ) - m.SetHeader("Subject", subject) + + headers := make(map[string]string) + headers["From"] = conf.Mail.From + headers["To"] = to + headers["Cc"] = cc + headers["Subject"] = subject + + var message string + for k, v := range headers { + message += fmt.Sprintf("%s: %s\r\n", k, v) + } var body string if body, err = toPlainText(s); err != nil { return err } - m.SetBody("text/plain", body) - port, _ := strconv.Atoi(conf.Mail.SMTPPort) - d := gomail.NewPlainDialer( - conf.Mail.SMTPAddr, - port, - conf.Mail.User, - conf.Mail.Password, - ) + message += "\r\n" + body - d.TLSConfig = &tls.Config{ - InsecureSkipVerify: true, - } + smtpServer := net.JoinHostPort(conf.Mail.SMTPAddr, conf.Mail.SMTPPort) + + err := smtp.SendMail( + smtpServer, + smtp.PlainAuth( + "", + conf.Mail.User, + conf.Mail.Password, + conf.Mail.SMTPAddr, + ), + conf.Mail.From, + conf.Mail.To, + []byte(message), + ) - if err := d.DialAndSend(m); err != nil { - panic(err) + if err != nil { + return fmt.Errorf("Failed to send mails: %s", err) } } return nil From 79523de1dbb75350c9dad1fafd45021eae2d7358 Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Tue, 11 Oct 2016 12:21:43 +0900 Subject: [PATCH 11/37] fix readme --- README.ja.md | 4 ++-- README.md | 4 ++-- report/mail.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.ja.md b/README.ja.md index 79a1a004cb..c8c0921f8c 100644 --- a/README.ja.md +++ b/README.ja.md @@ -121,9 +121,9 @@ VulsはSSHパスワード認証をサポートしていない。SSH公開鍵鍵 Vulsセットアップに必要な以下のソフトウェアをインストールする。 - SQLite3 -- git v2 +- git - gcc -- go v1.6 +- go v1.6 or later - https://golang.org/doc/install ```bash diff --git a/README.md b/README.md index 1140e04bae..756b675d87 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,9 @@ And also, SUDO with password is not supported for security reasons. So you have Vuls requires the following packages. - SQLite3 -- git v2 +- git - gcc -- go v1.6 +- go v1.6 or later - https://golang.org/doc/install ```bash diff --git a/report/mail.go b/report/mail.go index caa92adc39..21803874be 100644 --- a/report/mail.go +++ b/report/mail.go @@ -38,7 +38,7 @@ func (w MailWriter) Write(scanResults []models.ScanResult) (err error) { cc := strings.Join(conf.Mail.Cc[:], ", ") mailAddresses := append(conf.Mail.To, conf.Mail.Cc...) if _, err := mail.ParseAddressList(strings.Join(mailAddresses[:], ", ")); err != nil { - return fmt.Errorf("Failed to parse mail addresses: %s", err) + return fmt.Errorf("Failed to parse email addresses: %s", err) } subject := fmt.Sprintf("%s%s %s", @@ -80,7 +80,7 @@ func (w MailWriter) Write(scanResults []models.ScanResult) (err error) { ) if err != nil { - return fmt.Errorf("Failed to send mails: %s", err) + return fmt.Errorf("Failed to send emails: %s", err) } } return nil From 239d910dbebb7d8f4bc212dbed128106acdc1cb5 Mon Sep 17 00:00:00 2001 From: Sho Kohara Date: Tue, 11 Oct 2016 13:09:45 +0900 Subject: [PATCH 12/37] Fix ja document about typo --- README.ja.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.ja.md b/README.ja.md index c8c0921f8c..067678656a 100644 --- a/README.ja.md +++ b/README.ja.md @@ -562,7 +562,7 @@ vuls ALL=(root) NOPASSWD: /usr/bin/apt-get, /usr/bin/apt-cache # Usage: Prepare -Prepareサブコマンドは、Vuls内部で利用する以下のパッケージをスキャン対象サーバにインストーする。 +Prepareサブコマンドは、Vuls内部で利用する以下のパッケージをスキャン対象サーバにインストールする。 | Distribution| Release | Requirements | |:------------|-------------------:|:-------------| From fc2ea48c1d147eb131bfe997eba9b0ac3b0e455d Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Tue, 11 Oct 2016 19:43:50 +0900 Subject: [PATCH 13/37] fix docker readme --- setup/docker/go-cve-dictionary/latest/README.md | 2 +- setup/docker/vuls/latest/README.md | 2 +- setup/docker/vulsrepo/latest/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md index f57f79232e..60750f4cea 100644 --- a/setup/docker/go-cve-dictionary/latest/README.md +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -53,7 +53,7 @@ Please refer to [this](https://hub.docker.com/r/vuls/vuls/). ## Documentation -Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). ## Issues diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md index c86a0d0925..3210b54a27 100644 --- a/setup/docker/vuls/latest/README.md +++ b/setup/docker/vuls/latest/README.md @@ -83,7 +83,7 @@ Prease refer to [this](https://hub.docker.com/r/vuls/vulsrepo/). ## Documentation -Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). ## Issues diff --git a/setup/docker/vulsrepo/latest/README.md b/setup/docker/vulsrepo/latest/README.md index 71846dab0d..7b5ff687b6 100644 --- a/setup/docker/vulsrepo/latest/README.md +++ b/setup/docker/vulsrepo/latest/README.md @@ -31,7 +31,7 @@ $docker run -dt \ ## Documentation -Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). ## Issues From cf7990d444134d2774f602e5aec2256c065aba08 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 12 Oct 2016 08:59:05 +0900 Subject: [PATCH 14/37] Fix locale env var LANG to LANGUAGE --- scan/debian.go | 4 ++-- scan/redhat.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scan/debian.go b/scan/debian.go index e4ef25f284..5e88982966 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -301,7 +301,7 @@ func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []m for _, p := range before { names = append(names, p.Name) } - cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) + cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) r := o.ssh(cmd, sudo) if !r.isSuccess() { return nil, fmt.Errorf("Failed to SSH: %s.", r) @@ -323,7 +323,7 @@ func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []m } func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { - cmd := util.PrependProxyEnv("LANG=en_US.UTF-8 apt-get upgrade --dry-run") + cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get upgrade --dry-run") r := o.ssh(cmd, sudo) if r.isSuccess(0, 1) { return o.parseAptGetUpgrade(r.Stdout) diff --git a/scan/redhat.go b/scan/redhat.go index 71d37668ea..8a777ffebd 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -251,7 +251,7 @@ func (o *redhat) scanUnsecurePackages() ([]CvePacksInfo, error) { // For CentOS func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (CvePacksList, error) { - cmd := "LANG=en_US.UTF-8 yum --color=never check-update" + cmd := "LANGUAGE=en_US.UTF-8 yum --color=never check-update" r := o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. @@ -547,7 +547,7 @@ func (o *redhat) getAllChangelog(packInfoList models.PackageInfoList) (stdout st } // yum update --changelog doesn't have --color option. - command += fmt.Sprintf(" LANG=en_US.UTF-8 yum update --changelog %s", packageNames) + command += fmt.Sprintf(" LANGUAGE=en_US.UTF-8 yum update --changelog %s", packageNames) r := o.ssh(command, sudo) if !r.isSuccess(0, 1) { @@ -589,7 +589,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, err // get package name, version, rel to be upgrade. // cmd = "yum check-update --security" - cmd = "LANG=en_US.UTF-8 yum --color=never check-update" + cmd = "LANGUAGE=en_US.UTF-8 yum --color=never check-update" r = o.ssh(util.PrependProxyEnv(cmd), o.sudo()) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. From b5cf06cad834787fa225f2d2e20a93a17befb5a2 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 12 Oct 2016 16:51:17 +0900 Subject: [PATCH 15/37] Add GitHub issue template --- ISSUE_TEMPLATE | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ISSUE_TEMPLATE diff --git a/ISSUE_TEMPLATE b/ISSUE_TEMPLATE new file mode 100644 index 0000000000..51f9b478f7 --- /dev/null +++ b/ISSUE_TEMPLATE @@ -0,0 +1,36 @@ + +# Environment + +## Vuls + +Hash : ____ + +To check the commit hash of HEAD +$ vuls -v + +or +$ cd $GOPATH/src/github.com/future-architect/vuls +$ git rev-parse --short HEAD + +## OS +- Target Server: Write here +- Vuls Server: Write here + +## Go +- Go version: here + +# Current Output + +Please re-run the command using ```-debug``` and provide the output below. + +# Addition Details + +Can you also please fill in each of the remaining sections. + +## Expected Behavior + +## Actual Behavior + +## Steps to reproduce the behaviour + + From 5b80b166842cd8a6323c9b4debcf346695a44d29 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 12 Oct 2016 19:57:47 +0900 Subject: [PATCH 16/37] Remove all.json --- commands/history.go | 4 ---- report/json.go | 12 ------------ report/s3.go | 10 ---------- 3 files changed, 26 deletions(-) diff --git a/commands/history.go b/commands/history.go index 4d5644a81f..eccd55d029 100644 --- a/commands/history.go +++ b/commands/history.go @@ -81,10 +81,6 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } var hosts []string for _, f := range files { - // TODO this "if block" will be deleted in a future release - if f.Name() == "all.json" { - continue - } if filepath.Ext(f.Name()) != ".json" { continue } diff --git a/report/json.go b/report/json.go index 4781187453..acff8c8921 100644 --- a/report/json.go +++ b/report/json.go @@ -61,14 +61,6 @@ func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) { } var jsonBytes []byte - if jsonBytes, err = json.Marshal(scanResults); err != nil { - return fmt.Errorf("Failed to Marshal to JSON: %s", err) - } - all := filepath.Join(path, "all.json") - if err := ioutil.WriteFile(all, jsonBytes, 0600); err != nil { - return fmt.Errorf("Failed to write JSON. path: %s, err: %s", all, err) - } - for _, r := range scanResults { jsonPath := "" if len(r.Container.ContainerID) == 0 { @@ -117,10 +109,6 @@ func LoadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err err return } for _, file := range files { - // TODO this "if block" will be deleted in a future release - if file.Name() == "all.json" { - continue - } if filepath.Ext(file.Name()) != ".json" { continue } diff --git a/report/s3.go b/report/s3.go index 95b7951ea8..f55484101a 100644 --- a/report/s3.go +++ b/report/s3.go @@ -78,16 +78,6 @@ func (w S3Writer) Write(scanResults []models.ScanResult) (err error) { // http://docs.aws.amazon.com/sdk-for-go/latest/v1/developerguide/common-examples.title.html svc := getS3() timestr := time.Now().Format("20060102_1504") - key := fmt.Sprintf("%s/%s", timestr, "all.json") - _, err = svc.PutObject(&s3.PutObjectInput{ - Bucket: &c.Conf.S3Bucket, - Key: &key, - Body: bytes.NewReader(jsonBytes), - }) - if err != nil { - return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err) - } - for _, r := range scanResults { key := "" if len(r.Container.ContainerID) == 0 { From 95d6888c8725d481e2891ea70242ef99529aefc1 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Fri, 7 Oct 2016 01:52:23 +0900 Subject: [PATCH 17/37] Improve makefile, -version shows git hash, fix README --- Makefile | 28 +++++++++++++++++++---- README.ja.md | 57 ++++++++++++---------------------------------- README.md | 57 +++++++++++++--------------------------------- glide.lock | 38 +++++++++++++++---------------- main.go | 9 ++++++-- version/version.go | 24 ------------------- 6 files changed, 79 insertions(+), 134 deletions(-) delete mode 100644 version/version.go diff --git a/Makefile b/Makefile index 7b6d761df5..f47ffa49c3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,9 @@ .PHONY: \ + glide \ + deps \ + update \ + build \ + install \ all \ vendor \ lint \ @@ -12,12 +17,27 @@ SRCS = $(shell git ls-files '*.go') PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache +VERSION := $(shell git describe --tags --abbrev=0) +REVISION := $(shell git rev-parse --short HEAD) +LDFLAGS := -X 'main.version=$(VERSION)' \ + -X 'main.revision=$(REVISION)' -all: test +glide: + go get github.com/Masterminds/glide + +deps: glide + glide install + +update: glide + glide update -# vendor: -# @ go get -v github.com/mjibson/party -# party -d external -c -u +build: main.go deps + go build -ldflags "$(LDFLAGS)" -o vuls $< + +install: main.go deps + go install -ldflags "$(LDFLAGS)" + +all: test lint: @ go get -v github.com/golang/lint/golint diff --git a/README.ja.md b/README.ja.md index 067678656a..838bb1ae34 100644 --- a/README.ja.md +++ b/README.ja.md @@ -148,26 +148,19 @@ $ source /etc/profile.d/goenv.sh ## Step4. Deploy [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -go get - ```bash $ sudo mkdir /var/log/vuls $ sudo chown ec2-user /var/log/vuls $ sudo chmod 700 /var/log/vuls -$ go get github.com/kotakanbe/go-cve-dictionary -``` - -go-cve-dictionaryを既にインストール済みでupdateしたい場合は - -```bash -$ go get -u github.com/kotakanbe/go-cve-dictionary +$ +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git https://github.com/kotakanbe/go-cve-dictionary.git +$ cd go-cve-dictionary +$ make install ``` +バイナリは、`$GOPATH/bin`いかに生成される -で可能である。 - -go getでエラーが発生した場合は、以下の点を確認する。 -- Gitのバージョンがv2以降か? -- Go依存パッケージの問題でgo getに失敗する場合は [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide) を試す。 NVDから脆弱性データベースを取得する。 環境によって異なるが、AWS上では10分程度かかる。 @@ -182,10 +175,12 @@ $ ls -alh cve.sqlite3 ## Step5. Deploy Vuls 新規にターミナルを起動し、先ほど作成したEC2にSSH接続する。 - -go get ``` -$ go get github.com/future-architect/vuls +$ mkdir -p $GOPATH/src/github.com/future-architect +$ cd $GOPATH/src/github.com/future-architect +$ git clone https://github.com/future-architect/vuls.git +$ cd vuls +$ make install ``` vulsを既にインストール済みでupdateしたい場合は @@ -1074,28 +1069,6 @@ slack, emailは日本語対応済み TUIは日本語表示未対応 ---- -# Deploy With Glide - -If an error occurred while go get, try deploying with glide. -- Install [Glide](https://github.com/Masterminds/glide) -- Deploy go-cve-dictionary -``` -$ go get -d github.com/kotakanbe/go-cve-dictionary -$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -$ glide install -$ go install -``` -- Deploy vuls -``` -$ go get -d github.com/future-architect/vuls -$ cd $GOPATH/src/github.com/future-architect/vuls -$ glide install -$ go install -``` -- The binaries are created under $GOPARH/bin - ----- - # Update Vuls With Glide - Update go-cve-dictionary @@ -1103,16 +1076,14 @@ If the DB schema was changed, please specify new SQLite3 DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull -$ glide install -$ go install +$ make install ``` - Update vuls ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull -$ glide install -$ go install +$ make install ``` - バイナリファイルは`$GOPARH/bin`以下に作成される diff --git a/README.md b/README.md index 756b675d87..7aba0e2d4e 100644 --- a/README.md +++ b/README.md @@ -156,18 +156,18 @@ $ source /etc/profile.d/goenv.sh ## Step4. Deploy [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -go get - ```bash $ sudo mkdir /var/log/vuls $ sudo chown ec2-user /var/log/vuls $ sudo chmod 700 /var/log/vuls -$ go get github.com/kotakanbe/go-cve-dictionary +$ +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git clone https://github.com/kotakanbe/go-cve-dictionary.git +$ cd go-cve-dictionary +$ make install ``` - -If an error occurred while go get, check the following points. -- Update Git -- try [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide). +The binary was built under `$GOPARH/bin` Fetch vulnerability data from NVD. It takes about 10 minutes (on AWS). @@ -183,14 +183,14 @@ $ ls -alh cve.sqlite3 Launch a new terminal and SSH to the ec2 instance. -go get ``` -$ go get github.com/future-architect/vuls +$ mkdir -p $GOPATH/src/github.com/future-architect +$ cd $GOPATH/src/github.com/future-architect +$ git clone https://github.com/future-architect/vuls.git +$ cd vuls +$ make install ``` - -If an error occurred while go get, check the following points. -- Update Git -- try [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide). +The binary was built under `$GOPARH/bin` ## Step6. Config @@ -987,28 +987,6 @@ $ go-cve-dictionary fetchnvd -last2y ---- -# Deploy With Glide - -If an error occurred while go get, try deploying with glide. -- Install [Glide](https://github.com/Masterminds/glide) -- Deploy go-cve-dictionary -``` -$ go get -d github.com/kotakanbe/go-cve-dictionary -$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -$ glide install -$ go install -``` -- Deploy vuls -``` -$ go get -d github.com/future-architect/vuls -$ cd $GOPATH/src/github.com/future-architect/vuls -$ glide install -$ go install -``` -- The binaries are created under $GOPARH/bin - ----- - # Update Vuls With Glide - Update go-cve-dictionary @@ -1016,19 +994,16 @@ If the DB schema was changed, please specify new SQLite3 DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull -$ glide install -$ go install +$ make install ``` - Update vuls ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull -$ glide install -$ go install +$ make install ``` - -Binary Files are created under $GOPARH/bin +Binary file was built under $GOPARH/bin --- diff --git a/glide.lock b/glide.lock index 022ec975d9..a74c30ecd1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,10 +1,10 @@ -hash: 28d14f88e90c0765c1b660ddde796e51e197239d353bb79bfc5d8f8cf9b5f9ee -updated: 2016-09-20T10:05:42.83736026+09:00 +hash: 01cee4aa1d27c967f5f5165febfbd77fd0e89a6e7a9a89c84099c0d45dffd446 +updated: 2016-10-12T15:28:25.843936367+09:00 imports: - name: github.com/asaskevich/govalidator - version: 593d64559f7600f29581a3ee42177f5dbded27a9 + version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 - name: github.com/aws/aws-sdk-go - version: bc572378d109481c50d45d9dba4490d80386e98e + version: aad29423c2c09f0a1fa5346216bc041a4994eef4 subpackages: - aws - aws/credentials @@ -32,11 +32,11 @@ imports: - private/protocol/xml/xmlutil - private/protocol/query/queryutil - name: github.com/Azure/azure-sdk-for-go - version: 34467930a15f0d2872168deb11435b8ac3d863bb + version: 91f3d4a4d024e3c0d4d9412916d05cf84504a616 subpackages: - storage - name: github.com/boltdb/bolt - version: fff57c100f4dea1905678da7e90d92429dff2904 + version: 074dffcc83e9f421e261526d297cd93f22a34080 - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/cenkalti/backoff @@ -53,15 +53,15 @@ imports: - util/strutil - util/wordwrap - name: github.com/howeyc/gopass - version: 3ca23474a7c7203e0a0a070fd33508f6efdb9b3d + version: f5387c492211eb133053880d23dfae62aa14123d - name: github.com/jinzhu/gorm - version: 02f6ae3c4ed211472b0492cee02ff3ddfdc1830d + version: 39165d498058a823126af3cbf4d2a3b0e1acf11e - name: github.com/jinzhu/inflection version: 74387dc39a75e970e7a3ae6a3386b5bd2e5c5cff - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/jroimartin/gocui - version: 30f7d65597dc2c421ce452b164c36b7014ef94be + version: 4e9ce9a8e26f2ef33dfe297dbdfca148733b6b9b - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary @@ -79,27 +79,27 @@ imports: - name: github.com/kotakanbe/logrus-prefixed-formatter version: f4f7d41649cf1e75e736884da8d05324aa76ea25 - name: github.com/mattn/go-colorable - version: ed8eb9e318d7a84ce5915b495b7d35e0cfe7b5a8 + version: 6c903ff4aa50920ca86087a280590b36b3152b9c - name: github.com/mattn/go-isatty version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 - name: github.com/mattn/go-runewidth - version: d6bea18f789704b5f83375793155289da36a3c7f + version: 737072b4e32b7a5018b4a7125da8d12de90e8045 - name: github.com/mattn/go-sqlite3 - version: 3fb7a0e792edd47bf0cf1e919dfc14e2be412e15 + version: e5a3c16c5c1d80b24f633e68aecd6b0702786d3d - name: github.com/mgutz/ansi version: c286dcecd19ff979eeb73ea444e479b903f2cfcb - name: github.com/moul/http2curl version: b1479103caacaa39319f75e7f57fc545287fca0d - name: github.com/nsf/termbox-go - version: e8f6d27f72a2f2bb598eb3579afd5ea364ef67f7 + version: b6acae516ace002cb8105a89024544a1480655a5 - name: github.com/parnurzeal/gorequest - version: 2aea80ce763523ecc6452e61c3727ae9595a5809 + version: e37b9d1efacf7c94820b29b75dd7d0c2996b3fb1 - name: github.com/rifflock/lfshook - version: f9d14dda07b109a7aa56f135c31b34062eb14392 + version: 3f9d976bd7402de39b46357069fb6325a974572e - name: github.com/Sirupsen/logrus version: 3ec0642a7fb6488f65b06f9040adc67e3990296a - name: golang.org/x/crypto - version: 9e590154d2353f3f5e1b24da7275686040dcf491 + version: 4cd25d65a015cc83d41bf3454e6e8d6c116d16da subpackages: - ssh - ssh/agent @@ -108,14 +108,12 @@ imports: - ed25519 - ed25519/internal/edwards25519 - name: golang.org/x/net - version: 9313baa13d9262e49d07b20ed57dceafcd7240cc + version: cf4effbb9db1f3ef07f7e1891402991b6afbb276 subpackages: - context - publicsuffix - name: golang.org/x/sys - version: 30de6d19a3bd89a5f38ae4028e23aaa5582648af + version: 9bb9f0998d48b31547d975974935ae9b48c7a03c subpackages: - unix -- name: gopkg.in/alexcesaro/quotedprintable.v3 - version: 2caba252f4dc53eaf6b553000885530023f54623 devImports: [] diff --git a/main.go b/main.go index 35acb99b0e..92ee5bf22d 100644 --- a/main.go +++ b/main.go @@ -25,12 +25,17 @@ import ( "golang.org/x/net/context" "github.com/future-architect/vuls/commands" - "github.com/future-architect/vuls/version" "github.com/google/subcommands" _ "github.com/mattn/go-sqlite3" ) +// Version of Vuls +var version = "0.1.6" + +// Revision of Git +var revision string + func main() { subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.FlagsCommand(), "") @@ -47,7 +52,7 @@ func main() { flag.Parse() if *v { - fmt.Printf("%s %s\n", version.Name, version.Version) + fmt.Printf("vuls %s %s\n", version, revision) os.Exit(int(subcommands.ExitSuccess)) } diff --git a/version/version.go b/version/version.go deleted file mode 100644 index 3ba04ed2be..0000000000 --- a/version/version.go +++ /dev/null @@ -1,24 +0,0 @@ -/* Vuls - Vulnerability Scanner -Copyright (C) 2016 Future Architect, Inc. Japan. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package version - -// Name is Vuls -const Name string = "vuls" - -// Version of Vuls -const Version string = "0.1.6" From b42805d00c36ef07d3059e6f67c5e5373a611e8b Mon Sep 17 00:00:00 2001 From: Yasunari Momoi Date: Wed, 12 Oct 2016 23:11:34 +0900 Subject: [PATCH 18/37] fix some misspelling. --- README.fr.md | 2 +- README.ja.md | 12 ++++++------ README.md | 28 ++++++++++++++-------------- commands/prepare.go | 4 ++-- commands/scan.go | 4 ++-- scan/freebsd_test.go | 8 ++++---- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.fr.md b/README.fr.md index e8ae519fd8..d8fac6a7f6 100644 --- a/README.fr.md +++ b/README.fr.md @@ -201,7 +201,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 diff --git a/README.ja.md b/README.ja.md index 067678656a..5dd373fdba 100644 --- a/README.ja.md +++ b/README.ja.md @@ -255,7 +255,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 @@ -427,7 +427,7 @@ host = "172.31.4.82" notifyUsers = ["@username"] ``` - - hookURL : Incomming webhook's URL + - hookURL : Incoming webhook's URL - channel : channel name. channelに`${servername}`を指定すると、結果レポートをサーバごとに別チャネルにすることが出来る。 以下のサンプルでは、`#server1`チャネルと`#server2`チャネルに送信される。スキャン前にチャネルを作成する必要がある。 @@ -908,14 +908,14 @@ tui: ``` -Key binding is bellow. +Key binding is below. | key | | |:-----------------|:-------|:------| | TAB | move cursor among the panes | | Arrow up/down | move cursor to up/down | -| Ctrl+j, Ctrl+k | move cursor to up/donw | -| Ctrl+u, Ctrl+d | page up/donw | +| Ctrl+j, Ctrl+k | move cursor to up/down | +| Ctrl+u, Ctrl+d | page up/down | For details, see https://github.com/future-architect/vuls/blob/master/report/tui.go @@ -1209,7 +1209,7 @@ Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHAN ---- -# Licence +# License Please see [LICENSE](https://github.com/future-architect/vuls/blob/master/LICENSE). diff --git a/README.md b/README.md index 756b675d87..71422e0baf 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 @@ -426,7 +426,7 @@ You can customize your configuration using this template. notifyUsers = ["@username"] ``` - - hookURL : Incomming webhook's URL + - hookURL : Incoming webhook's URL - channel : channel name. If you set `${servername}` to channel, the report will be sent to each channel. In the following example, the report will be sent to the `#server1` and `#server2`. @@ -683,11 +683,11 @@ scan: Vuls supports different types of SSH. -By Defaut, using a native Go implementation from crypto/ssh. +By Default, using a native Go implementation from crypto/ssh. This is useful in situations where you may not have access to traditional UNIX tools. To use external ssh command, specify this option. -This is useful If you want to use ProxyCommand or chiper algorithm of SSH that is not supported by native go implementation. +This is useful If you want to use ProxyCommand or cipher algorithm of SSH that is not supported by native go implementation. Don't forget to add below line to /etc/sudoers on the target servers. (username: vuls) ``` Defaults:vuls !requiretty @@ -704,7 +704,7 @@ Defaults:vuls !requiretty ## -report-json , -report-text option At the end of the scan, scan results will be available in the `$PWD/result/current/` directory. -`all.(json|txt)` includes the scan results of all servres and `servername.(json|txt)` includes the scan result of the server. +`all.(json|txt)` includes the scan results of all servers and `servername.(json|txt)` includes the scan result of the server. ## Example: Scan all servers defined in config file ``` @@ -716,7 +716,7 @@ $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 ``` With this sample command, it will .. -- Ask SSH key passsword before scanning +- Ask SSH key password before scanning - Scan all servers defined in config file - Send scan results to slack and email - Only Report CVEs that CVSS score is over 7 @@ -831,10 +831,10 @@ It is possible to detect vulnerabilities in non-OS packages, such as something y **Check CPE Naming Format: 2.2** - [go-cpe-dictionary](https://github.com/kotakanbe/go-cpe-dictionary) is a good choice for geeks. - You can search a CPE name by the application name incremenally. + You can search a CPE name by the application name incrementally. - Configuration -To detect the vulnerbility of Ruby on Rails v4.2.1, cpeNames needs to be set in the servers section. +To detect the vulnerability of Ruby on Rails v4.2.1, cpeNames needs to be set in the servers section. ``` [servers] @@ -849,7 +849,7 @@ To detect the vulnerbility of Ruby on Rails v4.2.1, cpeNames needs to be set in # Usage: Scan Docker containers -It is common that keep Docker containers runnning without SSHd daemon. +It is common that keep Docker containers running without SSHd daemon. see [Docker Blog:Why you don't need to run SSHd in your Docker containers](https://blog.docker.com/2014/06/why-you-dont-need-to-run-sshd-in-docker/) Vuls scans Docker containers via `docker exec` instead of SSH. @@ -901,14 +901,14 @@ tui: ``` -Key binding is bellow. +Key binding is below. | key | | |:-----------------|:-------|:------| | TAB | move cursor among the panes | | Arrow up/down | move cursor to up/down | -| Ctrl+j, Ctrl+k | move cursor to up/donw | -| Ctrl+u, Ctrl+d | page up/donw | +| Ctrl+j, Ctrl+k | move cursor to up/down | +| Ctrl+u, Ctrl+d | page up/down | For details, see https://github.com/future-architect/vuls/blob/master/report/tui.go @@ -1044,7 +1044,7 @@ If your system is behind HTTP proxy, you have to specify --http-proxy option. - How to Daemonize go-cve-dictionary Use Systemd, Upstart or supervisord, daemontools... -- How to Enable Automatic-Update of Vunerability Data. +- How to Enable Automatic-Update of Vulnerability Data. Use job scheduler like Cron (with -last2y option). - How to Enable Automatic-Scan. @@ -1119,7 +1119,7 @@ Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHAN ---- -# Licence +# License Please see [LICENSE](https://github.com/future-architect/vuls/blob/master/LICENSE). diff --git a/commands/prepare.go b/commands/prepare.go index 5e24719f15..dc71624b1b 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -86,7 +86,7 @@ func (p *PrepareCmd) SetFlags(f *flag.FlagSet) { &p.askSudoPassword, "ask-sudo-password", false, - "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASON. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication", + "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication", ) } @@ -102,7 +102,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } } if p.askSudoPassword { - logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication") + logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication") return subcommands.ExitFailure } diff --git a/commands/scan.go b/commands/scan.go index da232db048..1e1df7b746 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -225,7 +225,7 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { &p.askSudoPassword, "ask-sudo-password", false, - "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication", + "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication", ) } @@ -241,7 +241,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } } if p.askSudoPassword { - logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication") + logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication") return subcommands.ExitFailure } diff --git a/scan/freebsd_test.go b/scan/freebsd_test.go index 2351f2b299..48dd62b1a2 100644 --- a/scan/freebsd_test.go +++ b/scan/freebsd_test.go @@ -139,17 +139,17 @@ WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html d := newBsd(config.ServerInfo{}) for _, tt := range tests { - aName, aCveIDs, aVunlnID := d.parseBlock(tt.in) + aName, aCveIDs, aVulnID := d.parseBlock(tt.in) if tt.name != aName { - t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVulnID) } for i := range tt.cveIDs { if tt.cveIDs[i] != aCveIDs[i] { t.Errorf("expected cveID: %s, actual %s", tt.cveIDs[i], aCveIDs[i]) } } - if tt.vulnID != aVunlnID { - t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + if tt.vulnID != aVulnID { + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVulnID) } } } From 21beb396b4f1ed58ade79a5187b8749c7832fd36 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 12 Oct 2016 20:13:37 +0900 Subject: [PATCH 19/37] Confirm before installing dependencies on prepare --- commands/prepare.go | 3 +-- scan/base.go | 7 ++++- scan/debian.go | 31 ++++++++++++++++++--- scan/freebsd.go | 4 +++ scan/redhat.go | 37 +++++++++++++------------- scan/serverapi.go | 65 ++++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 119 insertions(+), 28 deletions(-) diff --git a/commands/prepare.go b/commands/prepare.go index dc71624b1b..8f34993389 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -149,10 +149,9 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ return subcommands.ExitFailure } - logger.Info("Installing...") if errs := scan.Prepare(); 0 < len(errs) { for _, e := range errs { - logger.Errorf("Failed: %s", e) + logger.Errorf("Failed to prepare: %s", e) } return subcommands.ExitFailure } diff --git a/scan/base.go b/scan/base.go index 8353081a60..3a39d6c70a 100644 --- a/scan/base.go +++ b/scan/base.go @@ -33,8 +33,9 @@ import ( type base struct { ServerInfo config.ServerInfo Distro config.Distro + Platform models.Platform - Platform models.Platform + lackDependencies []string osPackages log *logrus.Entry @@ -77,6 +78,10 @@ func (l base) getPlatform() models.Platform { return l.Platform } +func (l base) getLackDependencies() []string { + return l.lackDependencies +} + func (l base) allContainers() (containers []config.Container, err error) { switch l.ServerInfo.Container.Type { case "", "docker": diff --git a/scan/debian.go b/scan/debian.go index 5e88982966..65a21d19f7 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -124,7 +124,31 @@ func (o *debian) checkIfSudoNoPasswd() error { return nil } +func (o *debian) checkDependencies() error { + switch o.Distro.Family { + case "ubuntu": + return nil + + case "debian": + // Debian needs aptitude to get changelogs. + // Because unable to get changelogs via apt-get changelog on Debian. + name := "aptitude" + cmd := name + " -h" + if r := o.ssh(cmd, noSudo); !r.isSuccess() { + o.lackDependencies = []string{name} + } + return nil + + default: + return fmt.Errorf("Not implemented yet: %s", o.Distro) + } +} + func (o *debian) install() error { + if len(o.lackDependencies) == 0 { + return nil + } + // apt-get update o.log.Infof("apt-get update...") cmd := util.PrependProxyEnv("apt-get update") @@ -134,15 +158,14 @@ func (o *debian) install() error { return fmt.Errorf(msg) } - if o.Distro.Family == "debian" { - // install aptitude - cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude") + for _, name := range o.lackDependencies { + cmd = util.PrependProxyEnv("apt-get install " + name) if r := o.ssh(cmd, sudo); !r.isSuccess() { msg := fmt.Sprintf("Failed to SSH: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } - o.log.Infof("Installed: aptitude") + o.log.Infof("Installed: " + name) } return nil } diff --git a/scan/freebsd.go b/scan/freebsd.go index 28215bf385..2d904f17ed 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -66,6 +66,10 @@ func (o *bsd) checkIfSudoNoPasswd() error { return nil } +func (o *bsd) checkDependencies() error { + return nil +} + func (o *bsd) install() error { return nil } diff --git a/scan/redhat.go b/scan/redhat.go index 8a777ffebd..6eef39cfce 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -112,45 +112,46 @@ func (o *redhat) checkIfSudoNoPasswd() error { // CentOS 6 ... yum-plugin-changelog // CentOS 7 ... yum-plugin-changelog // RHEL, Amazon ... no additinal packages needed -func (o *redhat) install() error { +func (o *redhat) checkDependencies() error { switch o.Distro.Family { case "rhel", "amazon": - o.log.Infof("Nothing to do") + // o.log.Infof("Nothing to do") return nil - } - // CentOS - return o.installYumChangelog() -} -func (o *redhat) installYumChangelog() error { - if o.Distro.Family == "centos" { + case "centos": var majorVersion int if 0 < len(o.Distro.Release) { majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0]) } else { - return fmt.Errorf( - "Not implemented yet: %s", o.Distro) + return fmt.Errorf("Not implemented yet: %s", o.Distro) } - var packName = "" + var name = "" if majorVersion < 6 { - packName = "yum-changelog" + name = "yum-changelog" } else { - packName = "yum-plugin-changelog" + name = "yum-plugin-changelog" } - cmd := "rpm -q " + packName + cmd := "rpm -q " + name if r := o.ssh(cmd, noSudo); r.isSuccess() { - o.log.Infof("Ignored: %s already installed", packName) return nil } + o.lackDependencies = []string{name} + return nil - o.log.Infof("Installing %s...", packName) - cmd = util.PrependProxyEnv("yum install -y " + packName) + default: + return fmt.Errorf("Not implemented yet: %s", o.Distro) + } +} + +func (o *redhat) install() error { + for _, name := range o.lackDependencies { + cmd := util.PrependProxyEnv("yum install -y " + name) if r := o.ssh(cmd, sudo); !r.isSuccess() { return fmt.Errorf("Failed to SSH: %s", r) } - o.log.Infof("Installed: %s", packName) + o.log.Infof("Installed: %s", name) } return nil } diff --git a/scan/serverapi.go b/scan/serverapi.go index 2e7b87aefc..34db2506d5 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -18,7 +18,10 @@ along with this program. If not, see . package scan import ( + "bufio" "fmt" + "os" + "strings" "time" "github.com/Sirupsen/logrus" @@ -40,7 +43,10 @@ type osTypeInterface interface { setDistro(string, string) getDistro() config.Distro - // getFamily() string + + // checkDependencies checks if dependencies are installed on the target server. + checkDependencies() error + getLackDependencies() []string checkIfSudoNoPasswd() error detectPlatform() error @@ -60,7 +66,7 @@ type osTypeInterface interface { setErrs([]error) } -// osPackages included by linux struct +// osPackages is included by base struct type osPackages struct { // installed packages Packages models.PackageInfoList @@ -425,12 +431,65 @@ func detectPlatforms() []error { // Prepare installs requred packages to scan vulnerabilities. func Prepare() []error { - return parallelSSHExec(func(o osTypeInterface) error { + errs := parallelSSHExec(func(o osTypeInterface) error { + if err := o.checkDependencies(); err != nil { + return err + } + return nil + }) + if len(errs) != 0 { + return errs + } + + var targets []osTypeInterface + for _, s := range servers { + deps := s.getLackDependencies() + if len(deps) != 0 { + targets = append(targets, s) + } + } + if len(targets) == 0 { + Log.Info("No need to install dependencies") + return nil + } + + Log.Info("Below servers are needed to install dependencies") + for _, s := range targets { + for _, d := range s.getLackDependencies() { + Log.Infof(" - %s on %s", d, s.getServerInfo().GetServerName()) + } + } + Log.Info("Is this ok to install dependencies on the servers? [y/N]") + + reader := bufio.NewReader(os.Stdin) + for { + text, err := reader.ReadString('\n') + if err != nil { + return []error{err} + } + switch strings.TrimSpace(text) { + case "", "N", "n": + return nil + case "y", "Y": + goto yes + default: + Log.Info("Please enter y or N") + } + } + +yes: + servers = targets + errs = parallelSSHExec(func(o osTypeInterface) error { if err := o.install(); err != nil { return err } return nil }) + if len(errs) != 0 { + return errs + } + Log.Info("All dependencies were installed correctly") + return nil } // Scan scan From 8d2180cf5a348a24a560d8b36d8a94b6cda66673 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 13 Oct 2016 16:14:05 +0900 Subject: [PATCH 20/37] Update README --- README.fr.md | 6 +++--- README.ja.md | 6 +++--- README.md | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.fr.md b/README.fr.md index d8fac6a7f6..69a34e6c41 100644 --- a/README.fr.md +++ b/README.fr.md @@ -107,14 +107,14 @@ Vuls requiert l'installation des paquets suivants : - sqlite - git - gcc -- go v1.6 +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` Ajoutez les lignes suivantes dans /etc/profile.d/goenv.sh diff --git a/README.ja.md b/README.ja.md index 5e9a17689d..030b4dde8a 100644 --- a/README.ja.md +++ b/README.ja.md @@ -123,14 +123,14 @@ Vulsセットアップに必要な以下のソフトウェアをインストー - SQLite3 - git - gcc -- go v1.6 or later +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` /etc/profile.d/goenv.sh を作成し、下記を追加する。 diff --git a/README.md b/README.md index bea9e4be0e..8aba4df40a 100644 --- a/README.md +++ b/README.md @@ -131,14 +131,14 @@ Vuls requires the following packages. - SQLite3 - git - gcc -- go v1.6 or later +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` Add these lines into /etc/profile.d/goenv.sh From 63f0a272c4be2b28423995d42517d6ab464c365f Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 13 Oct 2016 19:30:36 +0900 Subject: [PATCH 21/37] Update README --- README.ja.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.ja.md b/README.ja.md index 030b4dde8a..3978d60730 100644 --- a/README.ja.md +++ b/README.ja.md @@ -223,7 +223,7 @@ $ vuls prepare ## Step8. Start Scanning ``` -$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 +$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -report-json INFO[0000] Start scanning (config: /home/ec2-user/config.toml) INFO[0000] Start scanning INFO[0000] config: /home/ec2-user/config.toml diff --git a/README.md b/README.md index 8aba4df40a..efe0996650 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ see [Usage: Prepare](https://github.com/future-architect/vuls#usage-prepare) ## Step8. Start Scanning ``` -$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 +$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -report-json INFO[0000] Start scanning (config: /home/ec2-user/config.toml) INFO[0000] Start scanning INFO[0000] config: /home/ec2-user/config.toml From 66b27a7795b63b1d236e95379c0f2c88b0eb164b Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Sat, 15 Oct 2016 12:37:52 +0900 Subject: [PATCH 22/37] remove base docker image --- setup/docker/README.md | 7 ++++++- setup/docker/base/latest/Dockerfile | 12 ------------ setup/docker/base/latest/README.md | 19 ------------------- .../go-cve-dictionary/latest/Dockerfile | 12 +++++------- .../docker/go-cve-dictionary/latest/README.md | 18 +++++++++++++----- setup/docker/vuls/latest/Dockerfile | 12 +++++------- setup/docker/vuls/latest/README.md | 10 ++++++++++ 7 files changed, 39 insertions(+), 51 deletions(-) delete mode 100644 setup/docker/base/latest/Dockerfile delete mode 100644 setup/docker/base/latest/README.md diff --git a/setup/docker/README.md b/setup/docker/README.md index 70361cbf3c..76a577dee6 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -26,7 +26,9 @@ This image version is same as the github repository version. ```console $ for i in {2002..2016}; do \ docker run --rm -it \ - -v $PWD:/vuls vuls/go-cve-dictionary fetchnvd -years $i; \ + -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + vuls/go-cve-dictionary fetchnvd -years $i; \ done ``` @@ -49,6 +51,7 @@ keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker $ docker run --rm \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ vuls/vuls configtest \ -config=./config.toml # path to config.toml in docker ``` @@ -59,6 +62,7 @@ $ docker run --rm \ $ docker run --rm \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ vuls/vuls prepare \ -config=./config.toml # path to config.toml in docker ``` @@ -69,6 +73,7 @@ $ docker run --rm \ $ docker run --rm -it \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ -v /etc/localtime:/etc/localtime:ro \ -e "TZ=Asia/Tokyo" \ vuls/vuls scan \ diff --git a/setup/docker/base/latest/Dockerfile b/setup/docker/base/latest/Dockerfile deleted file mode 100644 index 43b4aedd7a..0000000000 --- a/setup/docker/base/latest/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.7.1 - -MAINTAINER hikachan sadayuki-matsuno - -# glide install -ENV GLIDE_VERSION 0.12.2 -ENV GLIDE_DOWNLOAD_URL https://github.com/Masterminds/glide/releases/download/v$GLIDE_VERSION/glide-v$GLIDE_VERSION-linux-amd64.tar.gz -RUN curl -fsSL "$GLIDE_DOWNLOAD_URL" -o glide.tar.gz \ - && mkdir /usr/local/glide \ - && tar -C /usr/local/glide -xzf glide.tar.gz \ - && ln -s /usr/local/glide/linux-amd64/glide /usr/local/bin/ \ - && rm glide.tar.gz diff --git a/setup/docker/base/latest/README.md b/setup/docker/base/latest/README.md deleted file mode 100644 index f28d508858..0000000000 --- a/setup/docker/base/latest/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Vuls-Base-Docker - -This is the Git repo of the official Docker image for vuls-base. -See the [Hub page](https://hub.docker.com/r/vuls/vuls-base/) for the full readme on how to use the Docker image and for information regarding contributing and issues. - -# Supported tags and respective `Dockerfile` links - -- [`latest` (*vuls-base:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/base/latest/Dockerfile) - -# Caution - -This image is built per commit. -If you want to use the latest docker image, you should remove the existing image, and pull it once again. - - -# What is vuls-base? - -Base images of vuls and go-cve-dictionary. - diff --git a/setup/docker/go-cve-dictionary/latest/Dockerfile b/setup/docker/go-cve-dictionary/latest/Dockerfile index 8a464819d0..c201bec3ff 100644 --- a/setup/docker/go-cve-dictionary/latest/Dockerfile +++ b/setup/docker/go-cve-dictionary/latest/Dockerfile @@ -1,19 +1,17 @@ -FROM vuls/vuls-base:latest +FROM golang:latest MAINTAINER hikachan sadayuki-matsuno ENV REPOSITORY github.com/kotakanbe/go-cve-dictionary +ENV LOGDIR /var/log/vuls ENV WORKDIR /vuls # go-cve-dictionary install RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ && cd $GOPATH/src/$REPOSITORY \ - && glide install \ - && go build \ - && mv $GOPATH/src/$REPOSITORY/go-cve-dictionary /usr/local/bin \ - && chmod +x /usr/local/bin/go-cve-dictionary \ - && mkdir -p /var/log/vuls + && make install \ + && mkdir -p $LOGDIR -VOLUME $WORKDIR +VOLUME [$WORKDIR, $LOGDIR] WORKDIR $WORKDIR ENV PWD $WORKDIR diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md index 60750f4cea..9e16d257c6 100644 --- a/setup/docker/go-cve-dictionary/latest/README.md +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -22,12 +22,19 @@ This is tool to build a local copy of the NVD (National Vulnerabilities Database # How to use this image +## check vuls version + +``` +$ docker run --rm vuls/go-cve-dictionary -v +``` + ## fetchnvd ```console $ for i in {2002..2016}; do \ docker run --rm -it \ -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ vuls/go-cve-dictionary fetchnvd -years $i; \ done ``` @@ -36,11 +43,12 @@ $ for i in {2002..2016}; do \ ```console $ docker run -dt \ - --name go-cve-dictionary \ - -v $PWD:/vuls \ - --expose 1323 \ - -p 1323:1323 \ - vuls/go-cve-dictionary server --bind=0.0.0.0 + --name go-cve-dictionary \ + -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + --expose 1323 \ + -p 1323:1323 \ + vuls/go-cve-dictionary server --bind=0.0.0.0 ``` Prease refer to [this](https://hub.docker.com/r/vuls/go-cve-dictionary). diff --git a/setup/docker/vuls/latest/Dockerfile b/setup/docker/vuls/latest/Dockerfile index 7ca2e8e1cf..1db73c550f 100644 --- a/setup/docker/vuls/latest/Dockerfile +++ b/setup/docker/vuls/latest/Dockerfile @@ -1,19 +1,17 @@ -FROM vuls/vuls-base:latest +FROM golang:latest MAINTAINER hikachan sadayuki-matsuno ENV REPOSITORY github.com/future-architect/vuls +ENV LOGDIR /var/log/vuls ENV WORKDIR /vuls # go-cve-dictionary install RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ && cd $GOPATH/src/$REPOSITORY \ - && glide install \ - && go build \ - && mv $GOPATH/src/$REPOSITORY/vuls /usr/local/bin \ - && mkdir -p /var/log/vuls \ - && chmod +x /usr/local/bin/vuls + && make install \ + && mkdir -p $LOGDIR -VOLUME $WORKDIR +VOLUME [$WORKDIR, $LOGDIR] WORKDIR $WORKDIR ENV PWD $WORKDIR diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md index 3210b54a27..6b878599c8 100644 --- a/setup/docker/vuls/latest/README.md +++ b/setup/docker/vuls/latest/README.md @@ -21,6 +21,12 @@ Please see the [Documentation](https://github.com/future-architect/vuls) # How to use this image +## check vuls version + +``` +$ docker run --rm vuls/vuls -v +``` + ## configtest Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). @@ -40,6 +46,7 @@ keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker $ docker run --rm \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ vuls/vuls configtest ``` @@ -50,6 +57,7 @@ $ docker run --rm \ $ docker run --rm \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ vuls/vuls prepare \ -config=./config.toml # path to config.toml in docker ``` @@ -60,6 +68,7 @@ $ docker run --rm \ $ docker run --rm -it \ -v ~/.ssh:/root/.ssh:ro \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ -v /etc/localtime:/etc/localtime:ro \ vuls/vuls scan \ -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ @@ -72,6 +81,7 @@ $ docker run --rm -it \ ```console $ docker run --rm -it \ -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ vuls/vuls tui ``` From 1e2b93d55b84aa1ff73f4a0af9a69df736304c47 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Fri, 14 Oct 2016 19:58:08 +0900 Subject: [PATCH 23/37] Support ignore CveIDs in config --- README.ja.md | 40 ++++++++++++++++++++++++++++++++++++++++ README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- commands/discover.go | 2 ++ config/config.go | 2 ++ config/tomlloader.go | 15 ++++++++++++++- models/models.go | 1 + report/slack.go | 3 --- scan/base.go | 27 ++++++++++++++++++++++++--- 8 files changed, 125 insertions(+), 8 deletions(-) diff --git a/README.ja.md b/README.ja.md index 3978d60730..8b398ff043 100644 --- a/README.ja.md +++ b/README.ja.md @@ -469,6 +469,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -488,6 +489,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6314"] #optional = [ # ["key", "value"], #] @@ -502,6 +504,7 @@ host = "172.31.4.82" - keyPath: SSH private key path - cpeNames: see [Usage: Scan vulnerability of non-OS package](https://github.com/future-architect/vuls/blob/master/README.ja.md#usage-scan-vulnerability-of-non-os-package) - containers: see [Usage: Scan Docker containers](https://github.com/future-architect/vuls/blob/master/README.ja.md#usage-scan-docker-containers) + - ignoreCves: CVE IDs that will not be reported. But output to JSON file. - optional: JSONレポートに含めたい追加情報 @@ -782,6 +785,43 @@ $ vuls scan \ -azure-container=vuls ``` +## Example: IgnoreCves + +Slack, Mail, テキスト出力しないくないCVE IDがある場合は、設定ファイルに定義することでレポートされなくなる。 +ただ、JSONファイルには以下のように出力される。 + +- config.toml +```toml +[default] +ignoreCves = ["CVE-2016-6313"] + +[servers.bsd] +host = "192.168.11.11" +user = "kanbe" +ignoreCves = ["CVE-2016-6314"] +``` + +- bsd.json +```json +[ + { + "ServerName": "bsd", + "Family": "FreeBSD", + "Release": "10.3-RELEASE", + "IgnoredCves" : { + "CveDetail" : { + "CVE-2016-6313", + ... + }, + "CveDetail" : { + "CVE-2016-6314", + ... + }, + } + } +] +``` + ## Example: Add optional key-value pairs to JSON 追加情報をJSONに含めることができる。 diff --git a/README.md b/README.md index efe0996650..18980be35c 100644 --- a/README.md +++ b/README.md @@ -389,6 +389,7 @@ subjectPrefix = "[vuls]" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -404,6 +405,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -476,6 +478,7 @@ You can customize your configuration using this template. # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -495,6 +498,7 @@ You can customize your configuration using this template. # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6314"] #optional = [ # ["key", "value"], #] @@ -508,6 +512,7 @@ You can customize your configuration using this template. - keyPath: SSH private key path - cpeNames: see [Usage: Scan vulnerability of non-OS package](https://github.com/future-architect/vuls#usage-scan-vulnerability-of-non-os-package) - containers: see [Usage: Scan Docker containers](https://github.com/future-architect/vuls#usage-scan-docker-containers) + - ignoreCves: CVE IDs that will not be reported. But output to JSON file. - optional: Add additional information to JSON report. Vuls supports two types of SSH. One is native go implementation. The other is external SSH command. For details, see [-ssh-external option](https://github.com/future-architect/vuls#-ssh-external-option) @@ -780,6 +785,43 @@ $ vuls scan \ -azure-container=vuls ``` +## Example: IgnoreCves + +Define ignoreCves in config if you don't want to report(slack, mail, text...) specific CVE IDs. But these ignoreCves will be output to JSON file like below. + +- config.toml +```toml +[default] +ignoreCves = ["CVE-2016-6313"] + +[servers.bsd] +host = "192.168.11.11" +user = "kanbe" +ignoreCves = ["CVE-2016-6314"] +``` + +- bsd.json +```json +[ + { + "ServerName": "bsd", + "Family": "FreeBSD", + "Release": "10.3-RELEASE", + "IgnoredCves" : { + "CveDetail" : { + "CVE-2016-6313", + ... + }, + "CveDetail" : { + "CVE-2016-6314", + ... + }, + } + } +] +``` + + ## Example: Add optional key-value pairs to JSON Optional key-value can be outputted to JSON. @@ -1097,4 +1139,3 @@ Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHAN # License Please see [LICENSE](https://github.com/future-architect/vuls/blob/master/LICENSE). - diff --git a/commands/discover.go b/commands/discover.go index 92d147cd67..cf9111c03d 100644 --- a/commands/discover.go +++ b/commands/discover.go @@ -116,6 +116,7 @@ subjectPrefix = "[vuls]" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2014-6271"] #optional = [ # ["key", "value"], #] @@ -132,6 +133,7 @@ host = "{{$ip}}" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2014-0160"] #optional = [ # ["key", "value"], #] diff --git a/config/config.go b/config/config.go index 93c3dffabd..6cd6141ebc 100644 --- a/config/config.go +++ b/config/config.go @@ -233,6 +233,8 @@ type ServerInfo struct { // Container Names or IDs Containers []string + IgnoreCves []string + // Optional key-value set that will be outputted to JSON Optional [][]interface{} diff --git a/config/tomlloader.go b/config/tomlloader.go index 0ca05f8d90..21e52640cb 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -51,7 +51,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { i := 0 for name, v := range conf.Servers { - if 0 < len(v.KeyPassword) { log.Warn("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE.") } @@ -108,6 +107,20 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { s.Containers = d.Containers } + s.IgnoreCves = v.IgnoreCves + for _, cve := range d.IgnoreCves { + found := false + for _, c := range s.IgnoreCves { + if cve == c { + found = true + break + } + } + if !found { + s.IgnoreCves = append(s.IgnoreCves, cve) + } + } + s.Optional = v.Optional for _, dkv := range d.Optional { found := false diff --git a/models/models.go b/models/models.go index 7c862cfe18..3a1877748d 100644 --- a/models/models.go +++ b/models/models.go @@ -89,6 +89,7 @@ type ScanResult struct { // NWLinks []NWLink KnownCves []CveInfo UnknownCves []CveInfo + IgnoredCves []CveInfo Optional [][]interface{} `gorm:"-"` } diff --git a/report/slack.go b/report/slack.go index c0faa437a1..655445e67f 100644 --- a/report/slack.go +++ b/report/slack.go @@ -59,7 +59,6 @@ type SlackWriter struct{} func (w SlackWriter) Write(scanResults []models.ScanResult) error { conf := config.Conf.Slack for _, s := range scanResults { - channel := conf.Channel if channel == "${servername}" { channel = fmt.Sprintf("#%s", s.ServerName) @@ -97,7 +96,6 @@ func (w SlackWriter) Write(scanResults []models.ScanResult) error { } func msgText(r models.ScanResult) string { - notifyUsers := "" if 0 < len(r.KnownCves) || 0 < len(r.UnknownCves) { notifyUsers = getNotifyUsers(config.Conf.Slack.NotifyUsers) @@ -108,7 +106,6 @@ func msgText(r models.ScanResult) string { } func toSlackAttachments(scanResult models.ScanResult) (attaches []*attachment) { - cves := scanResult.KnownCves if !config.Conf.IgnoreUnscoredCves { cves = append(cves, scanResult.UnknownCves...) diff --git a/scan/base.go b/scan/base.go index 3a39d6c70a..173c03a5dc 100644 --- a/scan/base.go +++ b/scan/base.go @@ -224,13 +224,31 @@ func (l base) isAwsInstanceID(str string) bool { } func (l *base) convertToModel() (models.ScanResult, error) { - var scoredCves, unscoredCves models.CveInfos + var scoredCves, unscoredCves, ignoredCves models.CveInfos for _, p := range l.UnsecurePackages { + // ignoreCves + found := false + for _, icve := range l.getServerInfo().IgnoreCves { + if icve == p.CveDetail.CveID { + ignoredCves = append(ignoredCves, models.CveInfo{ + CveDetail: p.CveDetail, + Packages: p.Packs, + DistroAdvisories: p.DistroAdvisories, + }) + found = true + break + } + } + if found { + continue + } + + // unscoredCves if p.CveDetail.CvssScore(config.Conf.Lang) <= 0 { unscoredCves = append(unscoredCves, models.CveInfo{ CveDetail: p.CveDetail, Packages: p.Packs, - DistroAdvisories: p.DistroAdvisories, // only Amazon Linux + DistroAdvisories: p.DistroAdvisories, }) continue } @@ -241,10 +259,11 @@ func (l *base) convertToModel() (models.ScanResult, error) { models.CpeName{Name: cpename}) } + // scoredCves cve := models.CveInfo{ CveDetail: p.CveDetail, Packages: p.Packs, - DistroAdvisories: p.DistroAdvisories, // only Amazon Linux + DistroAdvisories: p.DistroAdvisories, CpeNames: cpenames, } scoredCves = append(scoredCves, cve) @@ -257,6 +276,7 @@ func (l *base) convertToModel() (models.ScanResult, error) { sort.Sort(scoredCves) sort.Sort(unscoredCves) + sort.Sort(ignoredCves) return models.ScanResult{ ServerName: l.ServerInfo.ServerName, @@ -267,6 +287,7 @@ func (l *base) convertToModel() (models.ScanResult, error) { Platform: l.Platform, KnownCves: scoredCves, UnknownCves: unscoredCves, + IgnoredCves: ignoredCves, Optional: l.ServerInfo.Optional, }, nil } From 652b37e630945a35b1771b287f0264f679b61864 Mon Sep 17 00:00:00 2001 From: Takayuki Ushida Date: Mon, 17 Oct 2016 22:43:20 +0900 Subject: [PATCH 24/37] fix README --- README.ja.md | 6 +++--- README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.ja.md b/README.ja.md index 8b398ff043..345cc007e5 100644 --- a/README.ja.md +++ b/README.ja.md @@ -808,7 +808,7 @@ ignoreCves = ["CVE-2016-6314"] "ServerName": "bsd", "Family": "FreeBSD", "Release": "10.3-RELEASE", - "IgnoredCves" : { + "IgnoredCves" : [ "CveDetail" : { "CVE-2016-6313", ... @@ -816,8 +816,8 @@ ignoreCves = ["CVE-2016-6314"] "CveDetail" : { "CVE-2016-6314", ... - }, - } + } + ] } ] ``` diff --git a/README.md b/README.md index 18980be35c..04c7d2c6de 100644 --- a/README.md +++ b/README.md @@ -807,7 +807,7 @@ ignoreCves = ["CVE-2016-6314"] "ServerName": "bsd", "Family": "FreeBSD", "Release": "10.3-RELEASE", - "IgnoredCves" : { + "IgnoredCves" : [ "CveDetail" : { "CVE-2016-6313", ... @@ -815,8 +815,8 @@ ignoreCves = ["CVE-2016-6314"] "CveDetail" : { "CVE-2016-6314", ... - }, - } + } + ] } ] ``` From d23ef838f8ed46fd4cc1de6651510158b8395f78 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 19 Oct 2016 15:08:08 +0900 Subject: [PATCH 25/37] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 04c7d2c6de..3905fdcf35 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ Vuls is a tool created to solve the problems listed above. It has the following - Support software registered in CPE - Agentless architecture - User is required to only setup one machine that is connected to other target servers via SSH +- Nondestructive testing +- Pre-authorization is not necessary before scanning on AWS - Auto generation of configuration file template - Auto detection of servers set using CIDR, generate configuration file template - Email and Slack notification is possible (supports Japanese language) From a0ac8639988be6d00f79b863c27a5314216aa12f Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 19 Oct 2016 15:12:04 +0900 Subject: [PATCH 26/37] Update README.ja.md --- README.ja.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.ja.md b/README.ja.md index 345cc007e5..1857f75145 100644 --- a/README.ja.md +++ b/README.ja.md @@ -46,6 +46,8 @@ Vulsは上に挙げた手動運用での課題を解決するツールであり - CPEに登録されているソフトウェアが対象 - エージェントレスアーキテクチャ - スキャン対象サーバにSSH接続可能なマシン1台にセットアップするだけで動作 +- 非破壊スキャン(SSHでコマンド発行するだけ) +- AWSでの脆弱性/侵入テスト事前申請は必要なし - 設定ファイルのテンプレート自動生成 - CIDRを指定してサーバを自動検出、設定ファイルのテンプレートを生成 - EmailやSlackで通知可能(日本語でのレポートも可能) From 4c82458481932ec40afef6222b1a15e1a0c3d064 Mon Sep 17 00:00:00 2001 From: Mike Oswell Date: Fri, 21 Oct 2016 16:30:07 -0700 Subject: [PATCH 27/37] Support recent refactoring of gocui's SetCurrentView method. --- glide.lock | 2 +- report/tui.go | 42 ++++++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/glide.lock b/glide.lock index a74c30ecd1..c084c618bf 100644 --- a/glide.lock +++ b/glide.lock @@ -61,7 +61,7 @@ imports: - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/jroimartin/gocui - version: 4e9ce9a8e26f2ef33dfe297dbdfca148733b6b9b + version: 550f04e523205530542d0c4fe63c4c0ab5d046bd - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary diff --git a/report/tui.go b/report/tui.go index bd63026c2c..d3f3166acf 100644 --- a/report/tui.go +++ b/report/tui.go @@ -176,35 +176,41 @@ func keybindings(g *gocui.Gui) (err error) { } func nextView(g *gocui.Gui, v *gocui.View) error { + var err error + if v == nil { - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } switch v.Name() { case "side": - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") case "summary": - return g.SetCurrentView("detail") + _, err = g.SetCurrentView("detail") case "detail": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") default: - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") } + return err } func previousView(g *gocui.Gui, v *gocui.View) error { + var err error + if v == nil { - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } switch v.Name() { case "side": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") case "summary": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") case "detail": - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") default: - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } + return err } func movable(v *gocui.View, nextY int) (ok bool, yLimit int) { @@ -373,7 +379,7 @@ func cursorPageUp(g *gocui.Gui, v *gocui.View) error { func previousSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -381,7 +387,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if err := g.SetCurrentView("detail"); err != nil { + if _, err := g.SetCurrentView("detail"); err != nil { return err } } @@ -391,7 +397,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { func nextSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -399,7 +405,7 @@ func nextSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if err := g.SetCurrentView("detail"); err != nil { + if _, err := g.SetCurrentView("detail"); err != nil { return err } } @@ -464,7 +470,7 @@ func getLine(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if err := g.SetCurrentView("msg"); err != nil { + if _, err := g.SetCurrentView("msg"); err != nil { return err } } @@ -486,7 +492,7 @@ func showMsg(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if err := g.SetCurrentView("msg"); err != nil { + if _, err := g.SetCurrentView("msg"); err != nil { return err } } @@ -497,7 +503,7 @@ func delMsg(g *gocui.Gui, v *gocui.View) error { if err := g.DeleteView("msg"); err != nil { return err } - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } return nil @@ -532,7 +538,7 @@ func setSideLayout(g *gocui.Gui) error { fmt.Fprintln(v, result.ServerInfoTui()) } currentScanResult = scanHistory.ScanResults[0] - if err := g.SetCurrentView("side"); err != nil { + if _, err := g.SetCurrentView("side"); err != nil { return err } } From bb6725372b136a362378e53face3db8ab2d53082 Mon Sep 17 00:00:00 2001 From: Mike Oswell Date: Fri, 14 Oct 2016 22:36:13 -0700 Subject: [PATCH 28/37] Add support for reading CVE data from MySQL. --- commands/scan.go | 15 +++++++++++++-- config/config.go | 22 ++++++++++++++++++---- cveapi/cve_client.go | 11 ++++++++--- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/commands/scan.go b/commands/scan.go index 1e1df7b746..6c8167f11e 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -45,6 +45,7 @@ type ScanCmd struct { configPath string resultsDir string + cvedbtype string cvedbpath string cveDictionaryURL string cacheDBPath string @@ -90,7 +91,8 @@ func (*ScanCmd) Usage() string { [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -132,6 +134,12 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { defaultResultsDir := filepath.Join(wd, "results") f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results") + f.StringVar( + &p.cvedbtype, + "cve-dictionary-dbtype", + "sqlite3", + "DB type for fetching CVE dictionary (sqlite3 or mysql)") + f.StringVar( &p.cvedbpath, "cve-dictionary-dbpath", @@ -254,7 +262,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) logrus.Info("Start scanning") logrus.Infof("config: %s", p.configPath) if p.cvedbpath != "" { - logrus.Infof("cve-dictionary: %s", p.cvedbpath) + if p.cvedbtype == "sqlite3" { + logrus.Infof("cve-dictionary: %s", p.cvedbpath) + } } else { logrus.Infof("cve-dictionary: %s", p.cveDictionaryURL) } @@ -357,6 +367,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } c.Conf.ResultsDir = p.resultsDir + c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDictionaryURL = p.cveDictionaryURL c.Conf.CacheDBPath = p.cacheDBPath diff --git a/config/config.go b/config/config.go index 93c3dffabd..db11a50f8c 100644 --- a/config/config.go +++ b/config/config.go @@ -49,6 +49,7 @@ type Config struct { HTTPProxy string `valid:"url"` ResultsDir string + CveDBType string CveDBPath string CacheDBPath string @@ -75,10 +76,23 @@ func (c Config) Validate() bool { } } - if len(c.CveDBPath) != 0 { - if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { - errs = append(errs, fmt.Errorf( - "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + // If no valid DB type is set, default to sqlite3 + if c.CveDBType == "" { + c.CveDBType = "sqlite3" + } + + if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" { + errs = append(errs, fmt.Errorf( + "CVE DB type must be either 'sqlite3' or 'mysql'. -cve-dictionary-dbtype: %s", c.CveDBType)) + } + + + if c.CveDBType == "sqlite3" { + if len(c.CveDBPath) != 0 { + if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { + errs = append(errs, fmt.Errorf( + "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + } } } diff --git a/cveapi/cve_client.go b/cveapi/cve_client.go index eb33838183..44f5f282b6 100644 --- a/cveapi/cve_client.go +++ b/cveapi/cve_client.go @@ -49,7 +49,7 @@ func (api *cvedictClient) initialize() { func (api cvedictClient) CheckHealth() (ok bool, err error) { if config.Conf.CveDBPath != "" { - log.Debugf("get cve-dictionary from sqlite3") + log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType) return true, nil } @@ -135,8 +135,10 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet } func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) { - log.Debugf("open cve-dictionary db") + log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + cveconfig.Conf.DBType = config.Conf.CveDBType cveconfig.Conf.DBPath = config.Conf.CveDBPath + cveconfig.Conf.DebugSQL = config.Conf.DebugSQL if err := cvedb.OpenDB(); err != nil { return []cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) @@ -239,8 +241,11 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c } func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) { - log.Debugf("open cve-dictionary db") + log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + cveconfig.Conf.DBType = config.Conf.CveDBType cveconfig.Conf.DBPath = config.Conf.CveDBPath + cveconfig.Conf.DebugSQL = config.Conf.DebugSQL + if err := cvedb.OpenDB(); err != nil { return []cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) From 686e9f07a9fa88d47b41bb7afd5acccc01304f90 Mon Sep 17 00:00:00 2001 From: hirokazu yamada Date: Mon, 24 Oct 2016 15:36:16 +0900 Subject: [PATCH 29/37] Fixed error for the latest version of gocui --- glide.lock | 60 +++++++++++++++++++++++++-------------------------- report/tui.go | 6 +++--- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/glide.lock b/glide.lock index c084c618bf..86e4dcf6c1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,38 +1,38 @@ hash: 01cee4aa1d27c967f5f5165febfbd77fd0e89a6e7a9a89c84099c0d45dffd446 -updated: 2016-10-12T15:28:25.843936367+09:00 +updated: 2016-10-25T19:51:49.542919471+09:00 imports: - name: github.com/asaskevich/govalidator version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 - name: github.com/aws/aws-sdk-go - version: aad29423c2c09f0a1fa5346216bc041a4994eef4 + version: 9e5bedb97b1cd85e53fd99209f93fd1a8a9f1df7 subpackages: - aws - - aws/credentials - - aws/session - - service/s3 - aws/awserr + - aws/awsutil - aws/client + - aws/client/metadata - aws/corehandlers + - aws/credentials + - aws/credentials/ec2rolecreds + - aws/credentials/endpointcreds - aws/credentials/stscreds - aws/defaults + - aws/ec2metadata - aws/request - - private/endpoints - - aws/awsutil - - aws/client/metadata + - aws/session - aws/signer/v4 + - private/endpoints - private/protocol + - private/protocol/query + - private/protocol/query/queryutil + - private/protocol/rest - private/protocol/restxml + - private/protocol/xml/xmlutil - private/waiter + - service/s3 - service/sts - - aws/credentials/ec2rolecreds - - aws/credentials/endpointcreds - - aws/ec2metadata - - private/protocol/rest - - private/protocol/query - - private/protocol/xml/xmlutil - - private/protocol/query/queryutil - name: github.com/Azure/azure-sdk-for-go - version: 91f3d4a4d024e3c0d4d9412916d05cf84504a616 + version: 9016164015faa51e549605e7b4b117f7de2aa6f9 subpackages: - storage - name: github.com/boltdb/bolt @@ -40,7 +40,7 @@ imports: - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/cenkalti/backoff - version: 8edc80b07f38c27352fb186d971c628a6c32552b + version: b02f2bbce11d7ea6b97f282ef1771b0fe2f65ef3 - name: github.com/cheggaaa/pb version: ad4efe000aa550bb54918c06ebbadc0ff17687b9 - name: github.com/go-ini/ini @@ -55,23 +55,23 @@ imports: - name: github.com/howeyc/gopass version: f5387c492211eb133053880d23dfae62aa14123d - name: github.com/jinzhu/gorm - version: 39165d498058a823126af3cbf4d2a3b0e1acf11e + version: c1b9cf186e4bcd8e5d566ef43f2ae2dfe22dc34e - name: github.com/jinzhu/inflection version: 74387dc39a75e970e7a3ae6a3386b5bd2e5c5cff - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/jroimartin/gocui - version: 550f04e523205530542d0c4fe63c4c0ab5d046bd + version: 357a541add9e311f7b67dfbaf92e28c71680a6b7 - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary - version: f9f68fee57dca8e60fb5d9d6b34d3215d854fc06 + version: 8465a01aad6bf864813d61daba2e6d0a8b6a07d9 subpackages: - config - - models - db - - log - jvn + - log + - models - nvd - util - name: github.com/kotakanbe/go-pingscanner @@ -89,7 +89,7 @@ imports: - name: github.com/mgutz/ansi version: c286dcecd19ff979eeb73ea444e479b903f2cfcb - name: github.com/moul/http2curl - version: b1479103caacaa39319f75e7f57fc545287fca0d + version: c984a4ec331f8ef0e5cd782975a97c92bd8ab40c - name: github.com/nsf/termbox-go version: b6acae516ace002cb8105a89024544a1480655a5 - name: github.com/parnurzeal/gorequest @@ -99,21 +99,21 @@ imports: - name: github.com/Sirupsen/logrus version: 3ec0642a7fb6488f65b06f9040adc67e3990296a - name: golang.org/x/crypto - version: 4cd25d65a015cc83d41bf3454e6e8d6c116d16da + version: 1150b8bd09e53aea1d415621adae9bad665061a1 subpackages: - - ssh - - ssh/agent - - ssh/terminal - curve25519 - ed25519 - ed25519/internal/edwards25519 + - ssh + - ssh/agent + - ssh/terminal - name: golang.org/x/net - version: cf4effbb9db1f3ef07f7e1891402991b6afbb276 + version: 65dfc08770ce66f74becfdff5f8ab01caef4e946 subpackages: - context - publicsuffix - name: golang.org/x/sys - version: 9bb9f0998d48b31547d975974935ae9b48c7a03c + version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix -devImports: [] +testImports: [] diff --git a/report/tui.go b/report/tui.go index d3f3166acf..2b2477d2fc 100644 --- a/report/tui.go +++ b/report/tui.go @@ -48,14 +48,14 @@ func RunTui(jsonDirName string) subcommands.ExitStatus { return subcommands.ExitFailure } - g := gocui.NewGui() - if err := g.Init(); err != nil { + g, err := gocui.NewGui() + if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } defer g.Close() - g.SetLayout(layout) + g.SetManagerFunc(layout) if err := keybindings(g); err != nil { log.Errorf("%s", err) return subcommands.ExitFailure From b042a600c3ccfd7ef618cf2d27aa331de4be46b3 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Mon, 24 Oct 2016 22:56:21 +0900 Subject: [PATCH 30/37] Integrate OWASP Dependency Check --- README.ja.md | 25 ++++++++ README.md | 26 +++++++- commands/discover.go | 2 + commands/scan.go | 4 +- config/config.go | 4 +- config/tomlloader.go | 29 +++++++-- .../owasp-dependency-check/parser/parser.go | 64 +++++++++++++++++++ 7 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 contrib/owasp-dependency-check/parser/parser.go diff --git a/README.ja.md b/README.ja.md index 1857f75145..8aa35edb5e 100644 --- a/README.ja.md +++ b/README.ja.md @@ -891,6 +891,31 @@ Vulsは、[CPE](https://nvd.nist.gov/cpe.cfm)に登録されているソフト "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", ] ``` + + +# Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental) +[OWASP Dependency check](https://www.owasp.org/index.php/OWASP_Dependency_Check) は、プログラミング言語のライブラリを特定し(CPEを推測)、公開済みの脆弱性を検知するツール。 + +VulsとDependency Checkを連携させる方法は以下 +- Dependency Checkを、--format=XMLをつけて実行する +- そのXMLをconfig.toml内で以下のように定義する + + ``` + [servers] + + [servers.172-31-4-82] + host = "172.31.4.82" + user = "ec2-user" + keyPath = "/home/username/.ssh/id_rsa" + dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" + ``` + +VulsとDependency Checkの連携すると以下の利点がある +- ライブラリを更新した場合に、config.tomlのCPEの定義を変更しなくても良い +- Vulsの機能でSlack, Emailで通知可能 +- 日本語のレポートが可能 + - Dependency Checkは日本語レポートに対応していない + # Usage: Scan Docker containers diff --git a/README.md b/README.md index 3905fdcf35..1493043167 100644 --- a/README.md +++ b/README.md @@ -866,7 +866,7 @@ optional = [ ---- -# Usage: Scan vulnerability of non-OS package +# Usage: Scan vulnerabilites of non-OS packages It is possible to detect vulnerabilities in non-OS packages, such as something you compiled by yourself, language libraries and frameworks, that have been registered in the [CPE](https://nvd.nist.gov/cpe.cfm). @@ -890,6 +890,30 @@ To detect the vulnerability of Ruby on Rails v4.2.1, cpeNames needs to be set in "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", ] ``` + +# Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental) +[OWASP Dependency check](https://www.owasp.org/index.php/OWASP_Dependency_Check) is a utility that identifies project dependencies and checks if there are any known, publicly disclosed, vulnerabilities. + +Benefit of integrating Vuls And OWASP Dependency Check is below. +- Automatic Update of Vuls config when the libraries are updated. +- Reporting by Email or Slack by using Vuls. +- Reporting in Japanese + - OWASP Dependency Check supports only English + +How to integrate Vuls with OWASP Dependency Check +- Execute OWASP Dependency Check with --format=XML option. +- Define the xml file path of dependency check in config.toml. + + ``` + [servers] + + [servers.172-31-4-82] + host = "172.31.4.82" + user = "ec2-user" + keyPath = "/home/username/.ssh/id_rsa" + dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" + ``` + # Usage: Scan Docker containers diff --git a/commands/discover.go b/commands/discover.go index cf9111c03d..ddaf45fc5c 100644 --- a/commands/discover.go +++ b/commands/discover.go @@ -115,6 +115,7 @@ subjectPrefix = "[vuls]" #cpeNames = [ # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] +#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" #containers = ["${running}"] #ignoreCves = ["CVE-2014-6271"] #optional = [ @@ -132,6 +133,7 @@ host = "{{$ip}}" #cpeNames = [ # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] +#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" #containers = ["${running}"] #ignoreCves = ["CVE-2014-0160"] #optional = [ diff --git a/commands/scan.go b/commands/scan.go index 1e1df7b746..c69790462e 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -33,6 +33,7 @@ import ( "github.com/future-architect/vuls/scan" "github.com/future-architect/vuls/util" "github.com/google/subcommands" + "github.com/k0kubun/pp" "golang.org/x/net/context" ) @@ -245,6 +246,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } + c.Conf.Debug = p.debug err = c.Load(p.configPath, keyPass) if err != nil { logrus.Errorf("Error loading %s, %s", p.configPath, err) @@ -295,9 +297,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if 0 < len(servernames) { c.Conf.Servers = target } + logrus.Debugf("%s", pp.Sprintf("%v", target)) c.Conf.Lang = p.lang - c.Conf.Debug = p.debug c.Conf.DebugSQL = p.debugSQL // logger diff --git a/config/config.go b/config/config.go index 6cd6141ebc..f62584db7b 100644 --- a/config/config.go +++ b/config/config.go @@ -188,7 +188,6 @@ type SlackConf struct { // Validate validates configuration func (c *SlackConf) Validate() (errs []error) { - if !c.UseThisTime { return } @@ -228,7 +227,8 @@ type ServerInfo struct { KeyPath string KeyPassword string - CpeNames []string + CpeNames []string + DependencyCheckXMLPath string // Container Names or IDs Containers []string diff --git a/config/tomlloader.go b/config/tomlloader.go index 21e52640cb..6b3338f010 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -23,7 +23,7 @@ import ( "github.com/BurntSushi/toml" log "github.com/Sirupsen/logrus" - "github.com/k0kubun/pp" + "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" ) // TOMLLoader loads config @@ -31,7 +31,11 @@ type TOMLLoader struct { } // Load load the configuraiton TOML file specified by path arg. -func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { +func (c TOMLLoader) Load(pathToToml, keyPass string) error { + if Conf.Debug { + log.SetLevel(log.DebugLevel) + } + var conf Config if _, err := toml.DecodeFile(pathToToml, &conf); err != nil { log.Error("Load config failed", err) @@ -102,6 +106,23 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { s.CpeNames = d.CpeNames } + s.DependencyCheckXMLPath = v.DependencyCheckXMLPath + if len(s.DependencyCheckXMLPath) == 0 { + s.DependencyCheckXMLPath = d.DependencyCheckXMLPath + } + + // Load CPEs from OWASP Dependency Check XML + if len(s.DependencyCheckXMLPath) != 0 { + cpes, err := parser.Parse(s.DependencyCheckXMLPath) + if err != nil { + return fmt.Errorf( + "Failed to read OWASP Dependency Check XML: %s", err) + } + log.Infof("Loaded from OWASP Dependency Check XML: %s", + s.ServerName) + s.CpeNames = append(s.CpeNames, cpes...) + } + s.Containers = v.Containers if len(s.Containers) == 0 { s.Containers = d.Containers @@ -140,8 +161,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { servers[name] = s } - log.Debug("Config loaded") - log.Debugf("%s", pp.Sprintf("%v", servers)) Conf.Servers = servers - return + return nil } diff --git a/contrib/owasp-dependency-check/parser/parser.go b/contrib/owasp-dependency-check/parser/parser.go new file mode 100644 index 0000000000..c0e8e81884 --- /dev/null +++ b/contrib/owasp-dependency-check/parser/parser.go @@ -0,0 +1,64 @@ +package parser + +import ( + "encoding/xml" + "fmt" + "io/ioutil" + "os" + "sort" + "strings" +) + +type analysis struct { + Dependencies []dependency `xml:"dependencies>dependency"` +} + +type dependency struct { + Identifiers []identifier `xml:"identifiers>identifier"` +} + +type identifier struct { + Name string `xml:"name"` + Type string `xml:"type,attr"` +} + +func appendIfMissing(slice []string, str string) []string { + for _, s := range slice { + if s == str { + return slice + } + } + return append(slice, str) +} + +// Parse parses XML and collect list of cpe +func Parse(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return []string{}, fmt.Errorf("Failed to open: %s", err) + } + defer file.Close() + + b, err := ioutil.ReadAll(file) + if err != nil { + return []string{}, fmt.Errorf("Failed to read: %s", err) + } + + var anal analysis + if err := xml.Unmarshal(b, &anal); err != nil { + fmt.Errorf("Failed to unmarshal: %s", err) + } + + cpes := []string{} + for _, d := range anal.Dependencies { + for _, ident := range d.Identifiers { + if ident.Type == "cpe" { + name := strings.TrimPrefix(ident.Name, "(") + name = strings.TrimSuffix(name, ")") + cpes = appendIfMissing(cpes, name) + } + } + } + sort.Strings(cpes) + return cpes, nil +} From 234e312ee2bacbddedeee16c0b4aa64b0266f8a1 Mon Sep 17 00:00:00 2001 From: Tomohiro Miyakoshi Date: Fri, 28 Oct 2016 18:11:03 +0900 Subject: [PATCH 31/37] add '-ssh-external' option to prepare subcommand modify gofmt modify gofmt --- commands/prepare.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/commands/prepare.go b/commands/prepare.go index 8f34993389..0f4354e15d 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -37,6 +37,8 @@ type PrepareCmd struct { askSudoPassword bool askKeyPassword bool + + sshExternal bool } // Name return subcommand name @@ -60,8 +62,9 @@ func (*PrepareCmd) Usage() string { [-config=/path/to/config.toml] [-ask-key-password] [-debug] + [-ssh-external] - [SERVER]... + [SERVER]... ` } @@ -88,6 +91,13 @@ func (p *PrepareCmd) SetFlags(f *flag.FlagSet) { false, "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication", ) + + f.BoolVar( + &p.sshExternal, + "ssh-external", + false, + "Use external ssh command. Default: Use the Go native implementation") + } // Execute execute @@ -133,6 +143,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } c.Conf.Debug = p.debug + c.Conf.SSHExternal = p.sshExternal // Set up custom logger logger := util.NewCustomLogger(c.ServerInfo{}) From 85e6d753c7a7f17adff191a0335b6cb8776d7ebc Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Mon, 31 Oct 2016 18:02:41 +0900 Subject: [PATCH 32/37] Update glide files --- glide.lock | 10 +++++++--- glide.yaml | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 86e4dcf6c1..103a8273ee 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 01cee4aa1d27c967f5f5165febfbd77fd0e89a6e7a9a89c84099c0d45dffd446 -updated: 2016-10-25T19:51:49.542919471+09:00 +hash: ca64aef6e9e94c7be91f79b88edb847363c8a5bd48da4ad27784e9342c8db6e2 +updated: 2016-10-31T18:00:30.466846691+09:00 imports: - name: github.com/asaskevich/govalidator version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 @@ -45,6 +45,8 @@ imports: version: ad4efe000aa550bb54918c06ebbadc0ff17687b9 - name: github.com/go-ini/ini version: 6e4869b434bd001f6983749881c7ead3545887d8 +- name: github.com/go-sql-driver/mysql + version: 2a6c6079c7eff49a7e9d641e109d922f124a3e4c - name: github.com/google/subcommands version: 1c7173745a6001f67d8d96ab4e178284c77f7759 - name: github.com/gosuri/uitable @@ -56,6 +58,8 @@ imports: version: f5387c492211eb133053880d23dfae62aa14123d - name: github.com/jinzhu/gorm version: c1b9cf186e4bcd8e5d566ef43f2ae2dfe22dc34e + subpackages: + - dialects/mysql - name: github.com/jinzhu/inflection version: 74387dc39a75e970e7a3ae6a3386b5bd2e5c5cff - name: github.com/jmespath/go-jmespath @@ -65,7 +69,7 @@ imports: - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary - version: 8465a01aad6bf864813d61daba2e6d0a8b6a07d9 + version: 4daeddcd31acdf75cf8454ec976718a09e9640f6 subpackages: - config - db diff --git a/glide.yaml b/glide.yaml index afc3e18e9c..4608cf17aa 100644 --- a/glide.yaml +++ b/glide.yaml @@ -12,6 +12,7 @@ import: - aws/credentials - aws/session - service/s3 +- package: github.com/boltdb/bolt - package: github.com/cenkalti/backoff - package: github.com/google/subcommands - package: github.com/gosuri/uitable @@ -22,6 +23,7 @@ import: - package: github.com/kotakanbe/go-cve-dictionary subpackages: - config + - db - models - package: github.com/kotakanbe/go-pingscanner - package: github.com/kotakanbe/logrus-prefixed-formatter From 9a7a30c0bc7aea6b6cc4bfc14ad8580d384e093a Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Tue, 1 Nov 2016 10:54:59 +0900 Subject: [PATCH 33/37] Fix README #234 --- README.ja.md | 11 ++++++++--- README.md | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.ja.md b/README.ja.md index 8aa35edb5e..ca4148c006 100644 --- a/README.ja.md +++ b/README.ja.md @@ -577,17 +577,22 @@ Prepareサブコマンドは、Vuls内部で利用する以下のパッケージ ``` $ vuls prepare -help -prepare - [-config=/path/to/config.toml] [-debug] +prepare: + prepare + [-config=/path/to/config.toml] [-ask-key-password] - [SERVER]... + [-debug] + [-ssh-external] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning -config string /path/to/toml (default "$PWD/config.toml") -debug debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation ``` ---- diff --git a/README.md b/README.md index 1493043167..7c189689e4 100644 --- a/README.md +++ b/README.md @@ -583,17 +583,22 @@ Prepare subcommand installs required packages on each server. ``` $ vuls prepare -help -prepare - [-config=/path/to/config.toml] [-debug] +prepare: + prepare + [-config=/path/to/config.toml] [-ask-key-password] - [SERVER]... + [-debug] + [-ssh-external] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning -config string /path/to/toml (default "$PWD/config.toml") -debug debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation ``` ---- From 7681b277cf1b0ce21164c8b37206f872d9c0e267 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Tue, 1 Nov 2016 11:22:24 +0900 Subject: [PATCH 34/37] Fix changelog cache bug on Ubuntu and Debian #235 --- scan/debian.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scan/debian.go b/scan/debian.go index 65a21d19f7..8230b29900 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -513,7 +513,7 @@ func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) str } changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name) if err != nil { - o.log.Warnf("Failed to get chnagelog. bucket: %s, key:%s, err: %s", + o.log.Warnf("Failed to get changelog. bucket: %s, key:%s, err: %s", meta.Name, pack.Name, err) return "" } @@ -541,11 +541,13 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { o.log.Warnf("Failed to SSH: %s", r) // Ignore this Error. return nil, nil - } - err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, r.Stdout) - if err != nil { - return nil, fmt.Errorf("Failed to put changelog into cache") + + if 0 < len(strings.TrimSpace(r.Stdout)) { + err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, r.Stdout) + if err != nil { + return nil, fmt.Errorf("Failed to put changelog into cache") + } } // No error will be returned. Only logging. return o.getCveIDFromChangelog(r.Stdout, pack.Name, pack.Version), nil From 83469ce5cccca325801e553ce930848cf6e4f9ce Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Tue, 1 Nov 2016 15:09:53 +0900 Subject: [PATCH 35/37] Update glide.lock --- glide.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 103a8273ee..32a30e26b9 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: ca64aef6e9e94c7be91f79b88edb847363c8a5bd48da4ad27784e9342c8db6e2 -updated: 2016-10-31T18:00:30.466846691+09:00 +updated: 2016-11-01T15:05:15.23083077+09:00 imports: - name: github.com/asaskevich/govalidator version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 @@ -69,7 +69,7 @@ imports: - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary - version: 4daeddcd31acdf75cf8454ec976718a09e9640f6 + version: d0d8b0d3eee8022395d37edd95e88af7f5f970ad subpackages: - config - db From c5cfac62da0c53cf8908292e4aa6bc160e7ea95a Mon Sep 17 00:00:00 2001 From: Sadayuki Matsuno Date: Tue, 1 Nov 2016 20:24:37 +0900 Subject: [PATCH 36/37] fix readme --- setup/docker/README.md | 72 +++++++++++++++++++ .../docker/go-cve-dictionary/latest/README.md | 12 ++++ setup/docker/vuls/latest/README.md | 12 ++++ 3 files changed, 96 insertions(+) diff --git a/setup/docker/README.md b/setup/docker/README.md index 76a577dee6..d72aae70c2 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -13,6 +13,78 @@ This is the Git repo of the official Docker image for vuls. This image version is same as the github repository version. +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. + +1. Confirm your vuls version + +- go-cve-dictionary + +```console +$ docker run --rm vuls/go-cve-dictionary -v + +go-cve-dictionary v0.0.xxx xxxx +``` + +- vuls + +```console +$ docker run --rm vuls/vuls -v + +vuls v0.0.xxx xxxx +``` + +2. Remove your old docker images + +- go-cve-dictionary + +``` +$ docker rmi vuls/go-cve-dictionary +``` + +``` +$ docker rmi vuls/vuls +``` + +- vuls + +``` +$ docker rmi vuls/vuls +``` + +3. Pull new vuls docker images + +- go-cve-dictionary + +``` +$ docker pull vuls/go-cve-dictionary +``` + +- vuls + +``` +$ docker pull vuls/vuls +``` + +4. Confirm your vuls version + +```console +$ docker run --rm vuls/go-cve-dictionary -v + +go-cve-dictionary v0.1.xxx xxxx +``` + +- vuls + +```console +$ docker run --rm vuls/vuls -v + +vuls v0.1.xxx xxxx +``` + + # How to use this image 1. fetch nvd (vuls/go-cve-dictionary) diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md index 9e16d257c6..588294ae8b 100644 --- a/setup/docker/go-cve-dictionary/latest/README.md +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -12,6 +12,18 @@ See the [Hub page](https://hub.docker.com/r/vuls/go-cve-dictionary/) for the ful This image is built per commit. If you want to use the latest docker image, you should remove the existing image, and pull it once again. +- Remove old docker image + +``` +$ docker rmi vuls/go-cve-dictionary +``` + +- Pull new docker image + +``` +$ docker pull vuls/go-cve-dictionary +``` + # What is go-cve-dictionary? This is tool to build a local copy of the NVD (National Vulnerabilities Database) [1] and the Japanese JVN [2], which contain security vulnerabilities according to their CVE identifiers [3] including exhaustive information and a risk score. The local copy is generated in sqlite format, and the tool has a server mode for easy querying. diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md index 6b878599c8..0fe862850e 100644 --- a/setup/docker/vuls/latest/README.md +++ b/setup/docker/vuls/latest/README.md @@ -12,6 +12,18 @@ See the [Hub page](https://hub.docker.com/r/vuls/vuls/) for the full readme on h This image is built per commit. If you want to use the latest docker image, you should remove the existing image, and pull it once again. +- Remove old docker image + +``` +$ docker rmi vuls/vuls +``` + +- Pull new docker image + +``` +$ docker pull vuls/vuls +``` + # What is Vuls? Vuls is the Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. From e0d8147104726cddbc18b6588161b04eaad05cb0 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 2 Nov 2016 13:45:37 +0900 Subject: [PATCH 37/37] Update README #225 --- README.ja.md | 100 ++++++++++----------------------------------------- README.md | 53 ++++++++++----------------- 2 files changed, 38 insertions(+), 115 deletions(-) diff --git a/README.ja.md b/README.ja.md index ca4148c006..bc4b5da9ff 100644 --- a/README.ja.md +++ b/README.ja.md @@ -122,7 +122,7 @@ VulsはSSHパスワード認証をサポートしていない。SSH公開鍵鍵 Vulsセットアップに必要な以下のソフトウェアをインストールする。 -- SQLite3 +- SQLite3 or MySQL - git - gcc - go v1.7.1 or later @@ -606,7 +606,8 @@ scan: [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -653,7 +654,9 @@ scan: -containers-only Scan concontainers Only. Default: Scan both of hosts and containers -cve-dictionary-dbpath string - /path/to/sqlite3 (For get cve detail from cve.sqlite3) + /path/to/sqlite3 (For get cve detail from cve.sqlite3) + -cve-dictionary-dbtype string + DB type for fetching CVE dictionary (sqlite3 or mysql) (default "sqlite3") -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float @@ -869,6 +872,14 @@ optional = [ ] ``` +## Example: Use MySQL as a DB storage back-end + +``` +$ vuls scan \ + -cve-dictionary-dbtype=mysql \ + -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true" +``` + ---- # Usage: Scan vulnerability of non-OS package @@ -1028,89 +1039,14 @@ $ vuls scan -cve-dictionary-url=http://192.168.0.1:1323 # Usage: Update NVD Data -``` -$ go-cve-dictionary fetchnvd -h -fetchnvd: - fetchnvd - [-last2y] - [-dbpath=/path/to/cve.sqlite3] - [-debug] - [-debug-sql] - - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -last2y - Refresh NVD data in the last two years. -``` - -- Fetch data of the entire period - -``` -$ for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i; done -``` - -- Fetch data in the last 2 years - -``` -$ go-cve-dictionary fetchnvd -last2y -``` +see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-nvd-data) ---- # レポートの日本語化 -- JVNから日本語の脆弱性情報を取得 - ``` - $ go-cve-dictionary fetchjvn -h - fetchjvn: - fetchjvn - [-latest] - [-last2y] - [-years] 1998 1999 ... - [-dbpath=$PWD/cve.sqlite3] - [-http-proxy=http://192.168.0.1:8080] - [-debug] - [-debug-sql] - - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -http-proxy string - http://proxy-url:port (default: empty) - -last2y - Refresh JVN data in the last two years. - -latest - Refresh JVN data for latest. - -years - Refresh JVN data of specific years. - - ``` - -- すべての期間の脆弱性情報を取得(10分未満) - ``` - $ for i in {1998..2016}; do go-cve-dictionary fetchjvn -years $i; done - ``` - -- 2年分の情報を取得 - ``` - $ go-cve-dictionary fetchjvn -last2y - ``` - -- 最新情報のみ取得 - ``` - $ go-cve-dictionary fetchjvn -latest - ``` +see [go-cve-dictionary#usage-fetch-jvn-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-jvn-data) -- 脆弱性情報の自動アップデート -Cronなどのジョブスケジューラを用いて実現可能。 --latestオプションを指定して夜間の日次実行を推奨。 ## fetchnvd, fetchjvnの実行順序の注意 @@ -1144,10 +1080,11 @@ slack, emailは日本語対応済み TUIは日本語表示未対応 # Update Vuls With Glide - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 DB file. +If the DB schema was changed, please specify new SQLite3 or MySQL DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull +$ mv vendor /tmp/foo $ make install ``` @@ -1155,6 +1092,7 @@ $ make install ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull +$ mv vendor /tmp/bar $ make install ``` - バイナリファイルは`$GOPARH/bin`以下に作成される diff --git a/README.md b/README.md index 7c189689e4..75eb206561 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ And also, SUDO with password is not supported for security reasons. So you have Vuls requires the following packages. -- SQLite3 +- SQLite3 or MySQL - git - gcc - go v1.7.1 or later @@ -284,7 +284,7 @@ see https://github.com/future-architect/vuls/tree/master/setup/docker ![Vuls-Architecture](img/vuls-architecture.png) ## [go-cve-dictinary](https://github.com/kotakanbe/go-cve-dictionary) -- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3. +- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3 or MySQL. ## Scanning Flow ![Vuls-Scan-Flow](img/vuls-scan-flow.png) @@ -613,7 +613,8 @@ scan: [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -660,7 +661,9 @@ scan: -containers-only Scan concontainers Only. Default: Scan both of hosts and containers -cve-dictionary-dbpath string - /path/to/sqlite3 (For get cve detail from cve.sqlite3) + /path/to/sqlite3 (For get cve detail from cve.sqlite3) + -cve-dictionary-dbtype string + DB type for fetching CVE dictionary (sqlite3 or mysql) (default "sqlite3") -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float @@ -869,6 +872,14 @@ optional = [ ] ``` +## Example: Use MySQL as a DB storage back-end + +``` +$ vuls scan \ + -cve-dictionary-dbtype=mysql \ + -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true" +``` + ---- # Usage: Scan vulnerabilites of non-OS packages @@ -1027,46 +1038,19 @@ $ vuls scan -cve-dictionary-url=http://192.168.0.1:1323 # Usage: Update NVD Data -``` -$ go-cve-dictionary fetchnvd -h -fetchnvd: - fetchnvd - [-last2y] - [-dbpath=/path/to/cve.sqlite3] - [-debug] - [-debug-sql] +see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-nvd-data) - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -last2y - Refresh NVD data in the last two years. -``` - -- Fetch data of the entire period - -``` -$ go-cve-dictionary fetchnvd -entire -``` - -- Fetch data in the last 2 years - -``` -$ go-cve-dictionary fetchnvd -last2y -``` ---- # Update Vuls With Glide - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 DB file. +If the DB schema was changed, please specify new SQLite3 or MySQL DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull +$ mv vendor /tmp/foo $ make install ``` @@ -1074,6 +1058,7 @@ $ make install ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull +$ mv vendor /tmp/bar $ make install ``` Binary file was built under $GOPARH/bin