From 1ffe4d3eafe981607defd6f989ad48ce25b0e524 Mon Sep 17 00:00:00 2001 From: Eugene Knyazkov Date: Fri, 23 Sep 2016 01:10:08 +0300 Subject: [PATCH 1/9] Initial commit --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..aa39642 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# autopilotpattern-telegraf \ No newline at end of file From f84f4be693b3fc59dae66d7426717fe790e4596b Mon Sep 17 00:00:00 2001 From: Eugene Knyazkov Date: Fri, 23 Sep 2016 01:14:51 +0300 Subject: [PATCH 2/9] Initial revision of autopilot docker container for telegraf --- Dockerfile | 65 +++++++ LICENSE | 373 ++++++++++++++++++++++++++++++++++++++++ README.md | 80 ++++++++- _env.telegraf | 5 + bin/reload.sh | 55 ++++++ bin/sensor.sh | 36 ++++ docker-compose.yml | 78 +++++++++ etc/containerpilot.json | 50 ++++++ etc/telegraf.conf | 215 +++++++++++++++++++++++ etc/telegraf.ctmpl | 229 ++++++++++++++++++++++++ setup.sh | 129 ++++++++++++++ 11 files changed, 1314 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 _env.telegraf create mode 100755 bin/reload.sh create mode 100755 bin/sensor.sh create mode 100644 docker-compose.yml create mode 100644 etc/containerpilot.json create mode 100644 etc/telegraf.conf create mode 100644 etc/telegraf.ctmpl create mode 100755 setup.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f9dd468 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,65 @@ +#FROM telegraf:1.0-alpine +FROM telegraf:1.0 + +# Reset to root user to do some installs +USER root + +# Install packages +RUN apt-get update && apt-get -y install \ + bash \ + curl \ + unzip \ + netcat-openbsd \ + && rm -rf /var/cache/apt/* /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Add ContainerPilot and its configuration +# Releases at https://github.com/joyent/containerpilot/releases +ENV CONTAINERPILOT_VER 2.3.0 +ENV CONTAINERPILOT file:///etc/containerpilot.json + +RUN export CONTAINERPILOT_CHECKSUM=ec9dbedaca9f4a7a50762f50768cbc42879c7208 \ + && curl --retry 7 --fail -Lso /tmp/containerpilot.tar.gz \ + "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \ + && echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \ + && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \ + && rm /tmp/containerpilot.tar.gz + +# The our helper/glue scripts and configuration for this specific app +COPY bin /usr/local/bin +COPY etc /etc + +# Install Consul +# Releases at https://releases.hashicorp.com/consul +RUN export CONSUL_VERSION=0.6.4 \ + && export CONSUL_CHECKSUM=abdf0e1856292468e2c9971420d73b805e93888e006c76324ae39416edcf0627 \ + && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \ + && echo "${CONSUL_CHECKSUM} /tmp/consul.zip" | sha256sum -c \ + && unzip /tmp/consul -d /usr/local/bin \ + && rm /tmp/consul.zip \ + && mkdir /config + +# Create empty directories for Consul config and data +RUN mkdir -p /etc/consul \ + && chown -R root /etc/consul \ + && mkdir -p /var/lib/consul \ + && chown -R root /var/lib/consul + +# Install Consul template +# Releases at https://releases.hashicorp.com/consul-template/ +RUN export CONSUL_TEMPLATE_VERSION=0.14.0 \ + && export CONSUL_TEMPLATE_CHECKSUM=7c70ea5f230a70c809333e75fdcff2f6f1e838f29cfb872e1420a63cdf7f3a78 \ + && curl --retry 7 --fail -Lso /tmp/consul-template.zip "https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip" \ + && echo "${CONSUL_TEMPLATE_CHECKSUM} /tmp/consul-template.zip" | sha256sum -c \ + && unzip /tmp/consul-template.zip -d /usr/local/bin \ + && rm /tmp/consul-template.zip + +# Reset entrypoint from base image +ENTRYPOINT [] + +# Run telegraf +USER root +CMD ["/usr/local/bin/containerpilot", \ + "/entrypoint.sh", \ + "telegraf", \ + "-config", \ + "/etc/telegraf.conf"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a612ad9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md index aa39642..0c31351 100644 --- a/README.md +++ b/README.md @@ -1 +1,79 @@ -# autopilotpattern-telegraf \ No newline at end of file +# Autopilot telegraf +*Containerized telegraf server, based on the official telegraf/1.0 Docker image, adding [ContainerPilot](https://www.joyent.com/containerpilot) to announce this container's telegraf service to a Service Discovery layer, such as Consul or etcd. + +### Usage +Include this image in your Docker Compose project, query Consul for it's IP address and use it in your configurations, easily done via [Consul-Template](https://github.com/hashicorp/consul-template). The default ContainerPilot configuration talks to Consul and assumes the IP address to access consul is passed to the container in an envrionment varible, $CONSUL (or via docker link consul) + +Configuration of telegraf is managed via ContainerPilot `preStart` or `onChange` handlers. + +Telegraf output is convigured with InfluxDB output plugin. By default telegraf is looking for InfluxDB container started in the same cluster, but it's possible to point Telegraf to remove InfluxDB server by uncommenting and setting up INFLUXDB_HOST variable in env.telegraf file + +Telegraf input sources configured with prometheus input plugin and represent a list of urls pointing to container-pilot telemetry endpoints (http://container-ip:9090/metrics). Input sources reloaded automatically with `onChange` event handler. + +### Hello world example + +1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). +1. Install the [Docker Toolbox](https://docs.docker.com/installation/mac/) (including `docker` and `docker-compose`) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool) (`triton` replaces our old `sdc-*` CLI tools). +1. [Configure Docker and Docker Compose for use with Joyent.](https://docs.joyent.com/public-cloud/api-access/docker) + +Check that everything is configured correctly by running `./setup.sh`. This will check that your environment is setup correctly and will create an `_env` file that includes injecting an environment variable for the Consul hostname into the Telegraf and Nginx containers so we can take advantage of [Triton Container Name Service (CNS)](https://www.joyent.com/blog/introducing-triton-container-name-service). + +Start everything: + +```bash +docker-compose build +docker-compose up -d +``` +In result we'll have 3 containers running: +- consul +- telegraf_nginx_1 - nginx web-server is used just for demo purposes to scale and provide telemetry +- influxdb - currently running locally, but it's possible to connect with existing influxdb server +- telegraf + +To verify telegraf container status you can check container log (there should be a list of records, which indicate attempts to join new input source): +```bash +docker logs telegraf 2>&1 | grep EventMemberJoin +``` +Also you check the list of input source urls for telemetry currently used by telegraf with the following command: +```bash +docker exec -i -t telegraf /bin/grep :9090 /etc/telegraf.conf +``` +the list of urls includes consul container(first one in outout), telegraf container(localhost) and all other urls are nginx-container urls. +So you can check the number of urls in output, substruct 2 and it should give you a number of nginx containers + + +Lets scale up number of nginx containers to 3, wait for 15 seconds (give some time to telegraf to reconfigure itself) and check the number of input urls (or EventMemberJoin events in logs) +```bash +docker-compose scale nginx=3 +sleep 15 + +# check source urls +docker exec -i -t telegraf /bin/grep :9090 /etc/telegraf.conf + +# check logs +docker logs telegraf 2>&1 | grep EventMemberJoin +``` + +Lets scale down number of nginx containers to 1, wait for 15 seconds and check the number of input urls again: +```bash +docker-compose scale nginx=1 +sleep 15 + +# check source urls +docker exec -i -t telegraf /bin/grep :9090 /etc/telegraf.conf +``` + +Finally you can check actual result of telemery aggregation(via telegraf) on InfluxDB server. +You have to open InfluxDB UI with the following command: +```bash +open "http://$(triton ip influxdb):8083/" +``` +choose 'telegraf' database in dropdown located on the top-right corner, type and execute a query +``` +SHOW MEASUREMENTS +``` +there should be a record like 'nginx_connections_load' which represents data coming from nginx telemetry. +And the following query should display a list of nginx specfic telemetry recorods collected during last 5 minutes: +``` +SELECT * FROM nginx_connections_load WHERE time > now() - 5m +``` diff --git a/_env.telegraf b/_env.telegraf new file mode 100644 index 0000000..f2ed4de --- /dev/null +++ b/_env.telegraf @@ -0,0 +1,5 @@ +# InfluxDB settings +# (uncomment to change default influxdb host/database to custom url) +#INFLUXDB_HOST=influxdb # docker alias or real hostname +#INFLUXDB_DATABASE=telegraf +#INFLUXDB_DATA_ENGINE=tsm1 diff --git a/bin/reload.sh b/bin/reload.sh new file mode 100755 index 0000000..2d5c77c --- /dev/null +++ b/bin/reload.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +SERVICE_NAME=${SERVICE_NAME:-telegraf} +CONSUL=${CONSUL:-consul} + +# Render Telegraf configuration template using values from Consul, +# but do not reload because Telegraf has't started yet +preStart() { + # sleep 5 # give some time for other containerpilots to start before rendering config + consul-template \ + -once \ + -dedup \ + -consul ${CONSUL}:8500 \ + -template "/etc/telegraf.ctmpl:/etc/telegraf.conf" +} + +# Render Telegraf configuration template using values from Consul, +# then gracefully reload Telegraf +onChange() { + consul-template \ + -once \ + -dedup \ + -consul ${CONSUL}:8500 \ + -template "/etc/telegraf.ctmpl:/etc/telegraf.conf:/usr/local/bin/reload.sh reloadConfig" +} + +# Telegraf reload th SIGHUP +# Note: if we fire SIGHUP vs node before it has a chance to register the +# signal handler, then it will immediately exit. This ensures that +# the process is listening on port 8094 which should only be the +# case after we have the signal handler loaded. +reloadConfig() { + while : + do + netstat -ln | grep -q 8094 && pkill -SIGHUP telegraf && break + done +} +help() { + echo "Usage: ./reload.sh preStart => first-run configuration for Telegraf" + echo " ./reload.sh onChange => [default] update Telegraf config on upstream changes" + echo " ./reload.sh reloadConfig => reload Telegraf config on upstream changes" +} + +until + cmd=$1 + if [ -z "$cmd" ]; then + onChange + fi + shift 1 + $cmd "$@" + [ "$?" -ne 127 ] +do + onChange + exit +done diff --git a/bin/sensor.sh b/bin/sensor.sh new file mode 100755 index 0000000..6c1d595 --- /dev/null +++ b/bin/sensor.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +help() { + echo 'Uses cli tools free and top to determine current CPU and memory usage' + echo 'for the telemetry service.' +} + +# memory usage in percent +sys_memory() { + # awk oneliner to get memory usage + # free -m | awk 'NR==2{printf "Memory Usage: %s/%sMB (%.2f%%)\n", $3,$2,$3*100/$2 }' + # output: + # Memory Usage: 15804/15959MB (99.03%) + (>&2 echo "sys memory check fired") + local memory=$(free -m | awk 'NR==2{printf "%.2f", $3*100/$2 }') + echo ${memory} +} + +# cpu load +sys_cpu() { + # oneliner to display cpu load + # top -bn1 | grep load | awk '{printf "CPU Load: %.2f\n", $(NF-2)}' + (>&2 echo "sys cpu check fired") + local cpuload=$(top -bn1 | grep load | awk '{printf "%.2f", $(NF-2)}') + echo ${cpuload} +} + +cmd=$1 +if [ ! -z "$cmd" ]; then + shift 1 + $cmd "$@" + exit +fi + +help diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..022d65a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,78 @@ +# Telegraf autopilotpattern demo + +# Consul - start with a single host which will bootstrap the cluster. +# In production we'll want to use an HA cluster. +consul: + container_name: consul + image: progrium/consul:latest + restart: always + mem_limit: 128m + expose: + - 53 + - 8300 + - 8301 + - 8302 + - 8400 + - 8500 + ports: + - 8500 # expose only Consul's UI on the public IP + dns: + - 127.0.0.1 + labels: + - triton.cns.services=consul + command: -server -bootstrap -ui-dir /ui + +# Telegraf should autodiscovery all autopilot containers, +# collect telemetry from there, and send output to influxdb. +telegraf: + container_name: telegraf + #image: autopilotpattern/telegraf + build: . # for now use local image build + mem_limit: 128m + expose: + - 8094 # necessary for healthcheck + - 9090 # so we can see telemetry + restart: always + links: + - consul:consul + - influxdb:influxdb + environment: + - CONSUL_AGENT=1 + - CONSUL=consul + - INFLUXDB=influxdb + env_file: + - _env + - _env.telegraf + labels: + - triton.cns.services=telegraf + +# InfluxDB +influxdb: + container_name: influxdb + image: influxdb:latest + mem_limit: 512m + ports: + - "8083:8083" + - "8086:8086" + - "8090:8090" + env_file: + - '_env' + +# Ngix included for demonstration of input sources for telegraf +# we can scale it and demonstrate auto capturing of nginx containers telemetry by telegraf +nginx: + image: autopilotpattern/nginx + restart: always + mem_limit: 128m + ports: + - 80 # http port + - 9090 # so we can see telemetry + links: + - consul:consul + environment: + - CONSUL_AGENT=1 + - CONSUL=consul + - BACKEND=consul # backend is required, so lets point to consul just for demo purposes + env_file: _env + labels: + - triton.cns.services=nginx diff --git a/etc/containerpilot.json b/etc/containerpilot.json new file mode 100644 index 0000000..154cbde --- /dev/null +++ b/etc/containerpilot.json @@ -0,0 +1,50 @@ +{ + "consul": "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500", + "preStart": "/usr/local/bin/reload.sh preStart", + "services": [ + { + "name": "telegraf", + "port": 8094, + "health": "nc -vz localhost 8094", + "poll": 10, + "ttl": 25 + } + ], + "backends": [ + { + "name": "nginx", + "poll": 7, + "onChange": "/usr/local/bin/reload.sh onChange" + } + ], + "coprocesses": [{{ if .CONSUL_AGENT }} + { + "command": ["/usr/local/bin/consul", "agent", + "-data-dir=/var/lib/consul", + "-config-dir=/etc/consul", + "-rejoin", + "-retry-join", "{{ .CONSUL }}", + "-retry-max", "10", + "-retry-interval", "10s"], + "restarts": "unlimited" + }{{ end }}], + "telemetry": { + "port": 9090, + "sensors": [ + { + "name": "telegraf_sys_memory_percent", + "help": "percentage of memory used", + "type": "gauge", + "poll": 5, + "check": ["/usr/local/bin/sensor.sh", "sys_memory"] + }, + { + "name": "telegraf_sys_cpu_load", + "help": "cpu load", + "type": "gauge", + "poll": 5, + "check": ["/usr/local/bin/sensor.sh", "sys_cpu"] + } + ] + } +} diff --git a/etc/telegraf.conf b/etc/telegraf.conf new file mode 100644 index 0000000..52a5815 --- /dev/null +++ b/etc/telegraf.conf @@ -0,0 +1,215 @@ +# Telegraf Configuration + +# Global tags can be specified here in key="value" format. +[global_tags] + # dc = "us-east-1" # will tag all metrics with dc=us-east-1 + # rack = "1a" + ## Environment variables can be used as tags, and throughout the config file + # user = "$USER" + + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at + ## most metric_batch_size metrics. + metric_batch_size = 1000 + ## For failed writes, telegraf will cache metric_buffer_limit metrics for each + ## output, and will flush this buffer on a successful write. Oldest metrics + ## are dropped first when this buffer fills. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Default flushing interval for all outputs. You shouldn't set this below + ## interval. Maximum flush_interval will be flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## By default, precision will be set to the same timestamp order as the + ## collection interval, with the maximum being 1s. + ## Precision will NOT be used for service inputs, such as logparser and statsd. + ## Valid values are "ns", "us" (or "µs"), "ms", "s". + precision = "" + ## Run telegraf in debug mode + debug = false + ## Run telegraf in quiet mode + quiet = false + ## Override default hostname, if empty use os.Hostname() + hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + omit_hostname = false + +############################################################################### +# OUTPUT PLUGINS # +############################################################################### + +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + ## The full HTTP or UDP endpoint URL for your InfluxDB instance. + ## Multiple urls can be specified as part of the same cluster, + ## this means that only ONE of the urls will be written to each interval. + # urls = ["udp://localhost:8089"] # UDP endpoint example + urls = ["http://influxdb:8086"] # required (default) + + ## The target database for metrics (telegraf will create it if not exists). + database = "telegraf" # required + + ## Retention policy to write to. Empty string writes to the default rp. + retention_policy = "" + ## Write consistency (clusters only), can be: "any", "one", "quorum", "all" + write_consistency = "any" + + ## Write timeout (for the InfluxDB client), formatted as a string. + ## If not provided, will default to 5s. 0s means no timeout (not recommended). + timeout = "5s" + # username = "telegraf" + # password = "metricsmetricsmetricsmetrics" + ## Set the user agent for HTTP POSTs (can be useful for log differentiation) + # user_agent = "telegraf" + ## Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) + # udp_payload = 512 + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false + +############################################################################### +# INPUT PLUGINS # +############################################################################### + +# Read metrics about cpu usage +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + ## Whether to report total system cpu stats or not + totalcpu = true + ## Comment this line if you want the raw CPU time metrics + fielddrop = ["time_*"] + + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## By default, telegraf gather stats for all mountpoints. + ## Setting mountpoints will restrict the stats to the specified mountpoints. + # mount_points = ["/"] + + ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually + ## present on /run, /var/run, /dev/shm or /dev). + ignore_fs = ["tmpfs", "devtmpfs"] + + +# Read metrics about disk IO by device +[[inputs.diskio]] + ## By default, telegraf will gather stats for all devices including + ## disk partitions. + ## Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb"] + ## Uncomment the following line if you need disk serial numbers. + # skip_serial_number = false + + +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + + +# Get the number of processes and group them by status +[[inputs.processes]] + # no configuration + + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + + +# Read metrics about system load & uptime +[[inputs.system]] + # no configuration + +# # Generic TCP listener +[[inputs.tcp_listener]] + ## Address and port to host TCP listener on + service_address = ":8094" +# +# ## Number of TCP messages allowed to queue up. Once filled, the +# ## TCP listener will start dropping packets. + allowed_pending_messages = 10000 +# +# ## Maximum number of concurrent TCP connections to allow + max_tcp_connections = 250 +# +# ## Data format to consume. +# ## Each data format has it's own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" + + +# # Generic UDP listener +[[inputs.udp_listener]] + ## Address and port to host UDP listener on + service_address = ":8092" +# +# ## Number of UDP messages allowed to queue up. Once filled, the +# ## UDP listener will start dropping packets. + allowed_pending_messages = 10000 +# +# ## Data format to consume. +# ## Each data format has it's own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" + + +# # Gather health check statuses from services registered in Consul +# [[inputs.consul]] +# ## Most of these values defaults to the one configured on a Consul's agent level. +# ## Optional Consul server address (default: "localhost") +# # address = "localhost" +# ## Optional URI scheme for the Consul server (default: "http") +# # scheme = "http" +# ## Optional ACL token used in every request (default: "") +# # token = "" +# ## Optional username used for request HTTP Basic Authentication (default: "") +# # username = "" +# ## Optional password used for HTTP Basic Authentication (default: "") +# # password = "" +# ## Optional data centre to query the health checks from (default: "") +# # datacentre = "" + +# # Read metrics from one or many prometheus clients +[[inputs.prometheus]] + ## An array of urls to scrape metrics from. + urls = ["http://localhost:9090/metrics"] # default + +# ## Use bearer token for authorization +# # bearer_token = /path/to/bearer/token +# +# ## Optional SSL Config +# # ssl_ca = /path/to/cafile +# # ssl_cert = /path/to/certfile +# # ssl_key = /path/to/keyfile +# ## Use SSL but skip chain & host verification +# # insecure_skip_verify = false diff --git a/etc/telegraf.ctmpl b/etc/telegraf.ctmpl new file mode 100644 index 0000000..c4d0fc3 --- /dev/null +++ b/etc/telegraf.ctmpl @@ -0,0 +1,229 @@ +# Telegraf Configuration + +# Global tags can be specified here in key="value" format. +[global_tags] + # dc = "us-east-1" # will tag all metrics with dc=us-east-1 + # rack = "1a" + ## Environment variables can be used as tags, and throughout the config file + # user = "$USER" + + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at + ## most metric_batch_size metrics. + metric_batch_size = 1000 + ## For failed writes, telegraf will cache metric_buffer_limit metrics for each + ## output, and will flush this buffer on a successful write. Oldest metrics + ## are dropped first when this buffer fills. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Default flushing interval for all outputs. You shouldn't set this below + ## interval. Maximum flush_interval will be flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## By default, precision will be set to the same timestamp order as the + ## collection interval, with the maximum being 1s. + ## Precision will NOT be used for service inputs, such as logparser and statsd. + ## Valid values are "ns", "us" (or "µs"), "ms", "s". + precision = "" + ## Run telegraf in debug mode + debug = false + ## Run telegraf in quiet mode + quiet = false + ## Override default hostname, if empty use os.Hostname() + hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + omit_hostname = false + +############################################################################### +# OUTPUT PLUGINS # +############################################################################### + +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + ## The full HTTP or UDP endpoint URL for your InfluxDB instance. + ## Multiple urls can be specified as part of the same cluster, + ## this means that only ONE of the urls will be written to each interval. + # urls = ["udp://localhost:8089"] # UDP endpoint example + {{ $influxdbhost := env "INFLUXDB_HOST" }} + {{ if $influxdbhost }} + urls = ["http://{{ $influxdbhost }}:8086"] # required + {{ else }} + urls = ["http://influxdb:8086"] # required + {{ end }} + + ## The target database for metrics (telegraf will create it if not exists). + {{ $influxdbdatabase := env "INFLUXDB_DATABASE" }} + {{ if $influxdbdatabase }} + database = "{{ $influxdbdatabase }}" # required + {{else}} + database = "telegraf" # required (default) + {{ end }} + + ## Retention policy to write to. Empty string writes to the default rp. + retention_policy = "" + ## Write consistency (clusters only), can be: "any", "one", "quorum", "all" + write_consistency = "any" + + ## Write timeout (for the InfluxDB client), formatted as a string. + ## If not provided, will default to 5s. 0s means no timeout (not recommended). + timeout = "5s" + # username = "telegraf" + # password = "metricsmetricsmetricsmetrics" + ## Set the user agent for HTTP POSTs (can be useful for log differentiation) + # user_agent = "telegraf" + ## Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) + # udp_payload = 512 + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false + +############################################################################### +# INPUT PLUGINS # +############################################################################### + +# Read metrics about cpu usage +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + ## Whether to report total system cpu stats or not + totalcpu = true + ## Comment this line if you want the raw CPU time metrics + fielddrop = ["time_*"] + + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## By default, telegraf gather stats for all mountpoints. + ## Setting mountpoints will restrict the stats to the specified mountpoints. + # mount_points = ["/"] + + ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually + ## present on /run, /var/run, /dev/shm or /dev). + ignore_fs = ["tmpfs", "devtmpfs"] + + +# Read metrics about disk IO by device +[[inputs.diskio]] + ## By default, telegraf will gather stats for all devices including + ## disk partitions. + ## Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb"] + ## Uncomment the following line if you need disk serial numbers. + # skip_serial_number = false + + +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + + +# Get the number of processes and group them by status +[[inputs.processes]] + # no configuration + + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + + +# Read metrics about system load & uptime +[[inputs.system]] + # no configuration + +# # Generic TCP listener +[[inputs.tcp_listener]] + ## Address and port to host TCP listener on + service_address = ":8094" +# +# ## Number of TCP messages allowed to queue up. Once filled, the +# ## TCP listener will start dropping packets. + allowed_pending_messages = 10000 +# +# ## Maximum number of concurrent TCP connections to allow + max_tcp_connections = 250 +# +# ## Data format to consume. +# ## Each data format has it's own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" + + +# # Generic UDP listener +[[inputs.udp_listener]] + ## Address and port to host UDP listener on + service_address = ":8092" +# +# ## Number of UDP messages allowed to queue up. Once filled, the +# ## UDP listener will start dropping packets. + allowed_pending_messages = 10000 +# +# ## Data format to consume. +# ## Each data format has it's own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" + + +# # Gather health check statuses from services registered in Consul +# [[inputs.consul]] +# ## Most of these values defaults to the one configured on a Consul's agent level. +# ## Optional Consul server address (default: "localhost") +# # address = "localhost" +# ## Optional URI scheme for the Consul server (default: "http") +# # scheme = "http" +# ## Optional ACL token used in every request (default: "") +# # token = "" +# ## Optional username used for request HTTP Basic Authentication (default: "") +# # username = "" +# ## Optional password used for HTTP Basic Authentication (default: "") +# # password = "" +# ## Optional data centre to query the health checks from (default: "") +# # datacentre = "" + +# # Read metrics from one or many prometheus clients +[[inputs.prometheus]] + ## An array of urls to scrape metrics from. + {{ if service "containerpilot" }} + urls = [{{range $i, $e := service "containerpilot"}}{{if ne $i 0}}, {{end}}"http://{{$e.Address}}:{{$e.Port}}/metrics"{{end}}, "http://localhost:9090/metrics"] + {{ else }} + urls = ["http://localhost:9090/metrics"] + {{ end }} + +# ## Use bearer token for authorization +# # bearer_token = /path/to/bearer/token +# +# ## Optional SSL Config +# # ssl_ca = /path/to/cafile +# # ssl_cert = /path/to/certfile +# # ssl_key = /path/to/keyfile +# ## Use SSL but skip chain & host verification +# # insecure_skip_verify = false diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..a27107f --- /dev/null +++ b/setup.sh @@ -0,0 +1,129 @@ +#!/bin/bash +set -e -o pipefail + +help() { + echo 'Usage ./setup.sh [-f docker-compose.yml] [-p project]' + echo + echo 'Checks that your Triton and Docker environment is sane and configures' + echo 'an environment file to use.' + echo + echo 'Optional flags:' + echo ' -f use this file as the docker-compose config file' + echo ' -p use this name as the project prefix for docker-compose' +} + + +# default values which can be overriden by -f or -p flags +export COMPOSE_PROJECT_NAME=telegraf +export COMPOSE_FILE= + +# give the docker remote api more time before timeout +export COMPOSE_HTTP_TIMEOUT=300 + +# populated by `check` function whenever we're using Triton +TRITON_USER= +TRITON_DC= +TRITON_ACCOUNT= + +# --------------------------------------------------- +# Top-level commmands + + +# Check for correct configuration +check() { + + command -v docker >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Docker is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://docs.joyent.com/public-cloud/api-access/docker' + exit 1 + } + command -v json >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Error! JSON CLI tool is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://apidocs.joyent.com/cloudapi/#getting-started' + exit 1 + } + + # if we're not testing on Triton, don't bother checking Triton config + if [ ! -z "${COMPOSE_FILE}" ]; then + exit 0 + fi + + command -v triton >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' + exit 1 + } + + # make sure Docker client is pointed to the same place as the Triton client + local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') + local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') + TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') + TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') + if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' + tput sgr0 # clear + echo + echo "Docker user: ${docker_user}" + echo "Triton user: ${TRITON_USER}" + echo "Docker data center: ${docker_dc}" + echo "Triton data center: ${TRITON_DC}" + exit 1 + fi + + local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') + if [ ! "true" == "$triton_cns_enabled" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! Triton CNS is required and not enabled.' + tput sgr0 # clear + echo + exit 1 + fi + + echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env +} + +# --------------------------------------------------- +# parse arguments + +while getopts "f:p:h" optchar; do + case "${optchar}" in + f) export COMPOSE_FILE=${OPTARG} ;; + p) export COMPOSE_PROJECT_NAME=${OPTARG} ;; + esac +done +shift $(expr $OPTIND - 1 ) + +until + cmd=$1 + if [ ! -z "$cmd" ]; then + shift 1 + $cmd "$@" + if [ $? == 127 ]; then + help + fi + exit + fi +do + echo +done + +# default behavior +check From 17f24e28fe7749b61ae32d4ea9ea0fb87bebbfb5 Mon Sep 17 00:00:00 2001 From: Eugene Knyazkov Date: Fri, 23 Sep 2016 01:18:48 +0300 Subject: [PATCH 3/9] Small text changes in README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0c31351..8cf2fb5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Autopilot telegraf -*Containerized telegraf server, based on the official telegraf/1.0 Docker image, adding [ContainerPilot](https://www.joyent.com/containerpilot) to announce this container's telegraf service to a Service Discovery layer, such as Consul or etcd. +Containerized telegraf server, based on the official telegraf/1.0 Docker image, adding [ContainerPilot](https://www.joyent.com/containerpilot) to announce this container's telegraf service to a Service Discovery layer, such as Consul or etcd. ### Usage Include this image in your Docker Compose project, query Consul for it's IP address and use it in your configurations, easily done via [Consul-Template](https://github.com/hashicorp/consul-template). The default ContainerPilot configuration talks to Consul and assumes the IP address to access consul is passed to the container in an envrionment varible, $CONSUL (or via docker link consul) @@ -24,7 +24,7 @@ Start everything: docker-compose build docker-compose up -d ``` -In result we'll have 3 containers running: +In result we'll have 4 containers running: - consul - telegraf_nginx_1 - nginx web-server is used just for demo purposes to scale and provide telemetry - influxdb - currently running locally, but it's possible to connect with existing influxdb server @@ -34,6 +34,8 @@ To verify telegraf container status you can check container log (there should be ```bash docker logs telegraf 2>&1 | grep EventMemberJoin ``` +it should display a list of members(input sources) recently added. + Also you check the list of input source urls for telemetry currently used by telegraf with the following command: ```bash docker exec -i -t telegraf /bin/grep :9090 /etc/telegraf.conf From 21cbbbdd3e398fe25c2ade7a7d1a3f2e360878eb Mon Sep 17 00:00:00 2001 From: Eugene Knyazkov Date: Fri, 23 Sep 2016 01:28:17 +0300 Subject: [PATCH 4/9] Added Configuration section to README.md --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 8cf2fb5..c8e0b2d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,12 @@ Telegraf output is convigured with InfluxDB output plugin. By default telegraf i Telegraf input sources configured with prometheus input plugin and represent a list of urls pointing to container-pilot telemetry endpoints (http://container-ip:9090/metrics). Input sources reloaded automatically with `onChange` event handler. +### Configuration + +Please run setup.sh to generate required _env file and configure CONSUL env variable. + +You can also check _env.telegraf file. By setting INFLUXDB_HOST variable there you can point telegraf to already running instance of InfluxDB (you have to remove influxdb section from docker-compose.yml in this case). + ### Hello world example 1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). @@ -79,3 +85,10 @@ And the following query should display a list of nginx specfic telemetry recorod ``` SELECT * FROM nginx_connections_load WHERE time > now() - 5m ``` + +At the end of test you can shutdown containers with the following command: +``` +docker-compose kill +``` + + From f8a6d7a3078132cde62139e5131cfb79f5f0dac4 Mon Sep 17 00:00:00 2001 From: Casey Bisson Date: Sun, 10 Dec 2017 18:43:53 -0800 Subject: [PATCH 5/9] wip --- .dockerignore | 5 +++++ .gitignore | 7 +++++++ _env.telegraf | 5 ----- docker-compose.yml => examples/triton/docker-compose.yml | 0 setup.sh => examples/triton/setup.sh | 0 5 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitignore delete mode 100644 _env.telegraf rename docker-compose.yml => examples/triton/docker-compose.yml (100%) rename setup.sh => examples/triton/setup.sh (100%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b5f5a08 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.DS_Store +**/.DS_Store +examples +_env* +**/_env* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..08ec465 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# credentials +_env* +manta* + +# macos frustration +.DS_Store + diff --git a/_env.telegraf b/_env.telegraf deleted file mode 100644 index f2ed4de..0000000 --- a/_env.telegraf +++ /dev/null @@ -1,5 +0,0 @@ -# InfluxDB settings -# (uncomment to change default influxdb host/database to custom url) -#INFLUXDB_HOST=influxdb # docker alias or real hostname -#INFLUXDB_DATABASE=telegraf -#INFLUXDB_DATA_ENGINE=tsm1 diff --git a/docker-compose.yml b/examples/triton/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to examples/triton/docker-compose.yml diff --git a/setup.sh b/examples/triton/setup.sh similarity index 100% rename from setup.sh rename to examples/triton/setup.sh From b6396a1e1d8c056493a0b6328ad1b0823aecb63e Mon Sep 17 00:00:00 2001 From: Casey Bisson Date: Sun, 10 Dec 2017 20:26:01 -0800 Subject: [PATCH 6/9] seems to be correct --- examples/triton/setup.sh | 87 +++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 46 deletions(-) diff --git a/examples/triton/setup.sh b/examples/triton/setup.sh index a27107f..0aa976b 100755 --- a/examples/triton/setup.sh +++ b/examples/triton/setup.sh @@ -17,45 +17,25 @@ help() { export COMPOSE_PROJECT_NAME=telegraf export COMPOSE_FILE= -# give the docker remote api more time before timeout -export COMPOSE_HTTP_TIMEOUT=300 - -# populated by `check` function whenever we're using Triton -TRITON_USER= -TRITON_DC= -TRITON_ACCOUNT= - # --------------------------------------------------- -# Top-level commmands +# Top-level commands # Check for correct configuration check() { - command -v docker >/dev/null 2>&1 || { - echo - tput rev # reverse - tput bold # bold - echo 'Docker is required, but does not appear to be installed.' - tput sgr0 # clear - echo 'See https://docs.joyent.com/public-cloud/api-access/docker' - exit 1 - } - command -v json >/dev/null 2>&1 || { + # check for Triton Docker CLI + command -v triton-compose >/dev/null 2>&1 || { echo tput rev # reverse tput bold # bold - echo 'Error! JSON CLI tool is required, but does not appear to be installed.' + echo 'Triton Docker CLI tools are required, but do not appear to be installed.' tput sgr0 # clear - echo 'See https://apidocs.joyent.com/cloudapi/#getting-started' + echo 'See https://github.com/joyent/triton-docker-cli' exit 1 } - # if we're not testing on Triton, don't bother checking Triton config - if [ ! -z "${COMPOSE_FILE}" ]; then - exit 0 - fi - + # check for Triton CLI tool (it should be installed, given the above, but...) command -v triton >/dev/null 2>&1 || { echo tput rev # reverse @@ -66,26 +46,12 @@ check() { exit 1 } - # make sure Docker client is pointed to the same place as the Triton client - local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') - local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') - TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') - TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') - TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') - if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then - echo - tput rev # reverse - tput bold # bold - echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' - tput sgr0 # clear - echo - echo "Docker user: ${docker_user}" - echo "Triton user: ${TRITON_USER}" - echo "Docker data center: ${docker_dc}" - echo "Triton data center: ${TRITON_DC}" - exit 1 - fi + # set env vars for everything else that follows + eval "$(triton env ${TRITON_PROFILE})" + TRITON_DC=$(echo $SDC_URL | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') + TRITON_ACCOUNT_UUID=$(triton account get | awk -F": " '/id:/{print $2}') + # make sure CNS is enabled local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') if [ ! "true" == "$triton_cns_enabled" ]; then echo @@ -97,7 +63,36 @@ check() { exit 1 fi - echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env + + + echo '# Autopilot Pattern Telegraf configuration' > _env + echo >> _env + + echo '# Telegraf output plugin: InfluxDB ' >> _env + echo '# (uncomment to change defaults) ' >> _env + echo '#INFLUXDB_HOST=influxdb # docker alias or real hostname' >> _env + echo '#INFLUXDB_DATABASE=telegraf' >> _env + echo '#INFLUXDB_DATA_ENGINE=tsm1' >> _env + echo >> _env + + echo '# Triton Container Monitor (uses Prometheus input plugin in Telegraf)' >> _env + echo TRITON_ACCOUNT_UUID=${TRITON_ACCOUNT_UUID} >> _env + echo '# This works for Triton Public Cloud, but change it for other clouds:' >> _env + echo TRITON_CNS_SUFFIX=.triton.zone >> _env + echo '# Leave empty or unset and Autopilot Pattern Telegraf will automatically detect the DC:' >> _env + echo '#TRITON_DC=' >> _env + echo >> _env + + echo '# Triton Container Monitor authentication' >> _env + TRITON_CREDS_PATH=/root/.triton + echo TRITON_CREDS_PATH=${TRITON_CREDS_PATH} >> _env + echo TRITON_CA=$(cat "${DOCKER_CERT_PATH}"/ca.pem | tr '\n' '#') >> _env + echo TRITON_CA_PATH=${TRITON_CREDS_PATH}/ca.pem >> _env + echo TRITON_KEY=$(cat "${DOCKER_CERT_PATH}"/key.pem | tr '\n' '#') >> _env + echo TRITON_KEY_PATH=${TRITON_CREDS_PATH}/key.pem >> _env + echo TRITON_CERT=$(cat "${DOCKER_CERT_PATH}"/cert.pem | tr '\n' '#') >> _env + echo TRITON_CERT_PATH=${TRITON_CREDS_PATH}/cert.pem >> _env + echo >> _env } # --------------------------------------------------- From 404ff2365c68bf28e8a58463dccb2da164473f1b Mon Sep 17 00:00:00 2001 From: Casey Bisson Date: Sun, 10 Dec 2017 20:30:18 -0800 Subject: [PATCH 7/9] maybe correct compose file --- examples/triton/docker-compose.yml | 153 +++++++++++++++-------------- 1 file changed, 80 insertions(+), 73 deletions(-) diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index 022d65a..07a3511 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -1,78 +1,85 @@ +version: '2.1' # Telegraf autopilotpattern demo -# Consul - start with a single host which will bootstrap the cluster. -# In production we'll want to use an HA cluster. -consul: - container_name: consul - image: progrium/consul:latest - restart: always - mem_limit: 128m - expose: - - 53 - - 8300 - - 8301 - - 8302 - - 8400 - - 8500 - ports: - - 8500 # expose only Consul's UI on the public IP - dns: - - 127.0.0.1 - labels: - - triton.cns.services=consul - command: -server -bootstrap -ui-dir /ui +services: + # Telegraf should autodiscovery all Autopilot Pattern containers, + # as well as all instances in Triton Container Monitor (https://docs.joyent.com/public-cloud/api-access/prometheus), + # collect telemetry from there, and send output to influxdb. + telegraf: + image: autopilotpattern/telegraf:${TAG:-latest} + # Use a 128MB instance (also see com.joyent.package label below) + mem_limit: 128m + # Joyent recommends setting instances to always restart on Triton + restart: always + labels: + # This label sets the CNS name, Triton's automatic DNS + # Learn more at https://docs.joyent.com/public-cloud/network/cns + - triton.cns.services=telegraf + # This label selects the proper Joyent resource package + # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers + - com.joyent.package=g4-highcpu-128M + network_mode: bridge + ports: + # You may not want these port declarations for production. Without them, Prometheus will only + # listen on the private network. This will also result in a public prometheus CNS record being created, + # in the triton.zone domain. + - 9090 + env_file: _env + environment: + - CONSUL_AGENT=1 + - CONSUL=telegraph-consul.svc.${TRITON_CNS_SEARCH_DOMAIN_PRIVATE} -# Telegraf should autodiscovery all autopilot containers, -# collect telemetry from there, and send output to influxdb. -telegraf: - container_name: telegraf - #image: autopilotpattern/telegraf - build: . # for now use local image build - mem_limit: 128m - expose: - - 8094 # necessary for healthcheck - - 9090 # so we can see telemetry - restart: always - links: - - consul:consul - - influxdb:influxdb - environment: - - CONSUL_AGENT=1 - - CONSUL=consul - - INFLUXDB=influxdb - env_file: - - _env - - _env.telegraf - labels: - - triton.cns.services=telegraf + # Telegraf should autodiscovery all autopilot containers, + # collect telemetry from there, and send output to influxdb. + telegraf: + container_name: telegraf + #image: autopilotpattern/telegraf + build: . # for now use local image build + mem_limit: 128m + expose: + - 8094 # necessary for healthcheck + - 9090 # so we can see telemetry + restart: always + links: + - consul:consul + - influxdb:influxdb + environment: + - CONSUL_AGENT=1 + - CONSUL=consul + - INFLUXDB=influxdb + env_file: + - _env + labels: + - triton.cns.services=telegraf -# InfluxDB -influxdb: - container_name: influxdb - image: influxdb:latest - mem_limit: 512m - ports: - - "8083:8083" - - "8086:8086" - - "8090:8090" - env_file: - - '_env' + # InfluxDB + influxdb: + container_name: influxdb + image: influxdb:latest + mem_limit: 512m + ports: + - "8083:8083" + - "8086:8086" + - "8090:8090" + env_file: + - '_env' -# Ngix included for demonstration of input sources for telegraf -# we can scale it and demonstrate auto capturing of nginx containers telemetry by telegraf -nginx: - image: autopilotpattern/nginx - restart: always - mem_limit: 128m - ports: - - 80 # http port - - 9090 # so we can see telemetry - links: - - consul:consul - environment: - - CONSUL_AGENT=1 - - CONSUL=consul - - BACKEND=consul # backend is required, so lets point to consul just for demo purposes - env_file: _env - labels: - - triton.cns.services=nginx + # Consul is the service catalog + consul: + image: autopilotpattern/consul:0.7.2-r0.8 + command: > + /usr/local/bin/containerpilot + /bin/consul agent -server + -bootstrap-expect 1 + -config-dir=/etc/consul + -ui-dir /ui + # Change "-bootstrap" to "-bootstrap-expect 3", then scale to 3 or more to + # turn this into an HA Consul raft. + restart: always + mem_limit: 128m + ports: + # As above, this port delcaration should not be made for production. + - 8500 + labels: + - triton.cns.services=telegraph-consul + network_mode: bridge From b9c1639249bc885aeeb12cf0678c62b1c94c2c74 Mon Sep 17 00:00:00 2001 From: Casey Bisson Date: Sun, 10 Dec 2017 23:10:06 -0800 Subject: [PATCH 8/9] wip --- Dockerfile | 10 +-- bin/reload.sh | 51 ++++++++++--- etc/telegraf.ctmpl | 114 +---------------------------- examples/triton/docker-compose.yml | 1 + makefile | 51 +++++++++++++ 5 files changed, 99 insertions(+), 128 deletions(-) create mode 100644 makefile diff --git a/Dockerfile b/Dockerfile index f9dd468..9cb6e31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,14 @@ -#FROM telegraf:1.0-alpine -FROM telegraf:1.0 +FROM telegraf:1.4-alpine # Reset to root user to do some installs USER root # Install packages -RUN apt-get update && apt-get -y install \ +RUN apk add --no-cache iputils ca-certificates net-snmp-tools procps && \ bash \ curl \ - unzip \ - netcat-openbsd \ - && rm -rf /var/cache/apt/* /var/lib/apt/lists/* /tmp/* /var/tmp/* + jq + unzip # Add ContainerPilot and its configuration # Releases at https://github.com/joyent/containerpilot/releases diff --git a/bin/reload.sh b/bin/reload.sh index 2d5c77c..4c3dc66 100755 --- a/bin/reload.sh +++ b/bin/reload.sh @@ -1,40 +1,67 @@ #!/bin/bash -SERVICE_NAME=${SERVICE_NAME:-telegraf} -CONSUL=${CONSUL:-consul} - # Render Telegraf configuration template using values from Consul, # but do not reload because Telegraf has't started yet preStart() { - # sleep 5 # give some time for other containerpilots to start before rendering config + # Do we have env vars for Triton discovery? + # Copy creds from env vars to files on disk + if [ -n ${!TRITON_CREDS_PATH} ] \ + && [ -n ${!TRITON_CA} ] \ + && [ -n ${!TRITON_CERT} ] \ + && [ -n ${!TRITON_KEY} ] + then + mkdir -p ${TRITON_CREDS_PATH} + echo -e "${TRITON_CA}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/ca.pem + echo -e "${TRITON_CERT}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/cert.pem + echo -e "${TRITON_KEY}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/key.pem + fi + + # Are we on Triton? Do we _not_ have a user-defined DC? + # Set the DC automatically from mdata + if [ -n ${TRITON_DC} ] \ + && [ -f "/native/usr/sbin/mdata-get" ] + then + export TRITON_DC=$(/native/usr/sbin/mdata-get sdc:datacenter_name) + fi + + # Create Telegraf config consul-template \ -once \ -dedup \ - -consul ${CONSUL}:8500 \ + -consul-addr ${CONSUL}:8500 \ -template "/etc/telegraf.ctmpl:/etc/telegraf.conf" } + + # Render Telegraf configuration template using values from Consul, # then gracefully reload Telegraf onChange() { consul-template \ -once \ -dedup \ - -consul ${CONSUL}:8500 \ + -consul-addr ${CONSUL}:8500 \ -template "/etc/telegraf.ctmpl:/etc/telegraf.conf:/usr/local/bin/reload.sh reloadConfig" } -# Telegraf reload th SIGHUP -# Note: if we fire SIGHUP vs node before it has a chance to register the -# signal handler, then it will immediately exit. This ensures that -# the process is listening on port 8094 which should only be the -# case after we have the signal handler loaded. + + +# SIGHUP to reload the Telegraf config +# However: if if we fire the SIGHUP to Telegraf before it has a chance to +# register the signal handler, then it will immediately exit. +# This checks that Telgraf is listening on port 8094, which should only +# be true after the signal handler is loaded. reloadConfig() { while : do - netstat -ln | grep -q 8094 && pkill -SIGHUP telegraf && break + netstat -ln | grep -q 8094 \ + && pkill -SIGHUP telegraf \ + && break done } + + + help() { echo "Usage: ./reload.sh preStart => first-run configuration for Telegraf" echo " ./reload.sh onChange => [default] update Telegraf config on upstream changes" diff --git a/etc/telegraf.ctmpl b/etc/telegraf.ctmpl index c4d0fc3..4bd1d71 100644 --- a/etc/telegraf.ctmpl +++ b/etc/telegraf.ctmpl @@ -103,112 +103,6 @@ # INPUT PLUGINS # ############################################################################### -# Read metrics about cpu usage -[[inputs.cpu]] - ## Whether to report per-cpu stats or not - percpu = true - ## Whether to report total system cpu stats or not - totalcpu = true - ## Comment this line if you want the raw CPU time metrics - fielddrop = ["time_*"] - - -# Read metrics about disk usage by mount point -[[inputs.disk]] - ## By default, telegraf gather stats for all mountpoints. - ## Setting mountpoints will restrict the stats to the specified mountpoints. - # mount_points = ["/"] - - ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually - ## present on /run, /var/run, /dev/shm or /dev). - ignore_fs = ["tmpfs", "devtmpfs"] - - -# Read metrics about disk IO by device -[[inputs.diskio]] - ## By default, telegraf will gather stats for all devices including - ## disk partitions. - ## Setting devices will restrict the stats to the specified devices. - # devices = ["sda", "sdb"] - ## Uncomment the following line if you need disk serial numbers. - # skip_serial_number = false - - -# Get kernel statistics from /proc/stat -[[inputs.kernel]] - # no configuration - - -# Read metrics about memory usage -[[inputs.mem]] - # no configuration - - -# Get the number of processes and group them by status -[[inputs.processes]] - # no configuration - - -# Read metrics about swap memory usage -[[inputs.swap]] - # no configuration - - -# Read metrics about system load & uptime -[[inputs.system]] - # no configuration - -# # Generic TCP listener -[[inputs.tcp_listener]] - ## Address and port to host TCP listener on - service_address = ":8094" -# -# ## Number of TCP messages allowed to queue up. Once filled, the -# ## TCP listener will start dropping packets. - allowed_pending_messages = 10000 -# -# ## Maximum number of concurrent TCP connections to allow - max_tcp_connections = 250 -# -# ## Data format to consume. -# ## Each data format has it's own unique set of configuration options, read -# ## more about them here: -# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" - - -# # Generic UDP listener -[[inputs.udp_listener]] - ## Address and port to host UDP listener on - service_address = ":8092" -# -# ## Number of UDP messages allowed to queue up. Once filled, the -# ## UDP listener will start dropping packets. - allowed_pending_messages = 10000 -# -# ## Data format to consume. -# ## Each data format has it's own unique set of configuration options, read -# ## more about them here: -# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" - - -# # Gather health check statuses from services registered in Consul -# [[inputs.consul]] -# ## Most of these values defaults to the one configured on a Consul's agent level. -# ## Optional Consul server address (default: "localhost") -# # address = "localhost" -# ## Optional URI scheme for the Consul server (default: "http") -# # scheme = "http" -# ## Optional ACL token used in every request (default: "") -# # token = "" -# ## Optional username used for request HTTP Basic Authentication (default: "") -# # username = "" -# ## Optional password used for HTTP Basic Authentication (default: "") -# # password = "" -# ## Optional data centre to query the health checks from (default: "") -# # datacentre = "" - # # Read metrics from one or many prometheus clients [[inputs.prometheus]] ## An array of urls to scrape metrics from. @@ -222,8 +116,8 @@ # # bearer_token = /path/to/bearer/token # # ## Optional SSL Config -# # ssl_ca = /path/to/cafile -# # ssl_cert = /path/to/certfile -# # ssl_key = /path/to/keyfile + ssl_ca = "/path/ca.pem" + ssl_cert = "/path/to/cert.pem" + ssl_key = "/path/to/key.pem" # ## Use SSL but skip chain & host verification -# # insecure_skip_verify = false + insecure_skip_verify = false diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index 07a3511..cd8cd1e 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -23,6 +23,7 @@ services: # You may not want these port declarations for production. Without them, Prometheus will only # listen on the private network. This will also result in a public prometheus CNS record being created, # in the triton.zone domain. + - 8094 - 9090 env_file: _env environment: diff --git a/makefile b/makefile new file mode 100644 index 0000000..27f2897 --- /dev/null +++ b/makefile @@ -0,0 +1,51 @@ +# Makefile for building and shipping the container image. + +MAKEFLAGS += --warn-undefined-variables +.DEFAULT_GOAL := build +.PHONY: * + +# we get these from CI environment if available, otherwise from git +GIT_COMMIT ?= $(shell git rev-parse --short HEAD) +GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) + +namespace ?= autopilotpattern +tag := branch-$(shell basename $(GIT_BRANCH)) +image := $(namespace)/telegraf + +## Display this help message +help: + @awk '/^##.*$$/,/[a-zA-Z_-]+:/' $(MAKEFILE_LIST) | awk '!(NR%2){print $$0p}{p=$$0}' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' | sort + + +# ------------------------------------------------ +# Container builds + +## Builds the application container image locally +build: + docker build -t=$(image):$(tag) . + +## Push the current application container images to the Docker Hub +push: + docker push $(image):$(tag) + +## Tag the current images as 'latest' and push them to the Docker Hub +ship: + docker tag $(image):$(tag) $(image):latest + docker push $(image):$(tag) + docker push $(image):latest + + +# ------------------------------------------------ +# Test running + +## Print environment for build debugging +debug: + @echo GIT_COMMIT=$(GIT_COMMIT) + @echo GIT_BRANCH=$(GIT_BRANCH) + @echo namespace=$(namespace) + @echo tag=$(tag) + @echo image=$(image) + +check_var = $(foreach 1,$1,$(__check_var)) +__check_var = $(if $(value $1),,\ + $(error Missing $1 $(if $(value 2),$(strip $2)))) From 6d93461b4bc5eee566b2e8cadc28b648612742bc Mon Sep 17 00:00:00 2001 From: Casey Bisson Date: Fri, 15 Dec 2017 14:09:10 +0000 Subject: [PATCH 9/9] wip --- Dockerfile | 4 ++-- etc/telegraf.ctmpl | 11 ++++------- examples/triton/docker-compose.yml | 26 ++------------------------ 3 files changed, 8 insertions(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9cb6e31..6e4f7a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ FROM telegraf:1.4-alpine USER root # Install packages -RUN apk add --no-cache iputils ca-certificates net-snmp-tools procps && \ +RUN apk add --no-cache \ bash \ curl \ - jq + jq \ unzip # Add ContainerPilot and its configuration diff --git a/etc/telegraf.ctmpl b/etc/telegraf.ctmpl index 4bd1d71..68deeee 100644 --- a/etc/telegraf.ctmpl +++ b/etc/telegraf.ctmpl @@ -103,7 +103,7 @@ # INPUT PLUGINS # ############################################################################### -# # Read metrics from one or many prometheus clients +# # Use Prometheus input plugin for Triton Container Monitor metrics [[inputs.prometheus]] ## An array of urls to scrape metrics from. {{ if service "containerpilot" }} @@ -112,12 +112,9 @@ urls = ["http://localhost:9090/metrics"] {{ end }} -# ## Use bearer token for authorization -# # bearer_token = /path/to/bearer/token -# # ## Optional SSL Config - ssl_ca = "/path/ca.pem" - ssl_cert = "/path/to/cert.pem" - ssl_key = "/path/to/key.pem" + ssl_ca = "/root/.triton/ca.pem" + ssl_cert = "/root/.triton/cert.pem" + ssl_key = "/root/.triton/key.pem" # ## Use SSL but skip chain & host verification insecure_skip_verify = false diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index cd8cd1e..9b1ef7e 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -30,34 +30,11 @@ services: - CONSUL_AGENT=1 - CONSUL=telegraph-consul.svc.${TRITON_CNS_SEARCH_DOMAIN_PRIVATE} - # Telegraf should autodiscovery all autopilot containers, - # collect telemetry from there, and send output to influxdb. - telegraf: - container_name: telegraf - #image: autopilotpattern/telegraf - build: . # for now use local image build - mem_limit: 128m - expose: - - 8094 # necessary for healthcheck - - 9090 # so we can see telemetry - restart: always - links: - - consul:consul - - influxdb:influxdb - environment: - - CONSUL_AGENT=1 - - CONSUL=consul - - INFLUXDB=influxdb - env_file: - - _env - labels: - - triton.cns.services=telegraf - # InfluxDB influxdb: - container_name: influxdb image: influxdb:latest mem_limit: 512m + network_mode: bridge ports: - "8083:8083" - "8086:8086" @@ -78,6 +55,7 @@ services: # turn this into an HA Consul raft. restart: always mem_limit: 128m + network_mode: bridge ports: # As above, this port delcaration should not be made for production. - 8500