diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..9a75c204 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +# Dotfiles +.git* +.pkgr.yml +.travis.yml + +# Non-app files +*.md +*.txt +exampleProxyConfig.js + +# Non-app directories +debian +node_modules +packager +examples +docs + +# Local dev files +config.js diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml new file mode 100644 index 00000000..5675ac01 --- /dev/null +++ b/.github/workflows/package-and-publish-image.yml @@ -0,0 +1,64 @@ +# Build, package, and publish a container image to our supported +# image registries. We generate a number of tags in different scenarios. +# +# * Commit pushed on default branch: publish with semver and "latest" tags. +# * Tag on default branch pushed: publish with short commit sha and "edge" tags. +# +# We currently publish to dockerhub and ghcr. + +name: Package and publish container image + +on: + push: + branches: + - master + tags: + - "v*" + +jobs: + publish: + name: Build and publish image + runs-on: ubuntu-22.04 + steps: + - name: Generate container image meta tags + id: meta + uses: docker/metadata-action@v4 + with: + images: | + statsd/statsd + ghcr.io/statsd/statsd + flavor: | + latest=true + tags: | + type=semver,pattern=v{{version}},event=tag + type=semver,pattern=v{{major}}.{{minor}},event=tag + type=semver,pattern=v{{major}},event=tag + type=edge,branch=$repo.default_branch,event=push + type=sha,branch=$repo.default_branch,event=push + + - name: Setup buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub container registry + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build and publish image + id: docker_build + uses: docker/build-push-action@v4 + with: + push: true + platforms: linux/amd64,linux/s390x,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + + - name: Echo image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.gitignore b/.gitignore index 7d618a48..b9abaafd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ node_modules - +/config.js # WebStorm / IntelliJ IDEA project files .idea *.iml diff --git a/.pkgr.yml b/.pkgr.yml new file mode 100644 index 00000000..b412d0d2 --- /dev/null +++ b/.pkgr.yml @@ -0,0 +1,9 @@ +default_dependencies: false +targets: + ubuntu-14.04: + ubuntu-12.04: + debian-7: + centos-6: +before: + - mv packager/Procfile . +after_install: ./packager/postinst diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7cb41cb5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: node_js -node_js: -- '0.8' -- '0.10' -script: ./run_tests.sh -notifications: - email: false - irc: - - irc.freenode.org#statsd -deploy: - provider: npm - email: d@unwiredcouch.com - api_key: - secure: IE9nz50eZsRL1Dbcxj2eY0apO1Io2swGF3ezZCzny20WgqQXiiVs24rHUi1GywDELGDc7+Vp0zJfmXigE+zTMvx0N3fTASiuDzd3C7fULa4JUSH2DoHNOXx7WSkr4EmujDsB7y1mEBDHOdBlLWBRApExt67TlYzvZiT8/Sffq3k= - on: - tags: true - repo: etsy/statsd diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..35ce6077 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at +statsd-coc@googlegroups.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c88fd56..40a94306 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,20 +2,19 @@ You're interested in contributing to StatsD? *AWESOME*. Here are the basic steps: -fork StatsD from here: http://github.com/etsy/statsd - -1. Clone your fork -2. Hack away -3. If you are adding new functionality, document it in the [docs][d] -4. If necessary, rebase your commits into logical chunks, without errors -5. Verify your code by running the test suite, and adding additional tests if able. -6. Push the branch up to GitHub -7. Send a pull request to the etsy/statsd project. +1. fork StatsD from here: http://github.com/statsd/statsd +2. Clone your fork +3. Hack away +4. If you are adding new functionality, document it in the [docs][d] +5. If necessary, rebase your commits into logical chunks, without errors +6. Verify your code by running the test suite, and adding additional tests if able. +7. Push the branch up to GitHub +8. Send a pull request to the statsd/statsd project. We'll do our best to get your changes in! # Contributors -In lieu of a list of contributors, check out the [commit history for the project](https://github.com/etsy/statsd/graphs/contributors) +In lieu of a list of contributors, check out the [commit history for the project](https://github.com/statsd/statsd/graphs/contributors) -[d]: https://github.com/etsy/statsd/tree/master/docs +[d]: https://github.com/statsd/statsd/tree/master/docs diff --git a/Changelog.md b/Changelog.md index 5c62ab01..74549e55 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,108 @@ # Changelog +## v0.10.2 (08/22/2023) + +- Support publishing amd64 and s390x container images + +## v0.10.1 (05/11/2023) + +- Include tag pushes in docker builds + +## v0.10.0 (05/11/2023) + +- Multiple documentation updates +- Updated docker-compose syntax +- Upgrade tests to use python3 +- Upgrade node base for Docker image +- Automatic container publishing via GHA + +## v0.9.0 (08/27/2020) + +- Add support for graphite tagged metrics +- Fix dashboard to 0 last_exception time on startup +- Multiple documentation updates +- Correct some out-dated integration examples +- Update container image to use recent node version + +## v0.8.6 (02/19/2020) + +- Add an optional max TTL setting for gauges +- add filter option for metrics + +## v0.8.5 (07/23/2019) + +- Update lodash (sub dependency) for security fix +- Add the StatsD history to the docs +- Add third party server interfaces to docs +- Migrate docs from github wiki, and standardise markdown notation +- Minor formatting proposals +- Add docker image info to readme + +## v0.8.4 (07/11/2019) + +- update modern-syslog to 1.2.0 for node 12 compatibility +- update package.json version to 0.8.3 + +## v0.8.3 (07/11/2019) + +- correct backend flush loop +- test and declare support for Current and LTS node +- Correct reporter decleration in test runner +- Convert codebase from var -> let / const (#673) +- correct npm test script +- correct travis deploy step + +## v0.8.2 (04/02/2019) + +- update travis npm token +- update dockerfile to latest node-lts +- correct gitter link +- update dockerfile base image to node lts +- Add gitter chat badge +- run tests using python 3.7's pickle rather than 2.x cPickle (#669) + +## v0.8.1 (03/13/2019) + +- drop StatsD instance from proxy ring in instance of healthcheck failures (#665) +- Add myself (elliot blackburn) to maintainers.md (#666) +- add mysql backend link to docs/backend.md +- Adding myself to MAINTAINERS. +- begin testing on node lts and up +- Added "opencensus-backend" +- correct package.json links to new github organisation +- Update MAINTAINERS.md +- remove meta section of README +- Create MAINTAINERS.md +- Create DCO.txt +- Create CODE_OF_CONDUCT.md +- update README post transfer +- Fixing Markdown formatting +- fix simple typo +- Added StatsdClient Kotlin implementation +- fix formatting on backend interface docs +- Update: ignore files +- removes -q switch +- Fix for failing test on node 0.10 +- fix usage of process.EventEmitter +- Add plugin Warp10 to StatsD +- Updated graphite link to read the docs + +## v0.8.0 (05/05/2016) +- Modularized injest servers, with support for loading multiple servers +- Added configurable tcp injest server +- Added unix socket injest support +- Added tcp repeater functionality +- Added pickle protocol support to graphite backend +- Added configurable IPv6 and TCP support to proxy +- Added telnet admin interface to proxy +- Multiple variable scoping fixes +- Fixes to flush timer to reduce bucket drift +- Fixes to ruby and java example client code +- Dropped support for node v0.8.x +- Fixed dependency issues for modern node versions +- Updated npm hashring dependency to v3.2.0 +- Replaced npm node-syslog dependency with modern-syslog v1.1.2 + ## v0.7.2 (09/02/2014) - Fixes to detecting valid packets @@ -36,7 +139,7 @@ - added standard Deviation to timers stats (.std) - added last_flush_time and last_flush_length metrics to graphite backend - added ipv6 support -- added Statsd repeater backend +- added StatsD repeater backend - added helper script to decide which timers to sample down - added Windows service support - added Scala example diff --git a/DCO.txt b/DCO.txt new file mode 100644 index 00000000..8201f992 --- /dev/null +++ b/DCO.txt @@ -0,0 +1,37 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..cd483450 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM node:18.16.0 + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Install python +# RUN apk add --no-cache --update g++ gcc libgcc libstdc++ linux-headers make python + +# Setup node envs +ARG NODE_ENV +ENV NODE_ENV $NODE_ENV + +# Install dependencies +COPY package.json /usr/src/app/ +RUN npm install && npm cache clean --force + +# Copy required src (see .dockerignore) +COPY . /usr/src/app + +# Set graphite hostname to "graphite" +RUN \ + ls -la && \ + cp -v exampleConfig.js config.js && \ + sed -i 's/graphite.example.com/graphite/' config.js + +# Expose required ports +EXPOSE 8125/udp +EXPOSE 8126 + +# Start statsd +ENTRYPOINT [ "node", "stats.js", "config.js" ] diff --git a/LICENSE b/LICENSE index 4199459a..f86a9a68 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2010-2014 Etsy + Copyright (c) 2010-2016 Etsy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 00000000..78c91746 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,29 @@ +StatsD is maintained by the following people. + +All maintainers must agree to the [Developer Certificate of Origin][dco]. + +## Steps to becoming a maintainer +1. Open a PR where you add yourself to this file and agree to the DCO. Attach a little bit about your experience +with StatsD and how much time you think you can roughly spend on the project +2. Current maintainers will review +3. If it gets merged, you're in! + +## Retiring from being a maintainer +1. Open a pull request moving yourself from current to past maintainers +2. Mention [@statsd/statsd-maintainers](https://github.com/orgs/statsd/teams/statsd-maintainers) on the PR +3. Have another maintainer approve/sign off +4. Merge it + +## Current maintainers + +- **Daniel Schauenberg**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. +- **Mike Heffner**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. +- **Elliot Blackburn**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. + +[dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt + +## Past maintainers +- [Tera Koch](https://github.com/pathzzrd) +- [Ben Burry](https://github.com/benburry) +- [Dan Rowe](https://github.com/draco2003) +- [Erik Kastner](https://github.com/kastner) diff --git a/README.md b/README.md index ed176338..fd70cf68 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,45 @@ -StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] -====== +# StatsD [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Docker Pulls](https://img.shields.io/docker/pulls/statsd/statsd)](https://hub.docker.com/r/statsd/statsd) A network daemon that runs on the [Node.js][node] platform and listens for statistics, like counters and timers, sent over [UDP][udp] or [TCP][tcp] and sends aggregates to one or more pluggable backend services (e.g., [Graphite][graphite]). -We ([Etsy][etsy]) [blogged][blog post] about how it works and why we created it. - -Inspiration ------------ - -StatsD was inspired (heavily) by the project (of the same name) at Flickr. -Here's a post where Cal Henderson described it in depth: -[Counting and timing][counting-timing] -Cal re-released the code recently: -[Perl StatsD][Flicker-StatsD] - -Key Concepts --------- +## Key Concepts * *buckets* + Each stat is in its own "bucket". They are not predefined anywhere. Buckets can be named anything that will translate to Graphite (periods make folders, etc) * *values* + Each stat will have a value. How it is interpreted depends on modifiers. In -general values should be integer. +general values should be integers. * *flush* + After the flush interval timeout (defined by `config.flushInterval`, default 10 seconds), stats are aggregated and sent to an upstream backend service. -Installation and Configuration ------------------------------- +## Installation and Configuration + +### Docker +StatsD supports docker in three ways: +* The official container image on [GitHub Container Registry](https://github.com/statsd/statsd/pkgs/container/statsd) +* The official container image on [DockerHub](https://hub.docker.com/r/statsd/statsd) +* Building the image from the bundled [Dockerfile](./Dockerfile) - * Install node.js +### Manual installation + * Install Node.js (All [`Current` and `LTS` Node.js versions](https://nodejs.org/en/about/releases/) are supported.) * Clone the project - * Create a config file from exampleConfig.js and put it somewhere + * Create a config file from `exampleConfig.js` and put it somewhere * Start the Daemon: + `node stats.js /path/to/config` - node stats.js /path/to/config - -Usage -------- +## Usage The basic line protocol expects metrics to be sent in the format: :| @@ -55,8 +49,7 @@ StatsD running with the default UDP server on localhost would be: echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 -More Specific Topics --------- +## More Specific Topics * [Metric Types][docs_metric_types] * [Graphite Integration][docs_graphite] * [Supported Servers][docs_server] @@ -65,11 +58,9 @@ More Specific Topics * [Server Interface][docs_server_interface] * [Backend Interface][docs_backend_interface] * [Metric Namespacing][docs_namespacing] -* [Statsd Cluster Proxy][docs_cluster_proxy] - -Debugging ---------- +* [StatsD Cluster Proxy][docs_cluster_proxy] +## Debugging There are additional config variables available for debugging: * `debug` - log exceptions and print out more diagnostic info @@ -78,11 +69,9 @@ There are additional config variables available for debugging: For more information, check the `exampleConfig.js`. -Tests ------ - +## Tests A test framework has been added using node-unit and some custom code to start -and manipulate statsd. Please add tests under test/ for any new features or bug +and manipulate StatsD. Please add tests under test/ for any new features or bug fixes encountered. Testing a live server can be tricky, attempts were made to eliminate race conditions but it may be possible to encounter a stuck state. If doing dev work, a `killall statsd` will kill any stray test servers in the @@ -90,31 +79,36 @@ background (don't do this on a production machine!). Tests can be executed with `./run_tests.sh`. +## History +StatsD was originally written at [Etsy][etsy] and released with a +[blog post][blog post] about how it works and why we created it. -Meta ---------- -- IRC channel: `#statsd` on freenode -- Mailing list: `statsd@librelist.com` +## Inspiration +StatsD was inspired (heavily) by the project of the same name at Flickr. +Here's a post where Cal Henderson described it in depth: +[Counting and timing][counting-timing]. +Cal re-released the code recently: +[Perl StatsD][Flicker-StatsD] [graphite]: http://graphite.readthedocs.org/ [etsy]: http://www.etsy.com -[blog post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ +[blog post]: https://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ [node]: http://nodejs.org [nodemods]: http://nodejs.org/api/modules.html [counting-timing]: http://code.flickr.com/blog/2008/10/27/counting-timing/ [Flicker-StatsD]: https://github.com/iamcal/Flickr-StatsD [udp]: http://en.wikipedia.org/wiki/User_Datagram_Protocol [tcp]: http://en.wikipedia.org/wiki/Transmission_Control_Protocol -[docs_metric_types]: https://github.com/etsy/statsd/blob/master/docs/metric_types.md -[docs_graphite]: https://github.com/etsy/statsd/blob/master/docs/graphite.md -[docs_server]: https://github.com/etsy/statsd/blob/master/docs/server.md -[docs_backend]: https://github.com/etsy/statsd/blob/master/docs/backend.md -[docs_admin_interface]: https://github.com/etsy/statsd/blob/master/docs/admin_interface.md -[docs_server_interface]: https://github.com/etsy/statsd/blob/master/docs/server_interface.md -[docs_backend_interface]: https://github.com/etsy/statsd/blob/master/docs/backend_interface.md +[docs_metric_types]: https://github.com/statsd/statsd/blob/master/docs/metric_types.md +[docs_graphite]: https://github.com/statsd/statsd/blob/master/docs/graphite.md +[docs_server]: https://github.com/statsd/statsd/blob/master/docs/server.md +[docs_backend]: https://github.com/statsd/statsd/blob/master/docs/backend.md +[docs_admin_interface]: https://github.com/statsd/statsd/blob/master/docs/admin_interface.md +[docs_server_interface]: https://github.com/statsd/statsd/blob/master/docs/server_interface.md +[docs_backend_interface]: https://github.com/statsd/statsd/blob/master/docs/backend_interface.md [docs_namespacing]: https://github.com/etsy/statsd/blob/master/docs/namespacing.md [docs_cluster_proxy]: https://github.com/etsy/statsd/blob/master/docs/cluster_proxy.md -[travis-ci_status_img]: https://travis-ci.org/etsy/statsd.svg?branch=master -[travis-ci_statsd]: https://travis-ci.org/etsy/statsd +[travis-ci_status_img]: https://travis-ci.org/statsd/statsd.svg?branch=master +[travis-ci_statsd]: https://travis-ci.org/statsd/statsd diff --git a/backends/console.js b/backends/console.js index b3a6065a..e435aa59 100644 --- a/backends/console.js +++ b/backends/console.js @@ -33,7 +33,7 @@ ConsoleBackend.prototype.flush = function(timestamp, metrics) { }; if(this.config.prettyprint) { - console.log(util.inspect(out, false, 5, true)); + console.log(util.inspect(out, {depth: 5, colors: true})); } else { console.log(out); } diff --git a/backends/graphite.js b/backends/graphite.js index 34cd299d..1e6e462c 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -11,7 +11,11 @@ * This backend supports the following config options: * * graphiteHost: Hostname of graphite server. - * graphitePort: Port to contact graphite server at. + * graphitePort: Port for the graphite text collector. Defaults to 2003. + * graphitePicklePort: Port for the graphite pickle collector. Defaults to 2004. + * graphiteProtocol: Either 'text' or 'pickle'. Defaults to 'text'. + * + * If graphiteHost is not specified, metrics are processed but discarded. */ var net = require('net'); @@ -23,6 +27,8 @@ var debug; var flushInterval; var graphiteHost; var graphitePort; +var graphitePicklePort; +var graphiteProtocol; var flush_counts; // prefix configuration @@ -33,6 +39,7 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -44,48 +51,104 @@ var setsNamespace = []; var graphiteStats = {}; -var post_stats = function graphite_post_stats(statString) { +var post_stats = function graphite_post_stats(stats) { var last_flush = graphiteStats.last_flush || 0; var last_exception = graphiteStats.last_exception || 0; var flush_time = graphiteStats.flush_time || 0; var flush_length = graphiteStats.flush_length || 0; + if (graphiteHost) { try { - var graphite = net.createConnection(graphitePort, graphiteHost); + var port = graphiteProtocol == 'pickle' ? graphitePicklePort : graphitePort; + var graphite = net.createConnection(port, graphiteHost); graphite.addListener('error', function(connectionException){ if (debug) { l.log(connectionException); } }); graphite.on('connect', function() { - var ts = Math.round(new Date().getTime() / 1000); - var ts_suffix = ' ' + ts + "\n"; + var ts = Math.round(Date.now() / 1000); var namespace = globalNamespace.concat(prefixStats).join("."); - statString += namespace + '.graphiteStats.last_exception' + globalSuffix + last_exception + ts_suffix; - statString += namespace + '.graphiteStats.last_flush' + globalSuffix + last_flush + ts_suffix; - statString += namespace + '.graphiteStats.flush_time' + globalSuffix + flush_time + ts_suffix; - statString += namespace + '.graphiteStats.flush_length' + globalSuffix + flush_length + ts_suffix; + stats.add(namespace + '.graphiteStats.last_exception' + globalSuffix, last_exception, ts); + stats.add(namespace + '.graphiteStats.last_flush' + globalSuffix, last_flush , ts); + stats.add(namespace + '.graphiteStats.flush_time' + globalSuffix, flush_time , ts); + stats.add(namespace + '.graphiteStats.flush_length' + globalSuffix, flush_length , ts); + var stats_payload = graphiteProtocol == 'pickle' ? stats.toPickle() : stats.toText(); var starttime = Date.now(); - this.write(statString); + this.write(stats_payload); this.end(); + graphiteStats.flush_time = (Date.now() - starttime); - graphiteStats.flush_length = statString.length; - graphiteStats.last_flush = Math.round(new Date().getTime() / 1000); + graphiteStats.flush_length = stats_payload.length; + graphiteStats.last_flush = Math.round(Date.now() / 1000); }); } catch(e){ if (debug) { l.log(e); } - graphiteStats.last_exception = Math.round(new Date().getTime() / 1000); + graphiteStats.last_exception = Math.round(Date.now() / 1000); } } }; +// Minimally necessary pickle opcodes. +var MARK = '(', + STOP = '.', + LONG = 'L', + STRING = 'S', + APPEND = 'a', + LIST = 'l', + TUPLE = 't'; + +// A single measurement for sending to graphite. +function Metric(key, value, ts) { + var m = this; + this.key = key; + this.value = value; + this.ts = ts; + + // return a string representation of this metric appropriate + // for sending to the graphite collector. does not include + // a trailing newline. + this.toText = function() { + return m.key + " " + m.value + " " + m.ts; + }; + + this.toPickle = function() { + return MARK + STRING + '\'' + m.key + '\'\n' + MARK + LONG + m.ts + 'L\n' + STRING + '\'' + m.value + '\'\n' + TUPLE + TUPLE + APPEND; + }; +} + +// A collection of measurements for sending to graphite. +function Stats() { + var s = this; + this.metrics = []; + this.add = function(key, value, ts) { + s.metrics.push(new Metric(key, value, ts)); + }; + + this.toText = function() { + return s.metrics.map(function(m) { return m.toText(); }).join('\n') + '\n'; + }; + + this.toPickle = function() { + var body = MARK + LIST + s.metrics.map(function(m) { return m.toPickle(); }).join('') + STOP; + + // The first four bytes of the graphite pickle format + // contain the length of the rest of the payload. + // We use Buffer because this is binary data. + var buf = new Buffer(4 + body.length); + + buf.writeUInt32BE(body.length,0); + buf.write(body,4); + + return buf; + }; +} + var flush_stats = function graphite_flush(ts, metrics) { - var ts_suffix = ' ' + ts + "\n"; var starttime = Date.now(); - var statString = ''; var numStats = 0; var key; var timer_data_key; @@ -97,20 +160,43 @@ var flush_stats = function graphite_flush(ts, metrics) { var timer_data = metrics.timer_data; var statsd_metrics = metrics.statsd_metrics; + // Sanitize key for graphite if not done globally + function sk(key) { + if (globalKeySanitize) { + return key; + } else { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.;=]/g, ''); + } + }; + + function format(namespace, key) { + var splitName = key.split(';'); + var keyName = sk(splitName[0]); + var tags = splitName.length > 1 ? (';' + splitName.slice(1).join(';')) : ''; + + return namespace.concat(keyName, [].slice.call(arguments, 2)).join('.') + globalSuffix + tags; + } + + // Flatten all the different types of metrics into a single + // collection so we can allow serialization to either the graphite + // text and pickle formats. + var stats = new Stats(); + for (key in counters) { - var namespace = counterNamespace.concat(key); var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate if (legacyNamespace === true) { - statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; + stats.add(format(counterNamespace, key), valuePerSecond, ts); if (flush_counts) { - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + stats.add(format(['stats_counts'], key), value, ts); } } else { - statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; + stats.add(format(counterNamespace, key, 'rate'), valuePerSecond, ts); if (flush_counts) { - statString += namespace.concat('count').join(".") + globalSuffix + value + ts_suffix; + stats.add(format(counterNamespace, key, 'count'), value, ts); } } @@ -118,18 +204,16 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(key); - var the_key = namespace.join("."); for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { - statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; + stats.add(format(timerNamespace, key, timer_data_key), timer_data[key][timer_data_key], ts); } else { for (var timer_data_sub_key in timer_data[key][timer_data_key]) { if (debug) { l.log(timer_data[key][timer_data_key][timer_data_sub_key].toString()); } - statString += the_key + '.' + timer_data_key + '.' + timer_data_sub_key + globalSuffix + - timer_data[key][timer_data_key][timer_data_sub_key] + ts_suffix; + stats.add(format(timerNamespace, key, timer_data_key, timer_data_sub_key), + timer_data[key][timer_data_key][timer_data_sub_key], ts); } } } @@ -137,33 +221,29 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(key); - statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; + stats.add(format(gaugesNamespace, key), gauges[key], ts); numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(key); - statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; + stats.add(format(setsNamespace, key, 'count'), sets[key].size(), ts); numStats += 1; } - var namespace = globalNamespace.concat(prefixStats); if (legacyNamespace === true) { - statString += prefixStats + '.numStats' + globalSuffix + numStats + ts_suffix; - statString += 'stats.' + prefixStats + '.graphiteStats.calculationtime' + globalSuffix + (Date.now() - starttime) + ts_suffix; + stats.add(prefixStats + '.numStats' + globalSuffix, numStats, ts); + stats.add('stats.' + prefixStats + '.graphiteStats.calculationtime' + globalSuffix, (Date.now() - starttime), ts); for (key in statsd_metrics) { - statString += 'stats.' + prefixStats + '.' + key + globalSuffix + statsd_metrics[key] + ts_suffix; + stats.add('stats.' + prefixStats + '.' + key + globalSuffix, statsd_metrics[key], ts); } } else { - statString += namespace.join(".") + '.numStats' + globalSuffix + numStats + ts_suffix; - statString += namespace.join(".") + '.graphiteStats.calculationtime' + globalSuffix + (Date.now() - starttime) + ts_suffix; + stats.add(format(globalNamespace, prefixStats, 'numStats'), numStats, ts); + stats.add(format(globalNamespace, prefixStats, 'graphiteStats', 'calculationtime'), (Date.now() - starttime) , ts); for (key in statsd_metrics) { - var the_key = namespace.concat(key); - statString += the_key.join(".") + globalSuffix + statsd_metrics[key] + ts_suffix; + stats.add(format(globalNamespace, prefixStats, key), statsd_metrics[key], ts); } } - post_stats(statString); + post_stats(stats); if (debug) { l.log("numStats: " + numStats); @@ -180,7 +260,9 @@ exports.init = function graphite_init(startup_time, config, events, logger) { debug = config.debug; l = logger; graphiteHost = config.graphiteHost; - graphitePort = config.graphitePort; + graphitePort = config.graphitePort || 2003; + graphitePicklePort = config.graphitePicklePort || 2004; + graphiteProtocol = config.graphiteProtocol || 'text'; config.graphite = config.graphite || {}; globalPrefix = config.graphite.globalPrefix; prefixCounter = config.graphite.prefixCounter; @@ -200,10 +282,9 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; legacyNamespace = legacyNamespace !== undefined ? legacyNamespace : true; - // In order to unconditionally add this string, it either needs to be - // a single space if it was unset, OR surrounded by a . and a space if - // it was set. - globalSuffix = globalSuffix !== undefined ? '.' + globalSuffix + ' ' : ' '; + // In order to unconditionally add this string, it either needs to be an + // empty string if it was unset, OR prefixed by a . if it was set. + globalSuffix = globalSuffix !== undefined ? '.' + globalSuffix : ''; if (legacyNamespace === false) { if (globalPrefix !== "") { @@ -235,10 +316,14 @@ exports.init = function graphite_init(startup_time, config, events, logger) { } graphiteStats.last_flush = startup_time; - graphiteStats.last_exception = startup_time; + graphiteStats.last_exception = 0; graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/backends/repeater.js b/backends/repeater.js index 5251e589..ccbf75eb 100644 --- a/backends/repeater.js +++ b/backends/repeater.js @@ -2,43 +2,152 @@ var util = require('util') , dgram = require('dgram') - , logger = require('../lib/logger'); + , logger = require('../lib/logger') + , Pool = require('generic-pool').Pool + , net = require('net'); + var l; var debug; +var instance; + +function logerror(err) { + if(err && debug) { + l.log(err); + } +} -function RepeaterBackend(startupTime, config, emitter){ + + +function UDPRepeaterBackend(startupTime, config, emitter) { var self = this; this.config = config.repeater || []; this.sock = (config.repeaterProtocol == 'udp6') ? dgram.createSocket('udp6') : dgram.createSocket('udp4'); + // Attach DNS error handler this.sock.on('error', function (err) { if (debug) { l.log('Repeater error: ' + err); } }); + // attach emitter.on('packet', function(packet, rinfo) { self.process(packet, rinfo); }); } -RepeaterBackend.prototype.process = function(packet, rinfo) { + +UDPRepeaterBackend.prototype.process = function(packet, rinfo) { var self = this; - hosts = self.config; + var hosts = self.config; for(var i=0; i Tue, 22 Aug 2023 13:14:48 +0000 + +statsd (0.10.1-1) unstable; urgency=low + + * Include tag pushes in docker builds + + -- Mike Heffner Thu, 11 May 2023 16:02:29 +0000 + +statsd (0.10.0-1) unstable; urgency=low + + * Multiple documentation updates + * Updated docker-compose syntax + * Upgrade tests to use python3 + * Upgrade node base for Docker image + * Automatic container publishing via GHA + + -- Mike Heffner Thu, 11 May 2023 13:03:29 +0000 + +statsd (0.9.0-1) unstable; urgency=low + + * Add support for graphite tagged metrics + * Fix dashboard to 0 last_exception time on startup + * Multiple documentation updates + * Correct some out-dated integration examples + * Update container image to use recent node version + + -- Elliot Blackburn Thu, 27 Aug 2020 15:30:29 +0000 + +statsd (0.8.6-1) unstable; urgency=low + + * Add an optional max TTL setting for gauges + * add filter option for metrics + * fix double flush scenario in docker containers + + -- Elliot Blackburn Tue, 19 Feb 2020 00:43:00 +0000 + +statsd (0.8.5-1) unstable; urgency=low + + * Update lodash (sub dependency) for security fix + * Add the statsd history to the docs + * Add third party server interfaces to docs + * Migrate docs from github wiki, and standardise markdown notation + * Minor formatting proposals + * Add docker image info to readme + + -- Elliot Blackburn Tue, 23 Jul 2019 00:00:00 +0000 + +statsd (0.8.4-1) unstable; urgency=low + + * update modern-syslog to 1.2.0 for node 12 compatibility + * update package.json version to 0.8.3 + + -- Elliot Blackburn Thu, 11 Jul 2019 00:00:00 +0000 + +statsd (0.8.3-1) unstable; urgency=low + + * correct backend flush loop + * test and declare support for Current and LTS node + * Correct reporter decleration in test runner + * Convert codebase from var -> let / const (#673) + * correct npm test script + * correct travis deploy step + + -- Elliot Blackburn Thu, 11 Jul 2019 00:00:00 +0000 + +statsd (0.8.2-1) unstable; urgency=low + + * update travis npm token + * update dockerfile to latest node-lts + * correct gitter link + * update dockerfile base image to node lts + * Add gitter chat badge + * run tests using python 3.7's pickle rather than 2.x cPickle (#669) + + -- Elliot Blackburn Tue, 02 Apr 2019 00:00:00 +0000 + +statsd (0.8.1-1) unstable; urgency=low + + * drop statsd instance from proxy ring in instance of healthcheck failures (#665) + * Add myself (elliot blackburn) to maintainers.md (#666) + * add mysql backend link to docs/backend + -- Elliot Blackburn Wed, 13 Mar 2019 00:00:00 +0000 + +statsd (0.8.1-1) unstable; urgency=low + + * drop statsd instance from proxy ring in instance of healthcheck failures (#665) + * Add myself (elliot blackburn) to maintainers.md (#666) + * add mysql backend link to docs/backend.md + * Adding myself to MAINTAINERS. + * begin testing on node lts and up + * remove extra newline from merge conflict fix + * Added "opencensus-backend" + * correct package.json links to new github organisation + * Update MAINTAINERS.md + * Update MAINTAINERS.md + * remove meta section of README + * Create MAINTAINERS.md + * Create DCO.txt + * Create CODE_OF_CONDUCT.md + * update README post transfer + * Fixing Markdown formatting + * fix simple typo + * Added StatsdClient Kotlin implementation + * fix formatting on backend interface docs + * Update: ignore files + * removes -q switch + * Fix for failing test on node 0.10 + * fix usage of process.EventEmitter + * Add plugin Warp10 to statsd + * Updated graphite link to read the docs + + -- Elliot Blackburn Wed, 13 Mar 2019 00:00:00 +0000 + +statsd (0.8.0-1) unstable; urgency=low + + * Modularized injest servers, with support for loading multiple servers + * Added configurable tcp injest server + * Added tcp repeater functionality + * Added unix socket injest support + * Added pickle protocol support to graphite backend + * Added configurable IPv6 and TCP support to proxy + * Added telnet admin interface to proxy + * Multiple variable scoping fixes + * Fixes to flush timer to reduce bucket drift + * Fixes to ruby and java example client code + * Dropped support for node v0.8.x + * Fixed dependency issues for modern node versions + * Updated npm hashring dependency to v3.2.0 + * Replaced npm node-syslog dependency with modern-syslog v1.1.2 + * Add systemd services for statsd and statsd-proxy + * Add servers directory to package + * Removed duplicated libs from package + * Add npm dependency for postinst + + -- Patrick Koch Thu, 5 May 2016 01:00:00 +0000 + +statsd (0.7.2-1) unstable; urgency=low + + * Fixes to detecting valid packets + * Align version number to git tag versions + * Add statsd_proxy upstart + * Install node dependencies on package installation + * Add proxyConfig.js + * Remove init script that does not work in favor of just having upstart + * Fix node version dependency + + -- Joseph Hughes Thu, 16 Apr 2014 13:03:10 +0200 + +statsd (0.7.1-1) unstable; urgency=low + + * move contributing information into CONTRIBUTING.md + * Updates winser to v0.1.6 + * examples: python: added efficiency note + * python: examples: fixed doctests for Python 3 + * Standardized debian log locations + * Enhancement: consume logger in graphite and repeater backends + * Enhancement: update backend documentation + * Enhancement: inject logger object into backend + * Send STDOUT and STDERR to the appropriate files + + -- Unknown Author <> Thu, 6 Feb 2014 01:00:00 +0000 + +statsd (v0.7.0-1) unstable; urgency=low + + * added cluster proxy + * measure and graph timestamp generation lag + * added median calculation for timers + * support for top percentiles for timers + * drop support for node v0.6.x + * support for setting the process title + * functionality for optionally omitting stats_counts metrics + * improved functionality to delete counters from the management console + * updates to Debian packaging + * added a clojure example client + * cleaned up the Go example client + * increased test coverage + * documentation updates + + -- Unknown Author <> Fri, 5 Dec 2014 01:00:00 +0000 + statsd (0.6.0-1) unstable; urgency=low * Non-maintainer upload. diff --git a/debian/control b/debian/control index 53dff2d2..c1434984 100644 --- a/debian/control +++ b/debian/control @@ -1,13 +1,13 @@ Source: statsd Section: devel Priority: optional -Maintainer: Stephen Koenig +Maintainer: Elliot Blackburn Standards-Version: 3.9.3 -Build-Depends: debhelper (>= 8.0.0) +Build-Depends: debhelper (>= 8.0.0), dh-systemd (>= 1.5) Package: statsd Architecture: all -Depends: nodejs (>= 0.6), adduser +Depends: nodejs (>= 8.0), adduser, npm Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. diff --git a/debian/postinst b/debian/postinst index e92c3698..9e1c11c7 100755 --- a/debian/postinst +++ b/debian/postinst @@ -4,6 +4,8 @@ set -e if [ "$1" = configure ]; then # Automatically added by dh_installinit + (cd /usr/share/statsd && /usr/bin/npm install --production) + if ! getent passwd _statsd > /dev/null; then adduser --system --quiet --home /nonexistent --no-create-home \ --shell /bin/false --force-badname --group --gecos "StatsD User" _statsd @@ -13,8 +15,13 @@ if [ "$1" = configure ]; then dpkg-statoverride --update --add _statsd _statsd 0755 /var/run/statsd fi - if [ -x "/etc/init.d/statsd" ]; then - update-rc.d statsd defaults >/dev/null - invoke-rc.d statsd start || exit $? - fi + cat << EOF +Available systemd services are disabled by default: + * statsd + * statsd-proxy + +To enable one of them use: + systemctl enable NAME +EOF + fi diff --git a/debian/proxyConfig.js b/debian/proxyConfig.js new file mode 100644 index 00000000..b1ab9444 --- /dev/null +++ b/debian/proxyConfig.js @@ -0,0 +1,11 @@ +{ +nodes: [ +{host: 'localhost', port: 8125, adminport: 8126}, +], +udp_version: 'udp4', +host: '0.0.0.0', +port: 8127, +forkCount: 0, +checkInterval: 1000, +cacheSize: 10000 +} diff --git a/debian/rules b/debian/rules index 2d33f6ac..88df0139 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,8 @@ #!/usr/bin/make -f %: - dh $@ + dh $@ --with systemd + +override_dh_installinit: + dh_installinit --name=statsd -- defaults + dh_installinit --name=statsd-proxy -- defaults diff --git a/debian/statsd-proxy.upstart b/debian/statsd-proxy.upstart new file mode 100644 index 00000000..2a04d718 --- /dev/null +++ b/debian/statsd-proxy.upstart @@ -0,0 +1,28 @@ +# statsd - Network daemon for aggregating statistics +# +# This is a network service that receives metric data via UDP from other +# applications. It aggregates this data and flushes it to a storage backend +# (typically Graphite) at regular intervals. +# +description "Network daemon for aggregating statistics" +author "Etsy" + +start on (local-filesystems and net-device-up IFACE!=lo) + +setuid _statsd +setgid _statsd + +respawn +respawn limit 10 5 + +chdir /usr/share/statsd + +pre-start script + NODE_BIN=$(which nodejs || which node) + [ -n $NODE_BIN ] || { stop; exit 0; } +end script + +script + NODE_BIN=$(which nodejs || which node) + exec $NODE_BIN proxy.js /etc/statsd/proxyConfig.js +end script diff --git a/debian/statsd.init b/debian/statsd.init deleted file mode 100644 index 960aefea..00000000 --- a/debian/statsd.init +++ /dev/null @@ -1,169 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: statsd -# Required-Start: $remote_fs $network $local_fs -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -### END INIT INFO - -# Do NOT "set -e" - -PATH=$PATH:/usr/local/bin:/usr/bin:/bin -NODE_BIN=$(which nodejs||which node) - -if [ ! -x "$NODE_BIN" ]; then - echo "Can't find executable nodejs or node in PATH=$PATH" - exit 1 -fi - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="StatsD" -NAME=statsd -USER=_statsd -DAEMON=$NODE_BIN -DAEMON_ARGS="/usr/share/statsd/stats.js /etc/statsd/localConfig.js" -PIDFILE=/var/run/$NAME/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -CHDIR="/usr/share/statsd" - -# Exit if the package is not installed -# [ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. -. /lib/lsb/init-functions - -# Create PIDDIR on runtime -if [ ! -d /var/run/$NAME ]; -then - mkdir /var/run/$NAME - chown $USER /var/run/$NAME -fi - -# Create LOGDIR on runtime -mkdir -p /var/log/$NAME - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet -m --pidfile $PIDFILE --startas $DAEMON --chuid $USER:$USER --background --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet -m --pidfile $PIDFILE --startas $DAEMON --chuid $USER:$USER --background --no-close --chdir $CHDIR -- \ - $DAEMON_ARGS >> /var/log/$NAME/$NAME.log 2> /var/log/$NAME/stderr.log \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=0/0/KILL/5 --pidfile $PIDFILE - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON - [ "$?" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME - return 0 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - status) - status_of_proc -p $PIDFILE $DAEMON "$NAME" && exit 0 || exit $? - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|restart|status|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/debian/statsd.install b/debian/statsd.install index 21b225b5..7933ef02 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -1,6 +1,8 @@ -stats.js /usr/share/statsd +stats.js /usr/share/statsd +proxy.js /usr/share/statsd +package.json /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends -lib/*.js /usr/share/statsd/lib servers/*.js /usr/share/statsd/servers debian/localConfig.js /etc/statsd +debian/proxyConfig.js /etc/statsd diff --git a/debian/statsd.statsd-proxy.service b/debian/statsd.statsd-proxy.service new file mode 100644 index 00000000..a832b061 --- /dev/null +++ b/debian/statsd.statsd-proxy.service @@ -0,0 +1,14 @@ +[Unit] +Description=Network daemon for aggregating statistics +Documentation=https://github.com/etsy/statsd/ +Wants=network.target + +[Service] +Type=simple +User=_statsd +ExecStart=/usr/bin/nodejs /usr/share/statsd/proxy.js /etc/statsd/proxyConfig.js +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/debian/statsd.statsd.service b/debian/statsd.statsd.service new file mode 100644 index 00000000..c672fcaf --- /dev/null +++ b/debian/statsd.statsd.service @@ -0,0 +1,14 @@ +[Unit] +Description=Network daemon for aggregating statistics +Documentation=https://github.com/etsy/statsd/ +Wants=network.target + +[Service] +Type=simple +User=_statsd +ExecStart=/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/localConfig.js +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..7704916f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '2' +services: + statsd: + build: . + links: + - carbon:graphite + ports: + - 8125:8125/udp + - 8126:8126 + + graphite-web: + image: dockerana/graphite + links: + - carbon + ports: + - 8000:8000 + volumes_from: + - carbon + + carbon: + image: dockerana/carbon + ports: + - 2003:2003 + - 2004:2004 + - 7002:7002 + volumes: + - /opt/graphite diff --git a/docs/additional_tools.md b/docs/additional_tools.md new file mode 100644 index 00000000..251c36aa --- /dev/null +++ b/docs/additional_tools.md @@ -0,0 +1,5 @@ +# Additional Tools + +The following are tools you might find useful when using, contributing, or testing StatsD. + +* [statsd-tg](http://octo.it/statsd-tg) – StatsD traffic generator; generates dummy traffic for load testing (C). diff --git a/docs/admin_interface.md b/docs/admin_interface.md index 853d5a00..beeb9b40 100644 --- a/docs/admin_interface.md +++ b/docs/admin_interface.md @@ -1,25 +1,31 @@ -TCP Stats Interface -------------------- +# TCP Stats Interface -A really simple TCP management interface is available by default on port 8126 +A really simple TCP management interface is available by default on port `8126` or overriden in the configuration file. Inspired by the memcache stats approach -this can be used to monitor a live statsd server. You can interact with the -management server by telnetting to port 8126, the following commands are -available: +this can be used to monitor a live StatsD server. You can interact with the +management server by telnetting to port `8126`, the following commands are +available based on the running server. + +## Common commands + +* health [up|down] - a way to get/set the health status of StatsD. Alone will get you the current health status. Passing a second command will set the status to the new value. Accepted values are _up_ and _down_. +* config - a dump of the current configuration +* quit - close the connection from the server side + +## StatsD specific commands * stats - some stats about the running server * counters - a dump of all the current counters * gauges - a dump of all the current gauges * timers - a dump of the current timers * delcounters - delete a counter or folder of counters -* delgauges - delete a gauge or folder of gauges +* delgauges - delete a gauge or folder of gauges * deltimers - delete a timer or folder of timers -* health - a way to set the health status of statsd The stats output currently will give you: -* uptime: the number of seconds elapsed since statsd started -* messages.last_msg_seen: the number of elapsed seconds since statsd received a message +* uptime: the number of seconds elapsed since StatsD started +* messages.last_msg_seen: the number of elapsed seconds since StatsD received a message * messages.bad_lines_seen: the number of bad lines seen since startup You can use the del commands to delete an individual metric like this : @@ -31,7 +37,7 @@ Or you can use the del command to delete a folder of metrics like this : #to delete counters sandbox.test.* echo "delcounters sandbox.test.*" | nc 127.0.0.1 8126 - + Each backend will also publish a set of statistics, prefixed by its module name. @@ -46,7 +52,7 @@ Those statistics will also be sent to graphite under the namespaces `stats.statsd.graphiteStats.last_exception` and `stats.statsd.graphiteStats.last_flush`. -A simple nagios check can be found in the utils/ directory that can be used to +A simple nagios check can be found in the `utils/` directory that can be used to check metric thresholds, for example the number of seconds since the last successful flush to graphite. @@ -55,3 +61,11 @@ The health output: * using health up or health down, you can change the current health status. * the healthStatus configuration option allows you to set the default health status at start. +## StatsD Proxy specific commands + +* status - the status of the current server + +The __status__ output currently will give you: + +* uptime: the number of seconds elapsed since StatsD proxy started +* nodes: a space separated list of host:port for each active node in the ring diff --git a/docs/backend.md b/docs/backend.md index c36ae8df..e9d82dd7 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -1,5 +1,4 @@ -Supported Backends ------------------- +# Supported Backends StatsD supports pluggable backend modules that can publish statistics from the local StatsD daemon to a backend service or data @@ -30,23 +29,38 @@ giving the relative path (e.g. `./backends/graphite`). A robust set of are also available as plugins to allow easy reporting into databases, queues and third-party services. -## Available Third-party backends -- [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) -- [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) -- [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) -- [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) -- [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) -- [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) -- [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) -- [librato-backend](https://github.com/librato/statsd-librato-backend) -- [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) -- [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) -- [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) -- [socket.io-backend](https://github.com/Chatham/statsd-socket.io) -- [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) -- [statsd-backend](https://github.com/dynmeth/statsd-backend) -- [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) -- [statsd aggregation backend](https://github.com/wanelo/gossip_girl) -- [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) - -[graphite]: http://graphite.wikidot.com +## Available third-party backends + +* [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) +* [atsd-backend](https://github.com/axibase/atsd-statsd-backend) +* [aws-cloudwatch-backend](https://github.com/camitz/aws-cloudwatch-statsd-backend) +* [node-bell](https://github.com/eleme/node-bell) +* [coralogix-backend](https://github.com/coralogix/statsd-coralogix-backend) +* [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) +* [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) +* elasticsearch-backend + * [Elasticsearch 5 and 6](https://github.com/markkimsal/statsd-elasticsearch-backend) + * [Elasticsearch 7](https://github.com/lorenzoaiello/statsd-elasticsearch7-backend) +* [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) +* [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) +* [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) +* [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) +* [jut-backend](https://github.com/jut-io/statsd-jut-backend) +* [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) +* [librato-backend](https://github.com/librato/statsd-librato-backend) +* [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) +* [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) +* [mysql backend](https://github.com/fradinni/nodejs-statsd-mysql-backend) +* [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) +* [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) +* [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) +* [redistimeseries backend](https://github.com/hashedin/statsd-redistimeseries-backend) +* [socket.io-backend](https://github.com/Chatham/statsd-socket.io) +* [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) +* [statsd-backend](https://github.com/dynmeth/statsd-backend) +* [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) +* [statsd aggregation backend](https://github.com/wanelo/gossip_girl) +* [warp10-backend](https://github.com/cityzendata/statsd-warp10-backend) +* [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) + +[graphite]: https://graphite.readthedocs.io/en/latest/ diff --git a/docs/backend_interface.md b/docs/backend_interface.md index bbfc6f01..2cd2a4fb 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -1,5 +1,4 @@ -Backend Interface ------------------ +# Backend Interface Backend modules are Node.js [modules][nodemods] that listen for a number of events emitted from StatsD. Each backend module should @@ -29,7 +28,7 @@ the `events` object: and `metrics` is a hash representing the StatsD statistics: ``` -metrics: { + metrics: { counters: counters, gauges: gauges, timers: timers, @@ -38,12 +37,12 @@ metrics: { timer_data: timer_data, statsd_metrics: statsd_metrics, pctThreshold: pctThreshold -} + } ``` The counter_rates and timer_data are precalculated statistics to simplify the creation of backends, the statsd_metrics hash contains metrics generated - by statsd itself. Each backend module is passed the same set of + by StatsD itself. Each backend module is passed the same set of statistics, so a backend module should treat the metrics as immutable structures. StatsD will reset timers and counters after each listener has handled the event. @@ -70,5 +69,3 @@ metrics: { This is emitted for every incoming packet. The `packet` parameter contains the raw received message string and the `rinfo` parameter contains remote address information from the UDP socket. - - diff --git a/docs/client_implementations.md b/docs/client_implementations.md new file mode 100644 index 00000000..f748c808 --- /dev/null +++ b/docs/client_implementations.md @@ -0,0 +1,111 @@ +# StatsD Clients + +A number of clients have been made for pushing metrics into StatsD and open sourced by the wider community. + +**Node** +* [lynx](https://github.com/dscape/lynx) — Node.js client used by Mozilla, Nodejitsu, etc. +* [Node-Statsd](https://github.com/sivy/node-statsd) — Node.js client +* [node-statsd-client](https://github.com/msiebuhr/node-statsd-client) — Node.js client +* [node-statsd-instrument](https://github.com/syrio/node-statsd-instrument) — Node.js client +* [statistik](https://github.com/godmodelabs/statistik) - Node.js client with timers & CLI +* [statsy](https://github.com/segmentio/statsy) - clean idiomatic statsd client + +**Java** +* [java-statsd-client](https://github.com/youdevise/java-statsd-client) — Lightweight (zero deps) Java client +* [Statsd over SLF4J](https://github.com/nzjess/statsd-over-slf4j) — Java client with SLF4J logging tie-in +* [play-statsd](https://github.com/vznet/play-statsd) — Play Framework 2.0 client for Java and Scala +* [statsd-netty](https://github.com/flozano/statsd-netty) — Netty-based Java 8 client + +**Python** +* [Py-Statsd](https://github.com/sivy/py-statsd) — Server and Client +* [Python-Statsd](https://github.com/WoLpH/python-statsd) — Python client +* [pystatsd](https://github.com/jsocol/pystatsd) — Python client +* [Django-Statsd](https://github.com/WoLpH/django-statsd) — Django client + +**Ruby** +* [statsd-instrument](https://github.com/Shopify/statsd-instrument) — Ruby client +* [statsd](https://github.com/reinh/statsd/) — Ruby client (needs new maintainer) +* [Statsd-Client](https://github.com/dawanda/statsd-client) — Ruby client (not maintained) + +**Swift** +* [swift-statsd-client](https://github.com/apple/swift-statsd-client) - Swift client + +**Perl** +* [Net::Statsd](https://github.com/cosimo/perl5-net-statsd) — Perl client, also available on [CPAN](https://metacpan.org/module/Net::Statsd) +* [Net::StatsD::Client](https://github.com/sivy/statsd-client) — Perl client, not available on CPAN +* [Etsy::StatsD](https://github.com/sanbeg/Etsy-Statsd) - Perl client, also available on [CPAN] (https://metacpan.org/module/Etsy::StatsD) + +**PHP** +* [Metrics](https://github.com/beberlei/metrics#metrics) +* [PHP client](https://gist.github.com/1065177/5f7debc212724111f9f500733c626416f9f54ee6) +* [php-statsd](https://github.com/seejohnrun/php-statsd) and Spark +* [php-statsd-client](https://github.com/godmodelabs/php-statsd-client) - supports SplClassLoader +* [statsd-php-client](https://github.com/iFixit/statsd-php-client) - Minimalist performant client +* [phpLeague-statsd-client](https://github.com/thephpleague/statsd) - Php League StatsD client +* [statsd-php-client](https://github.com/liuggio/statsd-php-client) - optimized client with monolog and symfony2 integrations available +* [statsd-php](https://github.com/domnikl/statsd-php) - PSR-4 compatible client + +**Clojure** +* [Clojure client](https://github.com/pyr/clj-statsd) + +**Io** +* [io-statsd](https://github.com/seejohnrun/io-statsd) — StatsD Client for Io + +**C** +* [C client](https://github.com/romanbsd/statsd-c-client) — A trivial C client + +**C++** +* [statsd-client-cpp](https://github.com/talebook/statsd-client-cpp) — StatsD Client in CPP +* [cpp-statsd-client](https://github.com/vthiery/cpp-statsd-client) — A header-only StatsD client implemented in C++ + +**.NET** +* [NStatsD.Client](https://github.com/robbihun/NStatsD.Client) — .NET 4.0 client +* [C# client](https://github.com/goncalopereira/statsd-csharp-client) — C# client +* [graphite-client](https://github.com/peschuster/graphite-client) — .NET client library for StatsD and Graphite +* [StatsC](https://bitbucket.org/pavlos256/statsc) — An asynchronous client with built-in support for batching +* [JustEat.StatsD](https://github.com/justeat/JustEat.StatsD) — A .NET library for publishing metrics to statsd. Targets both .NET full framework and .NET Standard 2.0. + +**Go** +* [GoE](https://godoc.org/github.com/pascaldekloe/goe/metrics) — Minimal & Performant +* [go-statsd-client](https://github.com/cactus/go-statsd-client) — Simple Go client +* [g2s](https://github.com/peterbourgon/g2s) +* [StatsD](https://github.com/quipo/statsd) +* [statsd](https://github.com/alexcesaro/statsd) — A simple and very fast StatsD client + +**Apache** +* [mod_statsd](https://github.com/jib/mod_statsd) - StatsD client to send stats straight from [Apache](https://modules.apache.org/) + +**Varnish** +* [libvmod-statsd](https://github.com/jib/libvmod-statsd) - StatsD client to send stats straight from [Varnish](http://varnish-cache.org) + +**PowerShell** +* [powershell-statsd](https://github.com/joehack3r/powershell-statsd) - PowerShell client + +**Browser** +* [StatsC](https://github.com/godmodelabs/statsc) - Push stats to StatsD from the browser! +* [StatsD HTTP Proxy](https://github.com/sokil/statsd-http-proxy) - HTTP proxy to StatsD with REST interface for using in browsers +* [StatsD HTTP Client](https://github.com/Molyakos/statsd-http-client) - StatsD client over http for using in browsers + +**Objective-C** +* [MCStatsd](https://github.com/Marketcircle/MCStatsd) - Cocoa client + +**ActionScript** +* [flash-statsd](https://github.com/simongregory/flash-statsd) - Flash client + +**WordPress** +* [wordpress-statsd](https://github.com/uglyrobot/wordpress-statsd) - WordPress Plugin + +**Drupal** +* [StatsD](https://www.drupal.org/project/statsd) - Drupal module + +**Haskell** +* [statsd-client](https://github.com/keithduncan/statsd-client) + +**R** +* [rstatsd](https://github.com/stumpyfr/rstatsd) + +**Lua** +* [lua-statsd](https://github.com/stvp/lua-statsd-client) + +**Nim** +* [statsd_client](https://github.com/FedericoCeratto/nim-statsd-client) diff --git a/docs/cluster_proxy.md b/docs/cluster_proxy.md index ca2d39a2..4d12403a 100644 --- a/docs/cluster_proxy.md +++ b/docs/cluster_proxy.md @@ -1,28 +1,26 @@ -Statsd Cluster Proxy -============== - -Statsd Cluster Proxy is a udp proxy that sits infront of multiple statsd instances. +# StatsD Cluster Proxy +StatsD Cluster Proxy is a udp proxy that sits infront of multiple StatsD instances. Create a proxyConfig.js file: `cp exampleProxyConfig.js proxyConfig.js` Once you have modified your config file run: - + `node proxy.js proxyConfig.js` -It uses a consistent hashring to send the unique metric names to the same statsd instances so that +It uses a consistent hashring to send the unique metric names to the same StatsD instances so that the aggregation works properly. -It handles a simple health check that dynamically recalculates the hashring if a statsd instance goes offline. +It handles a simple health check that dynamically recalculates the hashring if a StatsD instance goes offline. Config Options are documented in the [exampleProxyConfig.js][exampleProxyConfig.js] -Notes --------------- -In your statsd configuration make sure to have the following configuration set: `deleteIdleStats: true` +## Notes + +In your StatsD configuration make sure to have the following configuration set: `deleteIdleStats: true` We plan to remove this restriction in the near future: [#pull/348][pull_348] diff --git a/docs/graphite.md b/docs/graphite.md index 7a7bfc55..064fbdf0 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -1,23 +1,22 @@ -Configuring Graphite for StatsD -------------------------------- +# Configuring Graphite for StatsD Many users have been confused to see their hit counts averaged, gone missing when -the data is intermittent, or never stored when statsd is sending at a different +the data is intermittent, or never stored when StatsD is sending at a different interval than graphite expects. Careful setup of Graphite as suggested below should help to alleviate all these issues. When configuring Graphite, two main factors you need to consider are: -1. What is the highest resolution of data points kept by Graphite, and at which points in time is data downsampled to lower resolutions. This decision is by nature directly related to your functional requirements: how far back should you keep data? what is the data resolution you actually need? However, the retention rules you set must also be in sync with statsd. +1. What is the highest resolution of data points kept by Graphite, and at which points in time is data downsampled to lower resolutions. This decision is by nature directly related to your functional requirements: how far back should you keep data? what is the data resolution you actually need? However, the retention rules you set must also be in sync with StatsD. -2. How should data be aggregated when downsampled, in order to correctly preserve its meaning? Graphite of course knows nothing of the 'meaning' of your data, so let's explore the correct setup for the various metrics sent by statsd. +2. How should data be aggregated when downsampled, in order to correctly preserve its meaning? Graphite of course knows nothing of the 'meaning' of your data, so let's explore the correct setup for the various metrics sent by StatsD. ### Storage Schemas -To define retention and downsampling which match your needs, edit Graphite's conf/storage-schemas.conf file. Here is a simple example file that would handle all metrics sent by statsd: +To define retention and downsampling which match your needs, edit Graphite's conf/storage-schemas.conf file. Here is a simple example file that would handle all metrics sent by StatsD: [stats] pattern = ^stats.* - retentions = 10s:6h,1min:6d,10min:1800d + retentions = 10s:6h,1m:6d,10m:1800d -This translates to: for all metrics starting with 'stats' (i.e. all metrics sent by statsd), capture: +This translates to: for all metrics starting with 'stats' (i.e. all metrics sent by StatsD), capture: * 6 hours of 10 second data (what we consider "near-realtime") * 6 days of 1 minute data @@ -25,12 +24,12 @@ This translates to: for all metrics starting with 'stats' (i.e. all metrics sent These settings have been a good tradeoff so far between size-of-file (database files are fixed size) and data we care about. Each "stats" database file is about 3.2 megs with these retentions. -Retentions are read from the file in order and the first pattern that matches is used. +Retentions are read from the file in order and the first pattern that matches is used. Graphite stores each metric in its own database file, and the retentions take effect when a metric file is first created. This means that changing this config file would not affect any files already created. To view or alter the settings on existing files, use whisper-info.py and whisper-resize.py included with the Whisper package. -#### Correlation with statsd's flush interval: +#### Correlation with StatsD's flush interval: -In the case of the above example, what would happen if you flush from statsd any faster than every 10 seconds? in that case, multiple values for the same metric may reach Graphite at any given 10-second timespan, and only the last value would take hold and be persisted - so your data would immediately be partially lost. +In the case of the above example, what would happen if you flush from StatsD any faster than every 10 seconds? in that case, multiple values for the same metric may reach Graphite at any given 10-second timespan, and only the last value would take hold and be persisted - so your data would immediately be partially lost. To fix that, simply ensure your flush interval is at least as long as the highest-resolution retention. However, a long interval may cause other unfortunate mishaps, so keep reading - it pays to understand what's really going on. @@ -38,15 +37,15 @@ To fix that, simply ensure your flush interval is at least as long as the highes ### Storage Aggregation -The next step is ensuring your data isn't corrupted or discarded when downsampled. Continuing with the example above, take for instance the downsampling of .mean values calculated for all statsd timers: +The next step is ensuring your data isn't corrupted or discarded when downsampled. Continuing with the example above, take for instance the downsampling of .mean values calculated for all StatsD timers: -Graphite should downsample up to 6 samples representing 10-second mean values into a single value signfying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurences per flush interval, so you want these samples summed-up, not averaged! +Graphite should downsample up to 6 samples representing 10-second mean values into a single value signifying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurrences per flush interval, so you want these samples summed-up, not averaged! You would not even notice any problem till you look at a graph for data older than 6 hours ago, since Graphite would need only the high-res 10-second samples to render the first 6 hours, but would have to switch to lower resolution data for rendering a longer timespan. -Two other metric kinds also deserve a note: +Two other metric kinds also deserve a note: -* Counts which are normalized by statsd to signify a per-second count should not be summed, since their meaning does not change when downsampling. +* Counts which are normalized by StatsD to signify a per-second count should not be summed, since their meaning does not change when downsampling. * Metrics for minimum/maximum values should not be averaged but rather preserve the lowest/highest point, respectively. @@ -101,6 +100,6 @@ Similar to retentions, the aggregations in effect for any metric are set once th ### In conclusion -Graphite's handling of your statsd metrics should be verified at least once: is data mysteriously lost at any point? is data downsampled properly? are you defining graphs for counter metrics without knowing what timespan does each y-value actually represent? (admittedly, in some cases you may not even care about the y-values in the graph, as only the trend is of any interest. The coolest graphs seem to always lack y-values...) +Graphite's handling of your StatsD metrics should be verified at least once: is data mysteriously lost at any point? is data downsampled properly? are you defining graphs for counter metrics without knowing what timespan does each y-value actually represent? (admittedly, in some cases you may not even care about the y-values in the graph, as only the trend is of any interest. The coolest graphs seem to always lack y-values...) For more information, see: http://graphite.readthedocs.org/en/latest/config-carbon.html diff --git a/docs/graphite_pickle.md b/docs/graphite_pickle.md new file mode 100644 index 00000000..71122b39 --- /dev/null +++ b/docs/graphite_pickle.md @@ -0,0 +1,78 @@ +# Pickling for Graphite + +The graphite StatsD backend can optionally be configured to use pickle +for its over-the-wire protocol. + +```javascript + { graphiteHost: "your.graphite.host", + graphiteProtocol: "pickle" } +``` + +The default is to use the graphite text protocol, which can require +more CPU processing by the graphite endpoint. + +The message format expected by the graphite pickle endpoint consists +of a header and payload. + +## The Payload + +The message payload is a list of tuples. Each tuple contains the measurement +for a single metric name. The measurement is encoded as a second, +nested tuple containing timestamp and measured value. + +This ends up looking like: + +```python +[ ( "path.to.metric.name", ( timestamp, "value" ) ), + ( "path.to.another.name", ( timestamp, "value" ) ) ] +``` + +The graphite receiver `carbon.protocols.MetricPickleReceiver` coerces +both the timestamp and measured value into `float`. + +The timestamp must be seconds since epoch encoded as a number. + +The measured value is encoded as a string. This may change in the +future. + +We have chosen to not implement pickle's object memoization. This +simplifies what is sent across the wire. It is not likely any +optimization would result within a single poll cycle. + +Here is some Python code showing how a given set of metrics can be +serialized in a more simple way. + +```python +import pickle + +metrics = [ ( "a.b.c", ( 1234L, "5678" ) ), ( "d.e.f.g", ( 1234L, "9012" ) ) ] +pickle.dumps(metrics) +# "(lp0\n(S'a.b.c'\np1\n(L1234L\nS'5678'\np2\ntp3\ntp4\na(S'd.e.f.g'\np5\n(L1234L\nS'9012'\np6\ntp7\ntp8\na." + +payload = "(l(S'a.b.c'\n(L1234L\nS'5678'\ntta(S'd.e.f.g'\n(L1234L\nS'9012'\ntta." +pickle.loads(payload) +# [('a.b.c', (1234L, '5678')), ('d.e.f.g', (1234L, '9012'))] +``` + +The trailing `L` for long fields is unnecessary, but we are adding the +character to match Python pickle output. It's a side-effect of +`repr(long(1234))`. + +## The Header + +The message header is a 32-bit integer sent over the wire as +four-bytes. This integer must describe the length of the pickled +payload. + +Here is some sample code showing how to construct the message header +containing the payload length. + +```python +import struct + +payload_length = 81 +header = struct.pack("!L", payload_length) +# '\x00\x00\x00Q' +``` + +The `Q` character is equivalent to `\x81` (ASCII encoding). diff --git a/docs/history.md b/docs/history.md new file mode 100644 index 00000000..c96b0fb0 --- /dev/null +++ b/docs/history.md @@ -0,0 +1,7 @@ +# What is StatsD? + +StatsD is a front-end proxy for the Graphite/Carbon metrics server, +originally written by Etsy's Erik Kastner. It is based on ideas from +Flickr and this post by Cal Henderson: Counting and Timing. The +server was written in Node, though there have been implementations +in other languages since then. diff --git a/docs/metric_types.md b/docs/metric_types.md index e839cd07..06c15839 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -1,9 +1,6 @@ -StatsD Metric Types -================== +# StatsD Metric Types - -Counting --------- +## Counting gorets:1|c @@ -11,18 +8,17 @@ This is a simple counter. Add 1 to the "gorets" bucket. At each flush the current count is sent and reset to 0. If the count at flush is 0 then you can opt to send no metric at all for this counter, by setting `config.deleteCounters` (applies only to graphite -backend). Statsd will send both the rate as well as the count at each flush. +backend). StatsD will send both the rate as well as the count at each flush. -### Sampling +## Sampling gorets:1|c|@0.1 Tells StatsD that this counter is being sent sampled every 1/10th of the time. -Timing ------- +## Timing - glork:320|ms + glork:320|ms|@0.1 The glork took 320ms to complete this time. StatsD figures out percentiles, average (mean), standard deviation, sum, lower and upper bounds for the flush interval. @@ -35,7 +31,7 @@ generate the following list of stats for each threshold: stats.timers.$KEY.upper_$PCT stats.timers.$KEY.sum_$PCT -Where `$KEY` is the stats key you specify when sending to statsd, and `$PCT` is +Where `$KEY` is the stats key you specify when sending to StatsD, and `$PCT` is the percentile threshold. Note that the `mean` metric is the mean value of all timings recorded during @@ -47,11 +43,11 @@ more detailed explanation of the calculation. If the count at flush is 0 then you can opt to send no metric at all for this timer, by setting `config.deleteTimers`. -Use the `config.histogram` setting to instruct statsd to maintain histograms +Use the `config.histogram` setting to instruct StatsD to maintain histograms over time. Specify which metrics to match and a corresponding list of ordered non-inclusive upper limits of bins (class intervals). (use `inf` to denote infinity; a lower limit of 0 is assumed) -Each `flushInterval`, statsd will store how many values (absolute frequency) +Each `flushInterval`, StatsD will store how many values (absolute frequency) fall within each bin (class interval), for all matching metrics. Examples: @@ -67,6 +63,10 @@ Examples: [ { metric: 'foo', bins: [] }, { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] +StatsD also maintains a counter for each timer metric. The 3rd field +specifies the sample rate for this counter (in this example @0.1). The field +is optional and defaults to 1. + Note: * first match for a metric wins. @@ -75,9 +75,9 @@ Note: histograms, as you can make each bin arbitrarily wide, i.e. class intervals of different sizes. -Gauges ------- -StatsD now also supports gauges, arbitrary values, which can be recorded. +## Gauges + +StatsD also supports gauges. A gauge will take on the arbitrary value assigned to it, and will maintain its value until it is next set. gaugor:333|g @@ -97,9 +97,9 @@ Note: This implies you can't explicitly set a gauge to a negative number without first setting it to zero. -Sets ----- -StatsD supports counting unique occurences of events between flushes, +## Sets + +StatsD supports counting unique occurrences of events between flushes, using a Set to store all occuring events. uniques:765|s @@ -107,8 +107,8 @@ using a Set to store all occuring events. If the count at flush is 0 then you can opt to send no metric at all for this set, by setting `config.deleteSets`. -Multi-Metric Packets --------------------- +## Multi-Metric Packets + StatsD supports receiving multiple metrics in a single packet by separating them with a newline. @@ -126,5 +126,3 @@ scenarios: of all the hops in your route. *(These payload numbers take into account the maximum IP + UDP header sizes)* - - diff --git a/docs/namespacing.md b/docs/namespacing.md index cba54762..fb5956b9 100644 --- a/docs/namespacing.md +++ b/docs/namespacing.md @@ -1,5 +1,5 @@ -Metric namespacing -------------------- +# Metric namespacing + The metric namespacing in the Graphite backend is configurable with regard to the prefixes. Per default all stats are put under `stats` in Graphite, which makes it easier to consolidate them all under one schema. However it is diff --git a/docs/protocol.md b/docs/protocol.md new file mode 100644 index 00000000..87aad92e --- /dev/null +++ b/docs/protocol.md @@ -0,0 +1,3 @@ +# The StatsD Protocol + +Coming soon! Meanwhile, see https://github.com/b/statsd_spec diff --git a/docs/server.md b/docs/server.md index 7b85b05a..8516f112 100644 --- a/docs/server.md +++ b/docs/server.md @@ -1,5 +1,4 @@ -Supported Servers ------------------- +# Supported Servers StatsD supports pluggable server modules that listen for incoming metrics. diff --git a/docs/server_implementations.md b/docs/server_implementations.md new file mode 100644 index 00000000..b1e0de70 --- /dev/null +++ b/docs/server_implementations.md @@ -0,0 +1,19 @@ +# Server Implementations + +The following is a list of projects that re-implement StatsD, if the the main project isn't for you, perhaps one of these is. + +* [brubeck](https://github.com/github/brubeck) - Server in C +* [clj-statsd-svr](https://github.com/netmelody/clj-statsd-svr) — Clojure server +* [g3statsd](https://github.com/bytedance/g3/tree/master/g3statsd) - Server in Rust +* [gographite](https://github.com/amir/gographite) — Server in Go +* [gostatsd](https://github.com/atlassian/gostatsd) — Server in Go +* [netdata](https://github.com/firehol/netdata) - Embedded StatsD server in the netdata server, in C, with visualization +* [Net::Statsd::Server](https://github.com/cosimo/perl5-net-statsd-server) — Perl server, also available on [CPAN](https://metacpan.org/module/Net::Statsd::Server) +* [Py-Statsd](https://github.com/sivy/py-statsd) — Server and Client +* [Ruby-Statsdserver](https://github.com/fetep/ruby-statsdserver) — Ruby server +* [statsd-c](https://github.com/jbuchbinder/statsd-c) — Server in C +* [statsdaemon (bitly)](https://github.com/bitly/statsdaemon) — Server in Go +* [statsdaemon (vimeo)](https://github.com/vimeo/statsdaemon) — Server in Go +* [statsdcc](https://github.com/wayfair/statsdcc) - Server in C++ +* [statsdpy](https://github.com/pandemicsyn/statsdpy) — Python/eventlet Server +* [statsite](https://github.com/armon/statsite.git) — Server in C diff --git a/docs/server_interface.md b/docs/server_interface.md index b0b1e416..454bde32 100644 --- a/docs/server_interface.md +++ b/docs/server_interface.md @@ -1,7 +1,8 @@ -Server Interface ------------------ +# Server Interface Server modules are Node.js [modules][nodemods] that receive metrics for StatsD. +Server interfaces can be distributed and installed via systems such as NPM. + Each server module should export the following initialization function: * `start(config, callback)`: This method is invoked from StatsD to initialize @@ -16,3 +17,7 @@ Each server module should export the following initialization function: The server module should return `true` from start() to indicate success. A return of `false` indicates a failure to load the module (missing configuration?) and will cause StatsD to exit. + +# Available third-party interfaces + +* [http-interface](https://github.com/msiebuhr/statsd-http-interface) Accepts data over HTTP. diff --git a/exampleConfig.js b/exampleConfig.js index b0053a59..0f080c49 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -1,28 +1,42 @@ /* -Graphite Required Variables: +Graphite Required Variable: -(Leave these unset to avoid sending stats to Graphite. - Set debug flag and leave these unset to run in 'dry' debug mode - +(Leave this unset to avoid sending stats to Graphite. + Set debug flag and leave this unset to run in 'dry' debug mode - useful for testing statsd clients without a Graphite server.) graphiteHost: hostname or IP of Graphite server - graphitePort: port of Graphite server Optional Variables: + graphitePort: port for the graphite text collector [default: 2003] + graphitePicklePort: port for the graphite pickle collector [default: 2004] + graphiteProtocol: either 'text' or 'pickle' [default: 'text'] backends: an array of backends to load. Each backend must exist by name in the directory backends/. If not specified, - the default graphite backend will be loaded. + the default graphite backend will be loaded. * example for console and graphite: [ "./backends/console", "./backends/graphite" ] - server: the server to load. The server must exist by name in the directory + + servers: an array of server configurations. + If not specified, the server, address, + address_ipv6, and port top-level configuration + options are used to configure a single server for + backwards-compatibility + Each server configuration supports the following keys: + server: the server to load. The server must exist by name in the directory servers/. If not specified, the default udp server will be loaded. * example for tcp server: "./servers/tcp" + address: address to listen on [default: 0.0.0.0] + address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] + port: port to listen for messages on [default: 8125] + socket: (only for tcp servers) path to unix domain socket which will be used to receive + metrics [default: undefined] + socket_mod: (only for tcp servers) file mode which should be applied to unix domain socket, relevant + only if socket option is used [default: undefined] + debug: debug flag [default: false] - address: address to listen on [default: 0.0.0.0] - address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on [default: 8125] mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] mgmt_port: port to run the management TCP interface on [default: 8126] @@ -46,13 +60,21 @@ Optional Variables: log: location of log file for frequent keys [default: STDOUT] deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers as opposed to sending 0. For gauges, this unsets the gauge (instead of sending - the previous value). Can be individually overriden. [default: false] + the previous value). Can be individually overridden. [default: false] deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] + gaugesMaxTTL: number of flush cycles to wait before the gauge is marked as inactive, to use in combination with deleteGauges [default: 1] deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. + + calculatedTimerMetrics: List of timer metrics that will be sent. Default will send all metrics. + To filter on percents and top percents: append '_percent' to the metric name. + Example: calculatedTimerMetrics: ['count', 'median', 'upper_percent', 'histogram'] console: prettyprint: whether to prettyprint the console backend @@ -80,8 +102,8 @@ Optional Variables: e.g. [ { host: '10.10.10.10', port: 8125 }, { host: 'observer', port: 88125 } ] - repeaterProtocol: whether to use udp4 or udp6 for repeaters. - ["udp4" or "udp6", default: "udp4"] + repeaterProtocol: whether to use udp4, udp6, or tcp for repeaters. + ["udp4," "udp6", or "tcp" default: "udp4"] histogram: for timers, an array of mappings of strings (to match metrics) and corresponding ordered non-inclusive upper limits of bins. diff --git a/exampleProxyConfig.js b/exampleProxyConfig.js index fcc8e98e..6182df90 100644 --- a/exampleProxyConfig.js +++ b/exampleProxyConfig.js @@ -3,6 +3,8 @@ Required Variables: port: StatsD Cluster Proxy listening port [default: 8125] + mgmt_port: StatsD Cluster Proxy telnet management port [default: 8126] + mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] nodes: list of StatsD instances host: address of an instance of StatsD port: port that this instance is listening on @@ -10,11 +12,17 @@ Required Variables: Optional Variables: - udp_version: defines if the address is an IPv4 or IPv6 address ['udp4' or 'udp6', default: 'udp4'] host: address to listen on over UDP [default: 0.0.0.0] + address_ipv6: defines if the listen address is an IPv4 or IPv6 address [true or false, default: false] checkInterval: health status check interval [default: 10000] cacheSize: size of the cache to store for hashring key lookups [default: 10000] forkCount: number of child processes (cluster module), number or 'auto' for utilize all cpus [default:0] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + Note: This will still send to the backends via udp regardless of the + server type for the proxy + * example for tcp server: + "./servers/tcp" */ { @@ -23,9 +31,11 @@ nodes: [ {host: '127.0.0.1', port: 8129, adminport: 8130}, {host: '127.0.0.1', port: 8131, adminport: 8132} ], -udp_version: 'udp4', +server: './servers/udp', host: '0.0.0.0', port: 8125, +udp_version:'udp4', +mgmt_port: 8126, forkCount: 0, checkInterval: 1000, cacheSize: 10000 diff --git a/examples/StatsD.scala b/examples/StatsD.scala index 7830b92d..80d65682 100644 --- a/examples/StatsD.scala +++ b/examples/StatsD.scala @@ -42,7 +42,7 @@ import akka.actor._ * @param multiMetrics If true, multiple stats will be sent in a single UDP packet * @param packetBufferSize If multiMetrics is true, this is the max buffer size before sending the UDP packet */ -class StatsD(context: ActorContext, +class StatsD(context: ActorRefFactory, host: String, port: Int, multiMetrics: Boolean = true, @@ -229,4 +229,4 @@ private class StatsDActor(host: String, } } } -} \ No newline at end of file +} diff --git a/examples/StatsdClient.java b/examples/StatsdClient.java index 3b31ff67..c6202cdc 100644 --- a/examples/StatsdClient.java +++ b/examples/StatsdClient.java @@ -67,6 +67,10 @@ public StatsdClient(String host, int port) throws UnknownHostException, IOExcept public StatsdClient(InetAddress host, int port) throws IOException { _address = new InetSocketAddress(host, port); _channel = DatagramChannel.open(); + /* Put this in non-blocking mode so send does not block forever. */ + _channel.configureBlocking(false); + /* Increase the size of the output buffer so that the size is larger than our buffer size. */ + _channel.setOption(StandardSocketOptions.SO_SNDBUF, 4096); setBufferSize((short) 1500); } @@ -250,6 +254,7 @@ public synchronized boolean flush() { } } catch (IOException e) { + /* This would be a good place to close the channel down and recreate it. */ log.error( String.format("Could not send stat %s to host %s:%d", sendBuffer.toString(), _address.getHostName(), _address.getPort()), e); diff --git a/examples/StatsdClient.kt b/examples/StatsdClient.kt new file mode 100644 index 00000000..733de62b --- /dev/null +++ b/examples/StatsdClient.kt @@ -0,0 +1,206 @@ +/** + * StatsdClient.kt + * + * + * Example usage: + * + * val client = StatsdClient("statsd.example.com", 8125) + * // increment by 1 + * client.increment("foo.bar.baz") + * // increment by 10 + * client.increment("foo.bar.baz", magnitude = 10) + * // sample rate + * client.increment("foo.bar.baz", sampleRate = 0.1) + * // magnitude and sample rate + * client.increment("foo.bar.baz", magnitude = 10, sampleRate = 0.1) + * // increment multiple keys by 1 + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar") + * // increment multiple keys by 10 + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar", magnitude = 10) + * // multiple keys with a sample rate and magnitude + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar", magnitude = 10, sampleRate = 0.1) + */ + +import org.apache.log4j.Logger +import java.io.IOException +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.StandardSocketOptions +import java.net.UnknownHostException +import java.nio.ByteBuffer +import java.nio.channels.DatagramChannel +import java.util.Locale +import java.util.Random +import java.util.Timer +import java.util.TimerTask + +class StatsdClient @Throws(IOException::class) +constructor(host: InetAddress, port: Int) : TimerTask() { + private var sendBuffer: ByteBuffer? = null + private var flushTimer: Timer? = null + private var multiMetrics = false + + private val address: InetSocketAddress = InetSocketAddress(host, port) + private val channel: DatagramChannel = DatagramChannel.open() + + @Throws(UnknownHostException::class, IOException::class) + constructor(host: String, port: Int) : this(InetAddress.getByName(host), port) + + init { + // Put this in non-blocking mode so send does not block forever. + channel.configureBlocking(false) + // Increase the size of the output buffer so that the size is larger than our buffer size. + channel.setOption(StandardSocketOptions.SO_SNDBUF, 4096) + setBufferSize(1500) + } + + @Synchronized + fun setBufferSize(packetBufferSize: Short) { + if (sendBuffer != null) { + flush() + } + sendBuffer = ByteBuffer.allocate(packetBufferSize.toInt()) + } + + @Synchronized + fun enableMultiMetrics(enable: Boolean) { + multiMetrics = enable + } + + @Synchronized + fun startFlushTimer(period: Long = 2000): Boolean { + return if (flushTimer == null) { + flushTimer = Timer() + + // We pass this object in as the TimerTask (which calls run()) + flushTimer!!.schedule(this, period, period) + true + } else { + false + } + } + + @Synchronized + fun stopFlushTimer() { + if (flushTimer != null) { + flushTimer!!.cancel() + flushTimer = null + } + } + + // used by Timer, we're a Runnable TimerTask + override fun run() { + flush() + } + + fun timing(key: String, value: Int, sampleRate: Double = 1.0): Boolean { + return send(sampleRate, String.format(Locale.ENGLISH, "%s:%d|ms", key, value)) + } + + fun decrement(vararg keys: String, magnitude: Int = -1, sampleRate: Double = 1.0): Boolean { + val stats = keys.map { String.format(Locale.ENGLISH, "%s:%s|c", it, magnitude) }.toTypedArray() + + return send(sampleRate, *stats) + } + + fun increment(vararg keys: String, magnitude: Int = 1, sampleRate: Double = 1.0): Boolean { + val stats = keys.map { String.format(Locale.ENGLISH, "%s:%s|c", it, magnitude) }.toTypedArray() + + return send(sampleRate, *stats) + } + + fun gauge(key: String, magnitude: Double, sampleRate: Double = 1.0): Boolean { + val stat = String.format(Locale.ENGLISH, "%s:%s|g", key, magnitude) + + return send(sampleRate, stat) + } + + private fun send(sampleRate: Double, vararg stats: String): Boolean { + return if (sampleRate < 1.0) { + stats.any { + if (RNG.nextDouble() <= sampleRate) { + val stat = String.format(Locale.ENGLISH, "%s|@%f", it, sampleRate) + + doSend(stat) + } else { + false + } + } + } else { + stats.any { doSend(it) } + } + } + + @Synchronized + private fun doSend(stat: String): Boolean { + try { + val data = stat.toByteArray(charset("utf-8")) + + // If we're going to go past the threshold of the buffer then flush. + // the +1 is for the potential '\n' in multi_metrics below + if (sendBuffer!!.remaining() < data.size + 1) { + flush() + } + + // multiple metrics are separated by '\n' + if (sendBuffer!!.position() > 0) { + sendBuffer!!.put('\n'.toByte()) + } + + sendBuffer!!.put(data) + + if (!multiMetrics) { + flush() + } + + return true + } catch (e: IOException) { + log.error(String.format("Could not send stat %s to host %s:%d", sendBuffer!!.toString(), address.hostName, address.port), e) + + return false + } + } + + @Synchronized + fun flush(): Boolean { + try { + val sizeOfBuffer = sendBuffer!!.position() + + if (sizeOfBuffer <= 0) { + return false + } // empty buffer + + // send and reset the buffer + sendBuffer!!.flip() + + val nbSentBytes = channel.send(sendBuffer, address) + + sendBuffer!!.limit(sendBuffer!!.capacity()) + sendBuffer!!.rewind() + + return if (sizeOfBuffer == nbSentBytes) { + true + } else { + log.error(String.format( + "Could not send entirely stat %s to host %s:%d. Only sent %d bytes out of %d bytes", + sendBuffer!!.toString(), + address.hostName, + address.port, + nbSentBytes, + sizeOfBuffer + )) + + false + } + } catch (e: IOException) { + /* This would be a good place to close the channel down and recreate it. */ + log.error(String.format("Could not send stat %s to host %s:%d", sendBuffer!!.toString(), address.hostName, address.port), e) + return false + } + } + + companion object { + private val RNG = Random() + private val log = Logger.getLogger(StatsdClient::class.java.name) + } +} diff --git a/examples/go/statsd.go b/examples/go/statsd.go index 962e567a..ef34e9e5 100644 --- a/examples/go/statsd.go +++ b/examples/go/statsd.go @@ -96,6 +96,18 @@ func (client *StatsdClient) Increment(stat string) { client.UpdateStats(stats, 1, 1) } +// Increments one stat counter by value provided without sampling +// +// Usage: +// +// import "statsd" +// client := statsd.New('localhost', 8125) +// client.IncrementByValue('foo.bar', 5) +func (client *StatsdClient) IncrementByValue(stat string, val int) { + stats := []string{stat} + client.UpdateStats(stats, val, 1) +} + // Increments one stat counter with sampling // // Usage: diff --git a/examples/python_example.py b/examples/python_example.py index 28f20410..3b21a1d0 100644 --- a/examples/python_example.py +++ b/examples/python_example.py @@ -150,7 +150,7 @@ def send(_dict, addr): >>> StatsdClient.send({"example.send":"11|c"}, ("127.0.0.1", 8125)) """ # TODO(rbtz@): IPv6 support - # TODO(rbtz@): Creating socket on each send is a waste of recources + # TODO(rbtz@): Creating socket on each send is a waste of resources udp_sock = socket(AF_INET, SOCK_DGRAM) # TODO(rbtz@): Add batch support for item in _dict.items(): diff --git a/examples/ruby_example.rb b/examples/ruby_example.rb index ec52f098..cc436685 100644 --- a/examples/ruby_example.rb +++ b/examples/ruby_example.rb @@ -68,6 +68,7 @@ def self.send(data, sample_rate=1) value = data[stat] sock.send("#{stat}:#{value}", 0, @@config[:host], @@config[:port]) end + sock.close end end end diff --git a/examples/ruby_example2.rb b/examples/ruby_example2.rb index 0281a99f..a52f3183 100644 --- a/examples/ruby_example2.rb +++ b/examples/ruby_example2.rb @@ -77,6 +77,7 @@ def self.send(data, sample_rate=1) send_data = "%s:%s" % [stat, val] udp.send send_data, 0, host, port end + udp.close rescue => e puts e.message end diff --git a/examples/statsd-client.sh b/examples/statsd-client.sh index 9b236cc1..51b2bf98 100755 --- a/examples/statsd-client.sh +++ b/examples/statsd-client.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Very simple bash client to send metrics to a statsd server # Example with gauge: ./statsd-client.sh 'my_metric:100|g' diff --git a/lib/config.js b/lib/config.js index b61bf3b7..11ff71c3 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,22 +1,22 @@ /*jshint node:true, laxcomma:true */ -var fs = require('fs') - , util = require('util'); +const fs = require('fs') +const util = require('util'); -var Configurator = function (file) { +let Configurator = function (file) { - var self = this; - var config = {}; - var oldConfig = {}; + let self = this; + let config = {}; + let oldConfig = {}; this.updateConfig = function () { - util.log('reading config file: ' + file); + util.log('[' + process.pid + '] reading config file: ' + file); fs.readFile(file, function (err, data) { if (err) { throw err; } old_config = self.config; - self.config = eval('config = ' + fs.readFileSync(file)); + self.config = eval('config = ' + data); self.emit('configChanged', self.config); }); }; @@ -30,14 +30,13 @@ var Configurator = function (file) { }); }; -util.inherits(Configurator, process.EventEmitter); +util.inherits(Configurator, require('events').EventEmitter); exports.Configurator = Configurator; exports.configFile = function(file, callbackFunc) { - var config = new Configurator(file); + let config = new Configurator(file); config.on('configChanged', function() { callbackFunc(config.config, config.oldConfig); }); }; - diff --git a/lib/helpers.js b/lib/helpers.js index 98c6225a..cf64b575 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -11,6 +11,19 @@ function isNumber(str) { return Boolean(str && !isNaN(str)); } +function isInteger(x) { + return (typeof x === 'number') && (x % 1 === 0); +} + +function isValidSampleRate(str) { + let validSampleRate = false; + if(str.length > 1 && str[0] === '@') { + const numberStr = str.substring(1); + validSampleRate = isNumber(numberStr) && numberStr[0] != '-'; + } + return validSampleRate; +} + function is_valid_packet(fields) { // test for existing metrics type @@ -19,8 +32,8 @@ function is_valid_packet(fields) { } // filter out malformed sample rates - if (fields[2] !== undefined) { - if (fields[2].length <= 1 || fields[2][0] != '@' || !isNumber(fields[2].substring(1))) { + if(fields[2] !== undefined) { + if(!isValidSampleRate(fields[2])) { return false; } } @@ -40,6 +53,27 @@ function is_valid_packet(fields) { return true; } -}; +} exports.is_valid_packet = is_valid_packet; +exports.isInteger= isInteger; + +exports.writeConfig = function(config, stream) { + stream.write("\n"); + for (const prop in config) { + if (!config.hasOwnProperty(prop)) { + continue; + } + if (typeof config[prop] !== 'object') { + stream.write(prop + ": " + config[prop] + "\n"); + continue; + } + const subconfig = config[prop]; + for (const subprop in subconfig) { + if (!subconfig.hasOwnProperty(subprop)) { + continue; + } + stream.write(prop + " > " + subprop + ": " + subconfig[subprop] + "\n"); + } + } +}; diff --git a/lib/logger.js b/lib/logger.js index fea607b9..8867f825 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -var Logger = function (config) { +const Logger = function (config) { this.config = config; this.backend = this.config.backend || 'stdout'; this.level = this.config.level || "LOG_INFO"; @@ -8,7 +8,7 @@ var Logger = function (config) { this.util = require('util'); } else { if (this.backend == 'syslog') { - this.util = require('node-syslog'); + this.util = require('modern-syslog'); this.util.init(config.application || 'statsd', this.util.LOG_PID | this.util.LOG_ODELAY, this.util.LOG_LOCAL0); } else { throw "Logger: Should be 'stdout' or 'syslog'."; @@ -24,15 +24,18 @@ Logger.prototype = { } this.util.log(type + ": " + msg); } else { + let level; if (!type) { - type = this.level; - if (!this.util[type]) { - throw "Undefined log level: " + type; - } - } else if (type == 'debug') { - type = "LOG_DEBUG"; + level = this.level; + } else { + level = "LOG_" + type.toUpperCase(); } - this.util.log(this.util[type], msg); + + if (!this.util[level]) { + throw "Undefined log level: " + level; + } + + this.util.log(this.util[level], msg); } } }; diff --git a/lib/mgmt_console.js b/lib/mgmt_console.js index e9431c5f..bd99e274 100644 --- a/lib/mgmt_console.js +++ b/lib/mgmt_console.js @@ -14,7 +14,7 @@ exports.delete_stats = function(stats_type, cmdline, stream) { //for each metric requested on the command line - for (var index in cmdline) { + for (const index in cmdline) { //get a list of deletable metrics that match the request deletable = existing_stats(stats_type, cmdline[index]); @@ -25,7 +25,7 @@ exports.delete_stats = function(stats_type, cmdline, stream) { } //delete all requested metrics - for (var del_idx in deletable) { + for (const del_idx in deletable) { delete stats_type[deletable[del_idx]]; stream.write("deleted: " + deletable[del_idx] + "\n"); } @@ -53,9 +53,9 @@ function existing_stats(stats_type, bucket){ //special case: match a whole 'folder' (and subfolders) of stats if (bucket.slice(-2) == ".*") { - var folder = bucket.slice(0,-1); + const folder = bucket.slice(0,-1); - for (var name in stats_type) { + for (const name in stats_type) { //check if stat is in bucket, ie~ name starts with folder if (name.substring(0, folder.length) == folder) { matches.push(name); diff --git a/lib/mgmt_server.js b/lib/mgmt_server.js new file mode 100644 index 00000000..172826d4 --- /dev/null +++ b/lib/mgmt_server.js @@ -0,0 +1,24 @@ +/*jshint node:true, laxcomma:true */ + +const net = require('net'); + +exports.start = function(config, on_data_callback, on_error_callback) { + const server = net.createServer(function(stream) { + stream.setEncoding('ascii'); + + stream.on('data', function(data) { + const cmdline = data.trim().split(" "); + const cmd = cmdline.shift(); + + on_data_callback(cmd, cmdline, stream); + }); + + stream.on('error', function(err) { + on_error_callback(err, stream); + }); + }); + + server.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); + + return true; +}; diff --git a/lib/process_metrics.js b/lib/process_metrics.js index 8228f989..52c160d6 100644 --- a/lib/process_metrics.js +++ b/lib/process_metrics.js @@ -1,53 +1,53 @@ /*jshint node:true, laxcomma:true */ -var process_metrics = function (metrics, flushInterval, ts, flushCallback) { - var starttime = Date.now(); - var key; - var counter_rates = {}; - var timer_data = {}; - var statsd_metrics = {}; - var counters = metrics.counters; - var timers = metrics.timers; - var timer_counters = metrics.timer_counters; - var pctThreshold = metrics.pctThreshold; - var histogram = metrics.histogram; +const process_metrics = function (metrics, calculatedTimerMetrics, flushInterval, ts, flushCallback) { + const starttime = Date.now(); + let key; + let counter_rates = {}; + let timer_data = {}; + let statsd_metrics = {}; + const counters = metrics.counters; + const timers = metrics.timers; + const timer_counters = metrics.timer_counters; + const pctThreshold = metrics.pctThreshold; + const histogram = metrics.histogram; for (key in counters) { - var value = counters[key]; + const value = counters[key]; // calculate "per second" rate counter_rates[key] = value / (flushInterval / 1000); } for (key in timers) { - var current_timer_data = {}; + const current_timer_data = {}; if (timers[key].length > 0) { timer_data[key] = {}; - var values = timers[key].sort(function (a,b) { return a-b; }); - var count = values.length; - var min = values[0]; - var max = values[count - 1]; + const values = timers[key].sort(function (a,b) { return a-b; }); + const count = values.length; + const min = values[0]; + const max = values[count - 1]; - var cumulativeValues = [min]; - var cumulSumSquaresValues = [min * min]; - for (var i = 1; i < count; i++) { + const cumulativeValues = [min]; + const cumulSumSquaresValues = [min * min]; + for (let i = 1; i < count; i++) { cumulativeValues.push(values[i] + cumulativeValues[i-1]); cumulSumSquaresValues.push((values[i] * values[i]) + cumulSumSquaresValues[i - 1]); } - var sum = min; - var sumSquares = min * min; - var mean = min; - var thresholdBoundary = max; + let sum = min; + let sumSquares = min * min; + let mean = min; + let thresholdBoundary = max; - var key2; + let key2; for (key2 in pctThreshold) { - var pct = pctThreshold[key2]; - var numInThreshold = count; + const pct = pctThreshold[key2]; + let numInThreshold = count; if (count > 1) { numInThreshold = Math.round(Math.abs(pct) / 100 * count); @@ -68,7 +68,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { mean = sum / numInThreshold; } - var clean_pct = '' + pct; + let clean_pct = '' + pct; clean_pct = clean_pct.replace('.', '_').replace('-', 'top'); current_timer_data["count_" + clean_pct] = numInThreshold; current_timer_data["mean_" + clean_pct] = mean; @@ -82,15 +82,15 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { sumSquares = cumulSumSquaresValues[count-1]; mean = sum / count; - var sumOfDiffs = 0; - for (var i = 0; i < count; i++) { + let sumOfDiffs = 0; + for (let i = 0; i < count; i++) { sumOfDiffs += (values[i] - mean) * (values[i] - mean); } - var mid = Math.floor(count/2); - var median = (count % 2) ? values[mid] : (values[mid-1] + values[mid])/2; + const mid = Math.floor(count/2); + const median = (count % 2) ? values[mid] : (values[mid-1] + values[mid])/2; - var stddev = Math.sqrt(sumOfDiffs / count); + const stddev = Math.sqrt(sumOfDiffs / count); current_timer_data["std"] = stddev; current_timer_data["upper"] = max; current_timer_data["lower"] = min; @@ -104,7 +104,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { // note: values bigger than the upper limit of the last bin are ignored, by design conf = histogram || []; bins = []; - for (var i = 0; i < conf.length; i++) { + for (let i = 0; i < conf.length; i++) { if (key.indexOf(conf[i].metric) > -1) { bins = conf[i].bins; break; @@ -116,9 +116,9 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { // the outer loop iterates bins, the inner loop iterates timer values; // within each run of the inner loop we should only consider the timer value range that's within the scope of the current bin // so we leverage the fact that the values are already sorted to end up with only full 1 iteration of the entire values range - var i = 0; - for (var bin_i = 0; bin_i < bins.length; bin_i++) { - var freq = 0; + let i = 0; + for (let bin_i = 0; bin_i < bins.length; bin_i++) { + let freq = 0; for (; i < count && (bins[bin_i] == 'inf' || values[i] < bins[bin_i]); i++) { freq += 1; } @@ -132,7 +132,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { } - timer_data[key] = current_timer_data; + timer_data[key] = filter_timer_metrics(current_timer_data, calculatedTimerMetrics); } statsd_metrics["processing_time"] = (Date.now() - starttime); @@ -144,4 +144,20 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { flushCallback(metrics); }; +var filter_timer_metrics = function (currentTimerMetrics, calculatedTimerMetrics = []) { + if (!Array.isArray(calculatedTimerMetrics) || calculatedTimerMetrics.length == 0) { + return currentTimerMetrics; + } else { + return Object.keys(currentTimerMetrics) + .filter((key) => { + // Generalizes filtering percent metrics by cleaning key from _ to _percent + let cleaned_key = key.replace(/_(top)?\d+$/, "_percent") + return calculatedTimerMetrics.includes(cleaned_key); + }) + .reduce((obj, key) => { + obj[key] = currentTimerMetrics[key]; + return obj; + }, {}); + } +} exports.process_metrics = process_metrics; diff --git a/lib/process_mgmt.js b/lib/process_mgmt.js index b512e656..a0dcf7ba 100644 --- a/lib/process_mgmt.js +++ b/lib/process_mgmt.js @@ -1,6 +1,6 @@ -var util = require('util'); +const util = require('util'); -var conf; +let conf; exports.init = function(config) { conf = config; diff --git a/lib/set.js b/lib/set.js index b48a7a7f..58211ab6 100644 --- a/lib/set.js +++ b/lib/set.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -var Set = function() { +const Set = function() { this.store = {}; }; @@ -14,18 +14,21 @@ Set.prototype = { }, insert: function(value) { if (value) { - this.store[value] = value; + this.store[value] = true; } }, clear: function() { this.store = {}; }, values: function() { - var values = []; - for (var value in this.store) { + let values = []; + for (const value in this.store) { values.push(value); } return values; + }, + size: function() { + return Object.keys(this.store).length; } }; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..241ea4fe --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2313 @@ +{ + "name": "statsd", + "version": "0.8.6", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@babel/code-frame": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", + "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "dev": true, + "requires": { + "@babel/highlight": "^7.0.0" + } + }, + "@babel/generator": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.4.4.tgz", + "integrity": "sha512-53UOLK6TVNqKxf7RUh8NE851EHRxOOeVXKbK2bivdb+iziMyk03Sr4eaE9OELCbyZAAafAKPDwF2TPUES5QbxQ==", + "dev": true, + "requires": { + "@babel/types": "^7.4.4", + "jsesc": "^2.5.1", + "lodash": "^4.17.11", + "source-map": "^0.5.0", + "trim-right": "^1.0.1" + } + }, + "@babel/helper-function-name": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.1.0.tgz", + "integrity": "sha512-A95XEoCpb3TO+KZzJ4S/5uW5fNe26DjBGqf1o9ucyLyCmi1dXq/B3c8iaWTfBk3VvetUxl16e8tIrd5teOCfGw==", + "dev": true, + "requires": { + "@babel/helper-get-function-arity": "^7.0.0", + "@babel/template": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "@babel/helper-get-function-arity": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0.tgz", + "integrity": "sha512-r2DbJeg4svYvt3HOS74U4eWKsUAMRH01Z1ds1zx8KNTPtpTL5JAsdFv8BNyOpVqdFhHkkRDIg5B4AsxmkjAlmQ==", + "dev": true, + "requires": { + "@babel/types": "^7.0.0" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.4.4.tgz", + "integrity": "sha512-Ro/XkzLf3JFITkW6b+hNxzZ1n5OQ80NvIUdmHspih1XAhtN3vPTuUFT4eQnela+2MaZ5ulH+iyP513KJrxbN7Q==", + "dev": true, + "requires": { + "@babel/types": "^7.4.4" + } + }, + "@babel/highlight": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", + "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", + "dev": true, + "requires": { + "chalk": "^2.0.0", + "esutils": "^2.0.2", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz", + "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", + "dev": true + }, + "@babel/template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.4.4.tgz", + "integrity": "sha512-CiGzLN9KgAvgZsnivND7rkA+AeJ9JB0ciPOD4U59GKbQP2iQl+olF1l76kJOupqidozfZ32ghwBEJDhnk9MEcw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.4.4", + "@babel/types": "^7.4.4" + } + }, + "@babel/traverse": { + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.4.5.tgz", + "integrity": "sha512-Vc+qjynwkjRmIFGxy0KYoPj4FdVDxLej89kMHFsWScq999uX+pwcX4v9mWRjW0KcAYTPAuVQl2LKP1wEVLsp+A==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/generator": "^7.4.4", + "@babel/helper-function-name": "^7.1.0", + "@babel/helper-split-export-declaration": "^7.4.4", + "@babel/parser": "^7.4.5", + "@babel/types": "^7.4.4", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.11" + } + }, + "@babel/types": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.4.4.tgz", + "integrity": "sha512-dOllgYdnEFOebhkKCjzSVFqw/PmmB8pH6RGOWkY4GsboQNd47b1fBThBSwlHAq9alF9vc1M3+6oqR47R50L0tQ==", + "dev": true, + "requires": { + "esutils": "^2.0.2", + "lodash": "^4.17.11", + "to-fast-properties": "^2.0.0" + } + }, + "ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "append-transform": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz", + "integrity": "sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==", + "dev": true, + "requires": { + "default-require-extensions": "^2.0.0" + } + }, + "archy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", + "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=", + "dev": true + }, + "arg": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.0.tgz", + "integrity": "sha512-ZWc51jO3qegGkVh8Hwpv636EkbesNV5ZNQPCtRa+0qytRYPEs9IYT9qITY9buezqUH5uqyzlWLcufrzU2rffdg==", + "dev": true + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "bind-obj-methods": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/bind-obj-methods/-/bind-obj-methods-2.0.0.tgz", + "integrity": "sha512-3/qRXczDi2Cdbz6jE+W3IflJOutRVica8frpBn14de1mBOkzDo+6tY33kNhvkw54Kn3PzRRD2VnGbGPcTAk4sw==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "caching-transform": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz", + "integrity": "sha512-Mtgcv3lh3U0zRii/6qVgQODdPA4G3zhG+jtbCWj39RXuUFTMzH0vcdMtaJS1jPowd+It2Pqr6y3NJMQqOqCE2w==", + "dev": true, + "requires": { + "hasha": "^3.0.0", + "make-dir": "^2.0.0", + "package-hash": "^3.0.0", + "write-file-atomic": "^2.4.2" + } + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "capture-stack-trace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", + "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==", + "dev": true + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "clean-yaml-object": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/clean-yaml-object/-/clean-yaml-object-0.1.0.tgz", + "integrity": "sha1-Y/sRDcLOGoTcIfbZM0h20BCui2g=", + "dev": true + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "dev": true + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-1.3.1.tgz", + "integrity": "sha1-AkQ+AtuW9LMrZ0IlRRq7bpUQAA4=", + "optional": true, + "requires": { + "keypress": "0.1.x" + } + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "connection-parse": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", + "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=", + "optional": true + }, + "convert-source-map": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.6.0.tgz", + "integrity": "sha512-eFu7XigvxdZ1ETfbgPBohgyQ/Z++C0eEhTor0qRwBw9unw+L0/6V8wkSuGgzdThkiS5lSpdptOQPD8Ak40a+7A==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.1" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "coveralls": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.4.tgz", + "integrity": "sha512-eyqUWA/7RT0JagiL0tThVhjbIjoiEUyWCjtUJoOPcWoeofP5WK/jb2OJYoBFrR6DvplR+AxOyuBqk4JHkk5ykA==", + "dev": true, + "requires": { + "growl": "~> 1.10.0", + "js-yaml": "^3.11.0", + "lcov-parse": "^0.0.10", + "log-driver": "^1.2.7", + "minimist": "^1.2.0", + "request": "^2.86.0" + } + }, + "cp-file": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/cp-file/-/cp-file-6.2.0.tgz", + "integrity": "sha512-fmvV4caBnofhPe8kOcitBwSn2f39QLjnAnGq3gO9dfd75mUytzKNZB1hde6QHunW2Rt+OwuBOMc3i1tNElbszA==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "make-dir": "^2.0.0", + "nested-error-stacks": "^2.0.0", + "pify": "^4.0.1", + "safe-buffer": "^5.0.1" + } + }, + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "default-require-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", + "integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=", + "dev": true, + "requires": { + "strip-bom": "^3.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, + "diff": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz", + "integrity": "sha1-fyjS657nsVqX79ic5j3P2qPMur8=", + "dev": true + }, + "domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "ejs": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.6.2.tgz", + "integrity": "sha512-PcW2a0tyTuPHz3tWyYqtK6r1fZ3gp+3Sop8Ph+ZYN81Ob5rwmbHEzaqs10N3BEsaGTkh/ooniXK+WwszGlc2+Q==", + "dev": true + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "events-to-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/events-to-array/-/events-to-array-1.1.2.tgz", + "integrity": "sha1-LUH1Y+H+QA7Uli/hpNXGp1Od9/Y=", + "dev": true + }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dev": true, + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + } + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "foreground-child": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-1.5.6.tgz", + "integrity": "sha1-T9ca0t/elnibmApcCilZN8svXOk=", + "dev": true, + "requires": { + "cross-spawn": "^4", + "signal-exit": "^3.0.0" + } + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "fs-exists-cached": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz", + "integrity": "sha1-zyVVTKBQ3EmuZla0HeQiWJidy84=", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "function-loop": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/function-loop/-/function-loop-1.0.2.tgz", + "integrity": "sha512-Iw4MzMfS3udk/rqxTiDDCllhGwlOrsr50zViTOO/W6lS/9y6B1J0BD2VZzrnWUYBJsl3aeqjgR5v7bWWhZSYbA==", + "dev": true + }, + "generic-pool": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.2.0.tgz", + "integrity": "sha1-i0ZcGnWI6p3SuxM72gu2a/74pj4=" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "dev": true + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, + "handlebars": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", + "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", + "dev": true, + "requires": { + "neo-async": "^2.6.0", + "optimist": "^0.6.1", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "hasha": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-3.0.0.tgz", + "integrity": "sha1-UqMvq4Vp1BymmmH/GiFPjrfIvTk=", + "dev": true, + "requires": { + "is-stream": "^1.0.1" + } + }, + "hashring": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", + "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", + "optional": true, + "requires": { + "connection-parse": "0.0.x", + "simple-lru-cache": "0.0.x" + } + }, + "hosted-git-info": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", + "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", + "dev": true + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "invert-kv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", + "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==", + "dev": true + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true, + "optional": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, + "istanbul-lib-coverage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", + "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==", + "dev": true + }, + "istanbul-lib-hook": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz", + "integrity": "sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA==", + "dev": true, + "requires": { + "append-transform": "^1.0.0" + } + }, + "istanbul-lib-instrument": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz", + "integrity": "sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA==", + "dev": true, + "requires": { + "@babel/generator": "^7.4.0", + "@babel/parser": "^7.4.3", + "@babel/template": "^7.4.0", + "@babel/traverse": "^7.4.3", + "@babel/types": "^7.4.0", + "istanbul-lib-coverage": "^2.0.5", + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", + "dev": true + } + } + }, + "istanbul-lib-report": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz", + "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==", + "dev": true, + "requires": { + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "supports-color": "^6.1.0" + }, + "dependencies": { + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "istanbul-lib-source-maps": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.6.tgz", + "integrity": "sha512-R47KzMtDJH6X4/YW9XTx+jrLnZnscW4VpNN+1PViSYTejLVPWv7oov+Duf8YQSPyVRUvueQqz1TcsC6mooZTXw==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "rimraf": "^2.6.3", + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "istanbul-reports": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.6.tgz", + "integrity": "sha512-SKi4rnMyLBKe0Jy2uUdx28h8oG7ph2PPuQPvIAh31d+Ci+lSiEu4C+h3oBPuJ9+mPKhOyW0M8gY4U5NM1WLeXA==", + "dev": true, + "requires": { + "handlebars": "^4.1.2" + } + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "keypress": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/keypress/-/keypress-0.1.0.tgz", + "integrity": "sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo=", + "optional": true + }, + "lcid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", + "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==", + "dev": true, + "requires": { + "invert-kv": "^2.0.0" + } + }, + "lcov-parse": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz", + "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=", + "dev": true + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, + "lodash.flattendeep": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", + "dev": true + }, + "log-driver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", + "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==", + "dev": true + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "make-error": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", + "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==", + "dev": true + }, + "map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "requires": { + "p-defer": "^1.0.0" + } + }, + "mem": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz", + "integrity": "sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==", + "dev": true, + "requires": { + "map-age-cleaner": "^0.1.1", + "mimic-fn": "^2.0.0", + "p-is-promise": "^2.0.0" + } + }, + "merge-source-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/merge-source-map/-/merge-source-map-1.1.0.tgz", + "integrity": "sha512-Qkcp7P2ygktpMPh2mCQZaf3jhN6D3Z/qVZHSdWvQ+2Ef5HgRAPBO57A77+ENm0CPx2+1Ce/MYKi3ymqdfuqibw==", + "dev": true, + "requires": { + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==", + "dev": true + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "dev": true, + "requires": { + "mime-db": "1.40.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "dev": true + } + } + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + } + } + }, + "modern-syslog": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/modern-syslog/-/modern-syslog-1.2.0.tgz", + "integrity": "sha512-dmFE23qpyZJf8MOdzuNKliW4j1PCqxaRtSzyNnv6QDUWjf1z8T4ZoQ7Qf0t6It2ewNv9/XJZSJoUgwpq3D0X7A==", + "optional": true, + "requires": { + "nan": "^2.13.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "nan": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", + "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==", + "optional": true + }, + "neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + }, + "nested-error-stacks": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz", + "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", + "dev": true + }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "nodeunit": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/nodeunit/-/nodeunit-0.11.3.tgz", + "integrity": "sha512-gDNxrDWpx07BxYNO/jn1UrGI1vNhDQZrIFphbHMcTCDc5mrrqQBWfQMXPHJ5WSgbFwD1D6bv4HOsqtTrPG03AA==", + "dev": true, + "requires": { + "ejs": "^2.5.2", + "tap": "^12.0.1" + } + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "requires": { + "path-key": "^2.0.0" + } + }, + "nyc": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/nyc/-/nyc-14.1.1.tgz", + "integrity": "sha512-OI0vm6ZGUnoGZv/tLdZ2esSVzDwUC88SNs+6JoSOMVxA+gKMB8Tk7jBwgemLx4O40lhhvZCVw1C+OYLOBOPXWw==", + "dev": true, + "requires": { + "archy": "^1.0.0", + "caching-transform": "^3.0.2", + "convert-source-map": "^1.6.0", + "cp-file": "^6.2.0", + "find-cache-dir": "^2.1.0", + "find-up": "^3.0.0", + "foreground-child": "^1.5.6", + "glob": "^7.1.3", + "istanbul-lib-coverage": "^2.0.5", + "istanbul-lib-hook": "^2.0.7", + "istanbul-lib-instrument": "^3.3.0", + "istanbul-lib-report": "^2.0.8", + "istanbul-lib-source-maps": "^3.0.6", + "istanbul-reports": "^2.2.4", + "js-yaml": "^3.13.1", + "make-dir": "^2.1.0", + "merge-source-map": "^1.1.0", + "resolve-from": "^4.0.0", + "rimraf": "^2.6.3", + "signal-exit": "^3.0.2", + "spawn-wrap": "^1.4.2", + "test-exclude": "^5.2.3", + "uuid": "^3.3.2", + "yargs": "^13.2.2", + "yargs-parser": "^13.0.0" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "opener": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", + "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==", + "dev": true + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + }, + "dependencies": { + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", + "dev": true + } + } + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "dev": true + }, + "os-locale": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", + "dev": true, + "requires": { + "execa": "^1.0.0", + "lcid": "^2.0.0", + "mem": "^4.0.0" + } + }, + "own-or": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/own-or/-/own-or-1.0.0.tgz", + "integrity": "sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=", + "dev": true + }, + "own-or-env": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/own-or-env/-/own-or-env-1.0.1.tgz", + "integrity": "sha512-y8qULRbRAlL6x2+M0vIe7jJbJx/kmUTzYonRAa2ayesR2qWLswninkVyeJe4x3IEXhdgoNodzjQRKAoEs6Fmrw==", + "dev": true, + "requires": { + "own-or": "^1.0.0" + } + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true + }, + "p-is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", + "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", + "dev": true + }, + "p-limit": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", + "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "package-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-3.0.0.tgz", + "integrity": "sha512-lOtmukMDVvtkL84rJHI7dpTYq+0rli8N2wlnqUcBuDWCfVhRUfOmnR9SsoHFMLpACvEV60dX7rd0rFaYDZI+FA==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.15", + "hasha": "^3.0.0", + "lodash.flattendeep": "^4.4.0", + "release-zalgo": "^1.0.0" + } + }, + "parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "requires": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "optional": true + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "psl": { + "version": "1.1.33", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.33.tgz", + "integrity": "sha512-LTDP2uSrsc7XCb5lO7A8BI1qYxRe/8EqlRvMeEl6rsnYAqDOl8xHR+8lSAIVfrNaSAlTPTNOCgNjWcoUL3AZsw==", + "dev": true + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + }, + "read-pkg-up": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz", + "integrity": "sha512-6etQSH7nJGsK0RbG/2TeDzZFa8shjQ1um+SwQQ5cwKy0dhSXdOncEhb1CPpvQG4h7FyOV6EB6YlV0yJvZQNAkA==", + "dev": true, + "requires": { + "find-up": "^3.0.0", + "read-pkg": "^3.0.0" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "release-zalgo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", + "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=", + "dev": true, + "requires": { + "es6-error": "^4.0.1" + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "dev": true, + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "resolve": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", + "integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + }, + "sequence": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/sequence/-/sequence-2.2.1.tgz", + "integrity": "sha1-f1YXiV1ENRwKBH52RGdpBJChawM=", + "optional": true + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "simple-lru-cache": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", + "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=", + "optional": true + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true + }, + "source-map-support": { + "version": "0.5.12", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.12.tgz", + "integrity": "sha512-4h2Pbvyy15EE02G+JOZpUCmqWJuqrs+sEkzewTm++BPi7Hvn/HwcqLAcNxYAyI0x13CpPPn+kMjl+hplXMHITQ==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "spawn-wrap": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-1.4.2.tgz", + "integrity": "sha512-vMwR3OmmDhnxCVxM8M+xO/FtIp6Ju/mNaDfCMMW7FDcLRTPFWUswec4LXJHTJE2hwTI9O0YBfygu4DalFl7Ylg==", + "dev": true, + "requires": { + "foreground-child": "^1.5.6", + "mkdirp": "^0.5.0", + "os-homedir": "^1.0.1", + "rimraf": "^2.6.2", + "signal-exit": "^3.0.2", + "which": "^1.3.0" + } + }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.4.tgz", + "integrity": "sha512-7j8LYJLeY/Yb6ACbQ7F76qy5jHkp0U6jgBfJsk97bwWlVUnUWsAgpyaCvo17h0/RQGnQ036tVDomiwoI4pDkQA==", + "dev": true + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "stack-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz", + "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "optional": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "tap": { + "version": "12.7.0", + "resolved": "https://registry.npmjs.org/tap/-/tap-12.7.0.tgz", + "integrity": "sha512-SjglJmRv0pqrQQ7d5ZBEY8ZOqv3nYDBXEX51oyycOH7piuhn82JKT/yDNewwmOsodTD/RZL9MccA96EjDgK+Eg==", + "dev": true, + "requires": { + "bind-obj-methods": "^2.0.0", + "browser-process-hrtime": "^1.0.0", + "capture-stack-trace": "^1.0.0", + "clean-yaml-object": "^0.1.0", + "color-support": "^1.1.0", + "coveralls": "^3.0.2", + "domain-browser": "^1.2.0", + "esm": "^3.2.5", + "foreground-child": "^1.3.3", + "fs-exists-cached": "^1.0.0", + "function-loop": "^1.0.1", + "glob": "^7.1.3", + "isexe": "^2.0.0", + "js-yaml": "^3.13.1", + "minipass": "^2.3.5", + "mkdirp": "^0.5.1", + "nyc": "^14.0.0", + "opener": "^1.5.1", + "os-homedir": "^1.0.2", + "own-or": "^1.0.0", + "own-or-env": "^1.0.1", + "rimraf": "^2.6.3", + "signal-exit": "^3.0.0", + "source-map-support": "^0.5.10", + "stack-utils": "^1.0.2", + "tap-mocha-reporter": "^3.0.9", + "tap-parser": "^7.0.0", + "tmatch": "^4.0.0", + "trivial-deferred": "^1.0.1", + "ts-node": "^8.0.2", + "tsame": "^2.0.1", + "typescript": "^3.3.3", + "write-file-atomic": "^2.4.2", + "yapool": "^1.0.0" + } + }, + "tap-mocha-reporter": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/tap-mocha-reporter/-/tap-mocha-reporter-3.0.9.tgz", + "integrity": "sha512-VO07vhC9EG27EZdOe7bWBj1ldbK+DL9TnRadOgdQmiQOVZjFpUEQuuqO7+rNSO2kfmkq5hWeluYXDWNG/ytXTQ==", + "dev": true, + "requires": { + "color-support": "^1.1.0", + "debug": "^2.1.3", + "diff": "^1.3.2", + "escape-string-regexp": "^1.0.3", + "glob": "^7.0.5", + "js-yaml": "^3.3.1", + "readable-stream": "^2.1.5", + "tap-parser": "^5.1.0", + "unicode-length": "^1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "tap-parser": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/tap-parser/-/tap-parser-5.4.0.tgz", + "integrity": "sha512-BIsIaGqv7uTQgTW1KLTMNPSEQf4zDDPgYOBRdgOfuB+JFOLRBfEu6cLa/KvMvmqggu1FKXDfitjLwsq4827RvA==", + "dev": true, + "requires": { + "events-to-array": "^1.0.1", + "js-yaml": "^3.2.7", + "readable-stream": "^2" + } + } + } + }, + "tap-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tap-parser/-/tap-parser-7.0.0.tgz", + "integrity": "sha512-05G8/LrzqOOFvZhhAk32wsGiPZ1lfUrl+iV7+OkKgfofZxiceZWMHkKmow71YsyVQ8IvGBP2EjcIjE5gL4l5lA==", + "dev": true, + "requires": { + "events-to-array": "^1.0.1", + "js-yaml": "^3.2.7", + "minipass": "^2.2.0" + } + }, + "temp": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.4.0.tgz", + "integrity": "sha1-ZxrWPVe+D+nXKUZks/xABjZnimA=", + "dev": true + }, + "test-exclude": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", + "integrity": "sha512-M+oxtseCFO3EDtAaGH7iiej3CBkzXqFMbzqYAACdzKui4eZA+pq3tZEwChvOdNfa7xxy8BfbmgJSIr43cC/+2g==", + "dev": true, + "requires": { + "glob": "^7.1.3", + "minimatch": "^3.0.4", + "read-pkg-up": "^4.0.0", + "require-main-filename": "^2.0.0" + } + }, + "tmatch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tmatch/-/tmatch-4.0.0.tgz", + "integrity": "sha512-Ynn2Gsp+oCvYScQXeV+cCs7citRDilq0qDXA6tuvFwDgiYyyaq7D5vKUlAPezzZR5NDobc/QMeN6e5guOYmvxg==", + "dev": true + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "dev": true, + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, + "trim-right": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", + "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", + "dev": true + }, + "trivial-deferred": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trivial-deferred/-/trivial-deferred-1.0.1.tgz", + "integrity": "sha1-N21NKdlR1jaKb3oK6FwvTV4GWPM=", + "dev": true + }, + "ts-node": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.3.0.tgz", + "integrity": "sha512-dyNS/RqyVTDcmNM4NIBAeDMpsAdaQ+ojdf0GOLqE6nwJOgzEkdRNzJywhDfwnuvB10oa6NLVG1rUJQCpRN7qoQ==", + "dev": true, + "requires": { + "arg": "^4.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.6", + "yn": "^3.0.0" + }, + "dependencies": { + "diff": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", + "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==", + "dev": true + } + } + }, + "tsame": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tsame/-/tsame-2.0.1.tgz", + "integrity": "sha512-jxyxgKVKa4Bh5dPcO42TJL22lIvfd9LOVJwdovKOnJa4TLLrHxquK+DlGm4rkGmrcur+GRx+x4oW00O2pY/fFw==", + "dev": true + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, + "typescript": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.2.tgz", + "integrity": "sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==", + "dev": true + }, + "uglify-js": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", + "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "dev": true, + "optional": true, + "requires": { + "commander": "~2.20.0", + "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "dev": true, + "optional": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "optional": true + } + } + }, + "underscore": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz", + "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ=", + "dev": true + }, + "unicode-length": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/unicode-length/-/unicode-length-1.0.3.tgz", + "integrity": "sha1-Wtp6f+1RhBpBijKM8UlHisg1irs=", + "dev": true, + "requires": { + "punycode": "^1.3.2", + "strip-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true, + "optional": true + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "winser": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/winser/-/winser-0.1.6.tgz", + "integrity": "sha1-CGY9wyh4oSu84WLYQNpQl7SEZsk=", + "optional": true, + "requires": { + "commander": "1.3.1", + "sequence": "2.2.1" + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + }, + "yapool": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yapool/-/yapool-1.0.0.tgz", + "integrity": "sha1-9pPymjFbUNmp2iZGp6ZkXJaYW2o=", + "dev": true + }, + "yargs": { + "version": "13.2.4", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.4.tgz", + "integrity": "sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "os-locale": "^3.1.0", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.0" + } + }, + "yargs-parser": { + "version": "13.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", + "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "yn": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.0.tgz", + "integrity": "sha512-kKfnnYkbTfrAdd0xICNFw7Atm8nKpLcLv9AZGEt+kczL/WQVai4e2V6ZN8U/O+iI6WrNuJjNNOyu4zfhl9D3Hg==", + "dev": true + } + } +} diff --git a/package.json b/package.json index 62482098..30623f08 100644 --- a/package.json +++ b/package.json @@ -1,32 +1,48 @@ { "name": "statsd", - "description": "A simple, lightweight network daemon to collect metrics over UDP", - "author": "Etsy", - "scripts": { - "test": "./run_tests.sh", - "start": "node stats.js config.js", - "install-windows-service": "node_modules\\.bin\\winser -i", - "uninstall-windows-service": "node_modules\\.bin\\winser -r" + "version": "0.10.2", + "description": "Network daemon for the collection and aggregation of realtime application metrics", + "author": { + "name": "Etsy", + "url": "https://codeascraft.com" }, + "license": "MIT", + "homepage": "https://github.com/statsd/statsd", + "bugs": "https://github.com/statsd/statsd/issues", + "keywords": [ + "statsd", + "etsy", + "metric", + "aggregation", + "realtime" + ], "repository": { "type": "git", - "url": "https://github.com/etsy/statsd.git" + "url": "https://github.com/statsd/statsd.git" + }, + "engines": { + "node": ">=8" }, - "version": "0.7.2", "dependencies": { + "generic-pool": "2.2.0" }, "devDependencies": { - "nodeunit": "0.7.x", + "nodeunit": "^0.11.3", "underscore": "1.4.x", "temp": "0.4.x" }, "optionalDependencies": { - "node-syslog":"1.1.7", - "hashring":"1.0.1", + "modern-syslog": "1.2.0", + "hashring": "3.2.0", "winser": "=0.1.6" }, - "engines": { - "node" : ">=0.8" + "bin": { + "statsd": "./bin/statsd" }, - "bin": { "statsd": "./bin/statsd" } + "scripts": { + "test": "node run_tests.js", + "start": "node stats.js config.js", + "install-windows-service": "node_modules\\.bin\\winser -i", + "uninstall-windows-service": "node_modules\\.bin\\winser -r" + } } diff --git a/packager/Procfile b/packager/Procfile new file mode 100644 index 00000000..64e1781e --- /dev/null +++ b/packager/Procfile @@ -0,0 +1 @@ +web: node stats.js ${STATSD_CONFIG:="config.js"} diff --git a/packager/postinst b/packager/postinst new file mode 100755 index 00000000..3ba76a2d --- /dev/null +++ b/packager/postinst @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +APP_NAME="statsd" +CLI="$APP_NAME" +APP_USER=$(${CLI} config:get APP_USER) +APP_GROUP=$(${CLI} config:get APP_GROUP) +APP_CONFIG="/etc/${APP_NAME}/config.js" + +[ -f "$APP_CONFIG" ] || cp /opt/${APP_NAME}/exampleConfig.js $APP_CONFIG +chown $APP_USER.$APP_GROUP $APP_CONFIG +ln -f -s $APP_CONFIG /opt/$APP_NAME/config.js +chmod 0640 $APP_CONFIG + +${CLI} scale web=0 || true +${CLI} scale web=1 || true diff --git a/proxy.js b/proxy.js index 7053f6b3..b78c94ac 100644 --- a/proxy.js +++ b/proxy.js @@ -1,21 +1,28 @@ +/*jshint node:true, laxcomma:true */ + var dgram = require('dgram') , net = require('net') , events = require('events') , logger = require('./lib/logger') , hashring = require('hashring') , cluster = require('cluster') + , helpers = require('./lib/helpers') + , mgmt_server = require('./lib/mgmt_server') , configlib = require('./lib/config'); var packet = new events.EventEmitter(); -var node_status = []; +var startup_time = Math.round(new Date().getTime() / 1000); +var node_status = {}; +var workers = []; // Keep track of all forked childs var node_ring = {}; +var servers_loaded; var config; var l; // logger configlib.configFile(process.argv[2], function (conf, oldConfig) { config = conf; - var udp_version = config.udp_version - , nodes = config.nodes; + var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + var nodes = config.nodes; l = new logger.Logger(config.log || {}); var forkCount = config.forkCount; @@ -26,22 +33,56 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { var logPrefix = "[" + process.pid + "] "; var log = function(msg, type) { l.log(logPrefix + msg, type); - } - + }; - if (forkCount > 1 && cluster.isMaster) { - logPrefix += "[master] "; - log("forking " + forkCount + " childs"); + var healthStatus = configlib.healthStatus || 'up'; + var healthCheckInterval = config.checkInterval || 10000; - for (var i = 0; i < forkCount; i++) { - cluster.fork(); + var broadcastMsg = function(msg) { + for (var i = 0; i < workers.length; i++) { + workers[i].send(msg); } + }; - cluster.on('exit', function(worker, code, signal) { - log('worker ' + worker.process.pid + ' died with exit code:' + code + " restarting", 'ERROR'); - cluster.fork(); - }); - return; + if (forkCount > 1) { + if (cluster.isMaster) { + var worker; + logPrefix += "[master] "; + log("forking " + forkCount + " childs", "INFO"); + + for (var i = 0; i < forkCount; i++) { + worker = cluster.fork(); + worker.on('message', broadcastMsg); + } + + cluster.on('online', function(worker) { + log('worker ' + worker.process.pid + ' is online', 'INFO'); + workers.push(worker); + }); + + cluster.on('exit', function(worker, code, signal) { + log('worker ' + worker.process.pid + ' died with exit code:' + code + " restarting", 'ERROR'); + + // Remove died worker from the array + for (var i = 0; i < workers.length; i++) { + if (workers[i].process.pid == worker.process.pid) { + workers.splice(i, 1); + } + } + + worker = cluster.fork(); + worker.on('message', broadcastMsg); + }); + + return; + + } else { + process.on('message', function(msg) { + if (msg.healthStatus) { + healthStatus = msg.healthStatus; + } + }); + } } //load the node_ring object with the available nodes and a weight of 100 @@ -57,62 +98,126 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { 'replicas': 0 }); - // Do an initial rount of health checks prior to starting up the server - doHealthChecks(); - - - // Setup the udp listener - var server = dgram.createSocket(udp_version, function (msg, rinfo) { - // Convert the raw packet to a string (defaults to UTF8 encoding) - var packet_data = msg.toString(); - // If the packet contains a \n then it contains multiple metrics - if (packet_data.indexOf("\n") > -1) { - var metrics; - metrics = packet_data.split("\n"); - // Loop through the metrics and split on : to get mertric name for hashing - for (var midx in metrics) { - var current_metric = metrics[midx]; - var bits = current_metric.split(':'); - var key = bits.shift(); + if (!servers_loaded) { + // Do an initial rount of health checks prior to starting up the server + doHealthChecks(); + + // Setup the udp listener + var server_config = config.server || './servers/udp'; + var servermod = require(server_config); + var server = servermod.start(config, function (msg, rinfo) { + // Convert the raw packet to a string (defaults to UTF8 encoding) + var packet_data = msg.toString(); + var current_metric + , bits + , key; + // If the packet contains a \n then it contains multiple metrics + if (packet_data.indexOf("\n") > -1) { + var metrics; + metrics = packet_data.split("\n"); + // Loop through the metrics and split on : to get metric name for hashing + for (var midx in metrics) { + current_metric = metrics[midx]; + bits = current_metric.split(':'); + key = bits.shift(); + if (current_metric !== '') { + var new_msg = new Buffer(current_metric); + packet.emit('send', key, new_msg); + } + } + + } else { + // metrics needs to be an array to fake it for single metric packets + current_metric = packet_data; + bits = current_metric.split(':'); + key = bits.shift(); if (current_metric !== '') { - var new_msg = new Buffer(current_metric); - packet.emit('send', key, new_msg); + packet.emit('send', key, msg); } } + }); + var client = dgram.createSocket(udp_version); + // Listen for the send message, and process the metric key and msg + packet.on('send', function(key, msg) { + // retrieves the destination for this key + var statsd_host = ring.get(key); - } else { - // metrics needs to be an array to fake it for single metric packets - var current_metric = packet_data; - var bits = current_metric.split(':'); - var key = bits.shift(); - if (current_metric !== '') { - packet.emit('send', key, msg); + // break the retrieved host to pass to the send function + if (statsd_host === undefined) { + log('Warning: No backend statsd nodes available!', 'WARNING'); + } else { + var host_config = statsd_host.split(':'); + + // Send the mesg to the backend + client.send(msg, 0, msg.length, host_config[1], host_config[0]); } - } - }); + }); - var client = dgram.createSocket(udp_version); - // Listen for the send message, and process the metric key and msg - packet.on('send', function(key, msg) { - // retreives the destination for this key - var statsd_host = ring.get(key); + mgmt_server.start( + config, + function(cmd, parameters, stream) { + switch(cmd) { + case "help": + stream.write("Commands: config, health, status, quit\n\n"); + break; - // break the retreived host to pass to the send function - if (statsd_host === undefined) { - log('Warning: No backend statsd nodes available!'); - } else { - var host_config = statsd_host.split(':'); + case "config": + helpers.writeConfig(config, stream); + break; - // Send the mesg to the backend - client.send(msg, 0, msg.length, host_config[1], host_config[0]); - } - }); + case "health": + if (parameters.length > 0) { + var cmdaction = parameters[0].toLowerCase(); + if (cmdaction === 'up') { + healthStatus = 'up'; + if (forkCount > 0) { + // Notify the other forks + process.send({ healthStatus: healthStatus }); + } + } else if (cmdaction === 'down') { + healthStatus = 'down'; + if (forkCount > 0) { + // Notify the other forks + process.send({ healthStatus: healthStatus }); + } + } + } + stream.write("health: " + healthStatus + "\n"); + break; + + case "status": + var now = Math.round(new Date().getTime() / 1000); + var uptime = now - startup_time; - // Bind the listening udp server to the configured port and host - server.bind(config.port, config.host || undefined); + stream.write("uptime: " + uptime + "\n"); - // Set the interval for healthchecks - setInterval(doHealthChecks, config.checkInterval || 10000); + stream.write("nodes: "); + ring.servers.forEach(function(server, index, array) { + stream.write(server.string + " "); + }); + stream.write("\n"); + break; + + case "quit": + stream.end(); + break; + + default: + stream.write("ERROR\n"); + break; + } + }, + function(err, stream) { + log("MGMT: Caught " + err + ", Moving on", "WARNING"); + } + ); + + servers_loaded = true; + log("server is up", "INFO"); + + // Set the interval for healthchecks + setInterval(doHealthChecks, healthCheckInterval); + } // Perform health check on all nodes function doHealthChecks() { @@ -121,52 +226,77 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { }); } + function markNodeAsHealthy(node_id) { + if (node_status[node_id] !== undefined) { + if (node_status[node_id] > 0) { + var new_server = {}; + new_server[node_id] = 100; + log('Adding node ' + node_id + ' to the ring.', 'WARNING'); + ring.add(new_server); + } + } + + node_status[node_id] = 0; + } + + function markNodeAsUnhealthy(node_id) { + if (node_status[node_id] === undefined) { + node_status[node_id] = 1; + } else { + node_status[node_id]++; + } + if (node_status[node_id] < 2) { + log('Removing node ' + node_id + ' from the ring.', 'WARNING'); + ring.remove(node_id); + } + } + // Perform health check on node function healthcheck(node) { + var ended = false; var node_id = node.host + ':' + node.port; - var client = net.connect({port: node.adminport, host: node.host}, - function() { - client.write('health\r\n'); + var client = net.connect( + {port: node.adminport, host: node.host}, + function onConnect() { + if (!ended) { + client.write('health\r\n'); + } + } + ); + + client.setTimeout(healthCheckInterval, function() { + client.end(); + markNodeAsUnhealthy(node_id); + client.removeAllListeners('data'); + ended = true; }); + client.on('data', function(data) { + if (ended) { + return; + } + var health_status = data.toString(); client.end(); + ended = true; + if (health_status.indexOf('up') < 0) { - if (node_status[node_id] === undefined) { - node_status[node_id] = 1; - } else { - node_status[node_id]++; - } - if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.'); - ring.remove(node_id); - } + markNodeAsUnhealthy(node_id); } else { - if (node_status[node_id] !== undefined) { - if (node_status[node_id] > 0) { - var new_server = {}; - new_server[node_id] = 100; - log('Adding node ' + node_id + ' to the ring.'); - ring.add(new_server); - } - } - node_status[node_id] = 0; + markNodeAsHealthy(node_id); } }); + client.on('error', function(e) { - if (e.code == 'ECONNREFUSED') { - if (node_status[node_id] === undefined) { - node_status[node_id] = 1; - } else { - node_status[node_id]++; - } - if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.'); - ring.remove(node_id); - } - } else { - log('Error during healthcheck on node ' + node_id + ' with ' + e.code); + if (ended) { + return; + } + + if (e.code !== 'ECONNREFUSED' && e.code !== 'EHOSTUNREACH' && e.code !== 'ECONNRESET') { + log('Error during healthcheck on node ' + node_id + ' with ' + e.code, 'ERROR'); } + + markNodeAsUnhealthy(node_id); }); } diff --git a/run_tests.sh b/run_tests.js similarity index 80% rename from run_tests.sh rename to run_tests.js index 333fa7d7..75be28cd 100755 --- a/run_tests.sh +++ b/run_tests.js @@ -1,6 +1,7 @@ #!/usr/bin/env node +let reporter; try { - var reporter = require('nodeunit').reporters.default; + reporter = require('nodeunit').reporters.default; } catch(e) { console.log("Cannot find nodeunit module."); diff --git a/servers/tcp.js b/servers/tcp.js index 68b707cd..49875446 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -1,28 +1,38 @@ -var net = require('net'); +const net = require('net'); +const fs = require('fs'); function rinfo(tcpstream, data) { this.address = tcpstream.remoteAddress; this.port = tcpstream.remotePort; - this.family = tcpstream.address().family; + this.family = tcpstream.address() ? tcpstream.address().family : 'IPv4'; this.size = data.length; } -exports.start = function(config, callback){ - var server = net.createServer(function(stream) { +exports.start = function(config, callback) { + const server = net.createServer(function(stream) { stream.setEncoding('ascii'); - var buffer = ''; + let buffer = ''; stream.on('data', function(data) { buffer += data; - var offset = buffer.lastIndexOf("\n"); + const offset = buffer.lastIndexOf("\n"); if (offset > -1) { - var packet = buffer.slice(0, offset + 1); + const packet = buffer.slice(0, offset + 1); buffer = buffer.slice(offset + 1); callback(packet, new rinfo(stream, packet)); } }); }); - server.listen(config.port || 8125, config.address || undefined); + server.on('listening', function() { + config.socket && config.socket_mod && fs.chmod(config.socket, config.socket_mod); + }); + + process.on('exit', function() { + config.socket && fs.unlinkSync(config.socket); + }); + + server.listen(config.socket || config.port || 8125, config.address || undefined); + this.server = server; return true; }; diff --git a/servers/udp.js b/servers/udp.js index a0a3072b..bca72608 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,8 +1,11 @@ -var dgram = require('dgram'); +const dgram = require('dgram'); + +exports.start = function(config, callback) { + const udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + const server = dgram.createSocket(udp_version, callback); -exports.start = function(config, callback){ - var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; - var server = dgram.createSocket(udp_version, callback); server.bind(config.port || 8125, config.address || undefined); + this.server = server; + return true; }; diff --git a/stats.js b/stats.js index a1a40cdb..75f57412 100644 --- a/stats.js +++ b/stats.js @@ -1,46 +1,47 @@ /*jshint node:true, laxcomma:true */ -var util = require('util') - , net = require('net') - , config = require('./lib/config') - , helpers = require('./lib/helpers') - , fs = require('fs') - , events = require('events') - , logger = require('./lib/logger') - , set = require('./lib/set') - , pm = require('./lib/process_metrics') - , process_mgmt = require('./lib/process_mgmt') - , mgmt = require('./lib/mgmt_console'); - +const util = require('util'); +const config = require('./lib/config'); +const helpers = require('./lib/helpers'); +const fs = require('fs'); +const events = require('events'); +const logger = require('./lib/logger'); +const set = require('./lib/set'); +const pm = require('./lib/process_metrics'); +const process_mgmt = require('./lib/process_mgmt'); +const mgmt_server = require('./lib/mgmt_server'); +const mgmt = require('./lib/mgmt_console'); // initialize data structures with defaults for statsd stats -var keyCounter = {}; -var counters = {}; -var timers = {}; -var timer_counters = {}; -var gauges = {}; -var sets = {}; -var counter_rates = {}; -var timer_data = {}; -var pctThreshold = null; -var flushInterval, keyFlushInt, serverLoaded, mgmtServer; -var startup_time = Math.round(new Date().getTime() / 1000); -var backendEvents = new events.EventEmitter(); -var healthStatus = config.healthStatus || 'up'; -var old_timestamp = 0; -var timestamp_lag_namespace; +let keyCounter = {}; +let counters = {}; +let timers = {}; +let timer_counters = {}; +let gauges = {}; +let gaugesTTL = {}; +let sets = {}; +let counter_rates = {}; +let timer_data = {}; +let pctThreshold = null; +let flushInterval, keyFlushInt, serversLoaded, mgmtServer; +let startup_time = Math.round(new Date().getTime() / 1000); +let backendEvents = new events.EventEmitter(); +let healthStatus = config.healthStatus || 'up'; +let old_timestamp = 0; +let timestamp_lag_namespace; +let keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { - var backendmod = require(name); + const backendmod = require(name); if (config.debug) { l.log("Loading backend: " + name, 'DEBUG'); } - var ret = backendmod.init(startup_time, config, backendEvents, l); + const ret = backendmod.init(startup_time, config, backendEvents, l); if (!ret) { - l.log("Failed to load backend: " + name); + l.log("Failed to load backend: " + name, "ERROR"); process.exit(1); } } @@ -51,31 +52,31 @@ function loadBackend(config, name) { // rinfo: contains remote address information and message length // (attributes are .address, .port, .family, .size - you're welcome) function startServer(config, name, callback) { - var servermod = require(name); + const servermod = require(name); if (config.debug) { l.log("Loading server: " + name, 'DEBUG'); } - var ret = servermod.start(config, callback); + const ret = servermod.start(config, callback); if (!ret) { - l.log("Failed to load server: " + name); + l.log("Failed to load server: " + name, "ERROR"); process.exit(1); } } // global for conf -var conf; +let conf; // Flush metrics to each backend. function flushMetrics() { - var time_stamp = Math.round(new Date().getTime() / 1000); + const time_stamp = Math.round(new Date().getTime() / 1000); if (old_timestamp > 0) { gauges[timestamp_lag_namespace] = (time_stamp - old_timestamp - (Number(conf.flushInterval)/1000)); } old_timestamp = time_stamp; - var metrics_hash = { + const metrics_hash = { counters: counters, gauges: gauges, timers: timers, @@ -89,20 +90,10 @@ function flushMetrics() { // After all listeners, reset the stats backendEvents.once('flush', function clear_metrics(ts, metrics) { - // TODO: a lot of this should be moved up into an init/constructor so we don't have to do it every - // single flushInterval.... - // allows us to flag all of these on with a single config but still override them individually - conf.deleteIdleStats = conf.deleteIdleStats !== undefined ? conf.deleteIdleStats : false; - if (conf.deleteIdleStats) { - conf.deleteCounters = conf.deleteCounters !== undefined ? conf.deleteCounters : true; - conf.deleteTimers = conf.deleteTimers !== undefined ? conf.deleteTimers : true; - conf.deleteSets = conf.deleteSets !== undefined ? conf.deleteSets : true; - conf.deleteGauges = conf.deleteGauges !== undefined ? conf.deleteGauges : true; - } // Clear the counters conf.deleteCounters = conf.deleteCounters || false; - for (var counter_key in metrics.counters) { + for (const counter_key in metrics.counters) { if (conf.deleteCounters) { if ((counter_key.indexOf("packets_received") != -1) || (counter_key.indexOf("metrics_received") != -1) || @@ -118,7 +109,7 @@ function flushMetrics() { // Clear the timers conf.deleteTimers = conf.deleteTimers || false; - for (var timer_key in metrics.timers) { + for (const timer_key in metrics.timers) { if (conf.deleteTimers) { delete(metrics.timers[timer_key]); delete(metrics.timer_counters[timer_key]); @@ -130,7 +121,7 @@ function flushMetrics() { // Clear the sets conf.deleteSets = conf.deleteSets || false; - for (var set_key in metrics.sets) { + for (const set_key in metrics.sets) { if (conf.deleteSets) { delete(metrics.sets[set_key]); } else { @@ -138,30 +129,59 @@ function flushMetrics() { } } - // normally gauges are not reset. so if we don't delete them, continue to persist previous value + // Normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { - for (var gauge_key in metrics.gauges) { - delete(metrics.gauges[gauge_key]); + for (const gauge_key in gaugesTTL) { + gaugesTTL[gauge_key]--; + + // if the gauge has been idle for more than the allowed TTL cycles delete it + if (gaugesTTL[gauge_key] < 1) { + delete(metrics.gauges[gauge_key]); + delete(gaugesTTL[gauge_key]); + } } } }); - pm.process_metrics(metrics_hash, flushInterval, time_stamp, function emitFlush(metrics) { + pm.process_metrics(metrics_hash, conf.calculatedTimerMetrics, flushInterval, time_stamp, function emitFlush(metrics) { backendEvents.emit('flush', time_stamp, metrics); }); + // Performing this setTimeout at the end of this method rather than the beginning + // helps ensure we adapt to negative clock skew by letting the method's latency + // introduce a short delay that should more than compensate. + setTimeout(flushMetrics, getFlushTimeout(flushInterval)); } -var stats = { +const stats = { messages: { last_msg_seen: startup_time, bad_lines_seen: 0 } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.;=]/g, ''); + } else { + return key; + } +} + +function getFlushTimeout(interval) { + const now = new Date().getTime() + const deltaTime = now - startup_time * 1000; + const timeoutAttempt = Math.round(deltaTime / interval) + 1; + const fixedTimeout = (startup_time * 1000 + timeoutAttempt * interval) - now; + + return fixedTimeout; +} + // Global for the logger -var l; +let l; config.configFile(process.argv[2], function (config) { conf = config; @@ -170,8 +190,28 @@ config.configFile(process.argv[2], function (config) { l = new logger.Logger(config.log || {}); + // force conf.gaugesMaxTTL to 1 if it not a positive integer > 1 + if (helpers.isInteger(conf.gaugesMaxTTL) && conf.gaugesMaxTTL > 1) { + conf.gaugesMaxTTL = conf.gaugesMaxTTL; + } else { + conf.gaugesMaxTTL = 1; + } + + // allows us to flag all of these on with a single config but still override them individually + conf.deleteIdleStats = conf.deleteIdleStats !== undefined ? conf.deleteIdleStats : false; + if (conf.deleteIdleStats) { + conf.deleteCounters = conf.deleteCounters !== undefined ? conf.deleteCounters : true; + conf.deleteTimers = conf.deleteTimers !== undefined ? conf.deleteTimers : true; + conf.deleteSets = conf.deleteSets !== undefined ? conf.deleteSets : true; + conf.deleteGauges = conf.deleteGauges !== undefined ? conf.deleteGauges : true; + } + + // if gauges are not being deleted, clear gaugesTTL counters to save memory + if (! conf.deleteGauges) { + gaugesTTL = {} + } // setup config for stats prefix - var prefixStats = config.prefixStats; + let prefixStats = config.prefixStats; prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; //setup the names for the stats stored in counters{} bad_lines_seen = prefixStats + ".bad_lines_seen"; @@ -184,24 +224,26 @@ config.configFile(process.argv[2], function (config) { counters[packets_received] = 0; counters[metrics_received] = 0; - if (!serverLoaded) { + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } + if (!serversLoaded) { // key counting - var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); + const keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); - // The default server is UDP - var server = config.server || './servers/udp' - serverLoaded = startServer(config, server, function (msg, rinfo) { + const handlePacket = function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; - var packet_data = msg.toString(); + let metrics; + const packet_data = msg.toString(); if (packet_data.indexOf("\n") > -1) { - var metrics = packet_data.split("\n"); + metrics = packet_data.split("\n"); } else { - var metrics = [ packet_data ] ; + metrics = [ packet_data ] ; } - for (var midx in metrics) { + for (const midx in metrics) { if (metrics[midx].length === 0) { continue; } @@ -210,11 +252,20 @@ config.configFile(process.argv[2], function (config) { if (config.dumpMessages) { l.log(metrics[midx].toString()); } - var bits = metrics[midx].toString().split(':'); - var key = bits.shift() - .replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + + let bits = metrics[midx].toString().split('|#'); + let tags = []; + if (bits.length > 1 && bits[1].length > 0) { + tags = bits[1].split(','); + } + bits = bits[0].split(':'); + let key = bits.shift(); + if (tags.length > 0) { + key += ';' + tags.map(function(tag) { + return tag.replace(';', '_').replace(':', '='); + }).join(';'); + } + key = sanitizeKeyName(key); if (keyFlushInterval > 0) { if (! keyCounter[key]) { @@ -227,9 +278,9 @@ config.configFile(process.argv[2], function (config) { bits.push("1"); } - for (var i = 0; i < bits.length; i++) { - var sampleRate = 1; - var fields = bits[i].split("|"); + for (let i = 0; i < bits.length; i++) { + let sampleRate = 1; + const fields = bits[i].split("|"); if (!helpers.is_valid_packet(fields)) { l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"'); counters[bad_lines_seen]++; @@ -240,7 +291,7 @@ config.configFile(process.argv[2], function (config) { sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]); } - var metric_type = fields[1].trim(); + const metric_type = fields[1].trim(); if (metric_type === "ms") { if (! timers[key]) { timers[key] = []; @@ -249,6 +300,10 @@ config.configFile(process.argv[2], function (config) { timers[key].push(Number(fields[0] || 0)); timer_counters[key] += (1 / sampleRate); } else if (metric_type === "g") { + // if deleteGauges is true reset the max TTL to its initial value + if (conf.deleteGauges) { + gaugesTTL[key] = conf.gaugesMaxTTL; + } if (gauges[key] && fields[0].match(/^[-+]/)) { gauges[key] += Number(fields[0] || 0); } else { @@ -269,47 +324,31 @@ config.configFile(process.argv[2], function (config) { } stats.messages.last_msg_seen = Math.round(new Date().getTime() / 1000); - }); - - mgmtServer = net.createServer(function(stream) { - stream.setEncoding('ascii'); - - stream.on('error', function(err) { - l.log('Caught ' + err +', Moving on'); - }); - - stream.on('data', function(data) { - var cmdline = data.trim().split(" "); - var cmd = cmdline.shift(); + }; + + // If config.servers isn't specified, use the top-level config for backwards-compatibility + const server_config = config.servers || [config]; + for (let i = 0; i < server_config.length; i++) { + // The default server is UDP + const server = server_config[i].server || './servers/udp'; + startServer(server_config[i], server, handlePacket); + } + mgmt_server.start( + config, + function(cmd, parameters, stream) { switch(cmd) { case "help": stream.write("Commands: stats, counters, timers, gauges, delcounters, deltimers, delgauges, health, config, quit\n\n"); break; case "config": - stream.write("\n"); - for (var prop in config) { - if (!config.hasOwnProperty(prop)) { - continue; - } - if (typeof config[prop] !== 'object') { - stream.write(prop + ": " + config[prop] + "\n"); - continue; - } - subconfig = config[prop]; - for (var subprop in subconfig) { - if (!subconfig.hasOwnProperty(subprop)) { - continue; - } - stream.write(prop + " > " + subprop + ": " + subconfig[subprop] + "\n"); - } - } + helpers.writeConfig(config, stream); break; case "health": - if (cmdline.length > 0) { - var cmdaction = cmdline[0].toLowerCase(); + if (parameters.length > 0) { + const cmdaction = parameters[0].toLowerCase(); if (cmdaction === 'up') { healthStatus = 'up'; } else if (cmdaction === 'down') { @@ -320,13 +359,13 @@ config.configFile(process.argv[2], function (config) { break; case "stats": - var now = Math.round(new Date().getTime() / 1000); - var uptime = now - startup_time; + const now = Math.round(new Date().getTime() / 1000); + const uptime = now - startup_time; stream.write("uptime: " + uptime + "\n"); - var stat_writer = function(group, metric, val) { - var delta; + const stat_writer = function(group, metric, val) { + let delta; if (metric.match("^last_")) { delta = now - val; @@ -339,13 +378,13 @@ config.configFile(process.argv[2], function (config) { }; // Loop through the base stats - for (var group in stats) { - for (var metric in stats[group]) { + for (const group in stats) { + for (const metric in stats[group]) { stat_writer(group, metric, stats[group][metric]); } } - backendEvents.once('status', function(writeCb) { + backendEvents.once('status', function() { stream.write("END\n\n"); }); @@ -377,15 +416,15 @@ config.configFile(process.argv[2], function (config) { break; case "delcounters": - mgmt.delete_stats(counters, cmdline, stream); + mgmt.delete_stats(counters, parameters, stream); break; case "deltimers": - mgmt.delete_stats(timers, cmdline, stream); + mgmt.delete_stats(timers, parameters, stream); break; case "delgauges": - mgmt.delete_stats(gauges, cmdline, stream); + mgmt.delete_stats(gauges, parameters, stream); break; case "quit": @@ -396,13 +435,14 @@ config.configFile(process.argv[2], function (config) { stream.write("ERROR\n"); break; } + }, + function(err, stream) { + l.log('MGMT: Caught ' + err +', Moving on', 'WARNING'); + } + ); - }); - }); - - mgmtServer.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); - - util.log("server is up"); + serversLoaded = true; + util.log("server is up", "INFO"); pctThreshold = config.percentThreshold || 90; if (!Array.isArray(pctThreshold)) { @@ -413,8 +453,8 @@ config.configFile(process.argv[2], function (config) { config.flushInterval = flushInterval; if (config.backends) { - for (var i = 0; i < config.backends.length; i++) { - loadBackend(config, config.backends[i]); + for (let j = 0; j < config.backends.length; j++) { + loadBackend(config, config.backends[j]); } } else { // The default backend is graphite @@ -422,31 +462,31 @@ config.configFile(process.argv[2], function (config) { } // Setup the flush timer - var flushInt = setInterval(flushMetrics, flushInterval); + const flushInt = setTimeout(flushMetrics, getFlushTimeout(flushInterval)); if (keyFlushInterval > 0) { - var keyFlushPercent = Number((config.keyFlush && config.keyFlush.percent) || 100); - var keyFlushLog = config.keyFlush && config.keyFlush.log; + const keyFlushPercent = Number((config.keyFlush && config.keyFlush.percent) || 100); + const keyFlushLog = config.keyFlush && config.keyFlush.log; keyFlushInt = setInterval(function () { - var sortedKeys = []; + const sortedKeys = []; - for (var key in keyCounter) { + for (const key in keyCounter) { sortedKeys.push([key, keyCounter[key]]); } sortedKeys.sort(function(a, b) { return b[1] - a[1]; }); - var logMessage = ""; - var timeString = (new Date()) + ""; + let logMessage = ""; + const timeString = (new Date()) + ""; // only show the top "keyFlushPercent" keys - for (var i = 0, e = sortedKeys.length * (keyFlushPercent / 100); i < e; i++) { + for (let i = 0, e = sortedKeys.length * (keyFlushPercent / 100); i < e; i++) { logMessage += timeString + " count=" + sortedKeys[i][1] + " key=" + sortedKeys[i][0] + "\n"; } if (keyFlushLog) { - var logFile = fs.createWriteStream(keyFlushLog, {flags: 'a+'}); + const logFile = fs.createWriteStream(keyFlushLog, {flags: 'a+'}); logFile.write(logMessage); logFile.end(); } else { diff --git a/test/graphite_delete_counters_tests.js b/test/graphite_delete_counters_tests.js index 05093f41..87a34fbe 100644 --- a/test/graphite_delete_counters_tests.js +++ b/test/graphite_delete_counters_tests.js @@ -87,6 +87,7 @@ module.exports = { , dumpMessages: false \n\ , debug: false\n\ , deleteIdleStats: true\n\ + , gaugesMaxTTL: 2\n\ , graphitePort: " + this.testport + "\n\ , graphiteHost: \"127.0.0.1\"}"; @@ -266,5 +267,73 @@ module.exports = { }); }); }); - } + }, + + gauges_are_valid: function (test) { + test.expect(7); + + var testvalue = "+1"; + var me = this; + this.acceptor.once('connection',function(g){ + statsd_send('a_test_value:' + testvalue + '|g',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*3,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + + // create an associative array which has the flush cycle number as a key + // and a list of hashes of the metrics sent during that flush cycle as value + flushes = {}; + i_number = 0; + hashes.forEach(function (item) { + key = Object.keys(item)[0] + if (key == '') { + i_number++; + return; + } + if (! (i_number in flushes)) { + flushes[i_number] = []; + } + + flushes[i_number].push(item); + }); + + var testavgvalue_test = function(post){ + var mykey = 'stats.gauges.a_test_value'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + + test.ok(_.any(flushes[0],testavgvalue_test), 'stats.gauges.a_test_value after first flush should be ' + 1); + test.ok(_.any(flushes[1],testavgvalue_test), 'stats.gauges.a_test_value after second flush should be ' + 1); + + var testavgvalue_test_after_delete = function(post){ + var mykey = 'stats.gauges.a_test_value'; + return !(_.include(_.keys(post),mykey)); + }; + + test.ok(_.every(flushes[2],testavgvalue_test_after_delete), 'stats.gauges.a_test_value after third flush should not be present'); + + var numstat_test = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(flushes[0],numstat_test), 'statsd.numStats after first flush should be 5'); + test.ok(_.any(flushes[1],numstat_test), 'statsd.numStats after second flush should be 5'); + + var numstat_test_after_delete = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 4); + }; + test.ok(_.any(flushes[2],numstat_test_after_delete), 'statsd.numStats after third flush should be 4'); + + test.done() + }); + + }); + }); +} } diff --git a/test/graphite_pickle_tests.js b/test/graphite_pickle_tests.js new file mode 100644 index 00000000..ca653dff --- /dev/null +++ b/test/graphite_pickle_tests.js @@ -0,0 +1,227 @@ +var fs = require('fs'), + net = require('net'), + temp = require('temp'), + cp = require('child_process'), + util = require('util'), + urlparse = require('url').parse, + _ = require('underscore'), + dgram = require('dgram'), + qsparse = require('querystring').parse, + http = require('http'); + +var spawn = cp.spawn; + +var writeconfig = function(text,worker,cb,obj){ + temp.open({suffix: '-statsdconf.js'}, function(err, info) { + if (err) throw err; + fs.writeSync(info.fd, text); + fs.close(info.fd, function(err) { + if (err) throw err; + worker(info.path,cb,obj); + }); + }); +}; + +var statsd_send = function(data,sock,host,port,cb){ + send_data = new Buffer(data); + sock.send(send_data,0,send_data.length,port,host,function(err,bytes){ + if (err) { + throw err; + } + cb(); + }); +}; + +// keep collecting data until a specified timeout period has elapsed +// this will let us capture all data chunks so we don't miss one +var collect_for = function(server,timeout,cb){ + // We have binary data arriving over the wire. Avoid strings. + var received = new Buffer(0); + var in_flight = 0; + var timed_out = false; + var collector = function(req,res){ + in_flight += 1; + req.on('data',function(data){ received = Buffer.concat([received,data]); }); + req.on('end',function(){ + in_flight -= 1; + if((in_flight < 1) && timed_out){ + server.removeListener('request',collector); + cb(received); + } + }); + }; + + setTimeout(function (){ + timed_out = true; + if((in_flight < 1)) { + server.removeListener('connection',collector); + cb(received); + } + },timeout); + + server.on('connection',collector); +}; + +// A python script that converts from the graphite pickle-based +// wire protocol into JSON written to stdout. +var script = + "import sys\n" + + "import pickle\n" + + "import struct\n" + + "import json\n" + + "payload = open(sys.argv[1], 'rb').read()\n" + + "pack_format = '!L'\n" + + "header_length = struct.calcsize(pack_format)\n" + + "payload_length, = struct.unpack(pack_format, payload[:header_length])\n" + + "batch_length = header_length + payload_length\n" + + "metrics = pickle.loads(payload[header_length:batch_length])\n" + + "print(json.dumps(metrics))\n"; + +// Write our binary payload and unpickling script to disk +// then process the unserialized results. +var unpickle = function(payload, cb) { + temp.open({suffix: '-payload.pickle'}, function(err, payload_info) { + if (err) throw err; + + // the header may contain null characters. explicit length is necessary. + var len = fs.writeSync(payload_info.fd, payload, 0, payload.length); + fs.close(payload_info.fd, function(err) { + if (err) throw err; + + temp.open({suffix:'-unpickle.py'}, function(err, unpickle_info) { + if (err) throw err; + + fs.writeSync(unpickle_info.fd, script); + fs.close(unpickle_info.fd, function(err) { + if (err) throw err; + + var cmd = 'python3 ' + unpickle_info.path + ' ' + payload_info.path; + var python = cp.exec(cmd, function(err, stdout, stderr) { + if (err) throw err; + var metrics = JSON.parse(stdout); + // Transform the output into the same list of dictionaries + // used by the other graphite_* tests so our tests look + // the same. + var hashes = _.map(metrics, function(m) { + var data = {}; + data[m[0]] = m[1][1]; + return data; + }); + cb(hashes); + }); + }); + }); + }); + }); +}; + +module.exports = { + setUp: function (callback) { + this.testport = 31337; + this.myflush = 200; + var configfile = "{graphService: \"graphite\"\n\ + , batch: 200 \n\ + , flushInterval: " + this.myflush + " \n\ + , percentThreshold: 90\n\ + , histogram: [ { metric: \"a_test_value\", bins: [1000] } ]\n\ + , port: 8125\n\ + , dumpMessages: false \n\ + , debug: false\n\ + , graphite: { legacyNamespace: false }\n\ + , graphitePicklePort: " + this.testport + "\n\ + , graphiteHost: \"127.0.0.1\"\n\ + , graphiteProtocol: \"pickle\"}"; + + this.acceptor = net.createServer(); + this.acceptor.listen(this.testport); + this.sock = dgram.createSocket('udp4'); + + this.server_up = true; + this.ok_to_die = false; + this.exit_callback_callback = process.exit; + + writeconfig(configfile,function(path,cb,obj){ + obj.path = path; + obj.server = spawn('node',['stats.js', path]); + obj.exit_callback = function (code) { + obj.server_up = false; + if(!obj.ok_to_die){ + console.log('node server unexpectedly quit with code: ' + code); + process.exit(1); + } + else { + obj.exit_callback_callback(); + } + }; + obj.server.on('exit', obj.exit_callback); + obj.server.stderr.on('data', function (data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + /* + obj.server.stdout.on('data', function (data) { + console.log('stdout: ' + data.toString().replace(/\n$/,'')); + }); + */ + obj.server.stdout.on('data', function (data) { + // wait until server is up before we finish setUp + if (data.toString().match(/server is up/)) { + cb(); + } + }); + + },callback,this); + }, + + tearDown: function (callback) { + this.sock.close(); + this.acceptor.close(); + this.ok_to_die = true; + if(this.server_up){ + this.exit_callback_callback = callback; + this.server.kill(); + } else { + callback(); + } + }, + + timers_are_valid: function (test) { + test.expect(6); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection',function(c){ + statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(payload){ + test.ok(payload.length > 0,'should receive some data'); + unpickle(payload, function(hashes) { + var numstat_test = function(post){ + var mykey = 'stats.statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(hashes,numstat_test), 'stats.statsd.numStats should be 5'); + + var testtimervalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.mean_90'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + var testtimerhistogramvalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be ' + 1); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); + + var count_test = function(post, metric){ + var mykey = 'stats.timers.a_test_value.' + metric; + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); + }; + test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); + test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); + + test.done(); + }); + }); + }); + }); + } +}; diff --git a/test/graphite_tests.js b/test/graphite_tests.js index d3846be0..d8893bb2 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -219,12 +219,12 @@ module.exports = { var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; return _.include(_.keys(post),mykey) && (post[mykey] == 1); }; - test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.mean should be ' + 1); - test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean should be ' + testvalue); + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be ' + 1); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); var count_test = function(post, metric){ var mykey = 'stats.timers.a_test_value.' + metric; - return _.first(_.filter(_.pluck(post, mykey), function (e) { return e })); + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); }; test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); @@ -358,5 +358,62 @@ module.exports = { }); }); }); + }, + + metric_names_are_sanitized: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); + test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); + test.ok(str.indexOf('foobar') !== -1, "Did not map 'foo+bar' => 'foobar'"); + test.done(); + }); + }); + }); + }); + }); + }, + + graphite_tags_are_supported: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o;tag1=val1:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar;tag1=val1;tag2=val2:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar;tag1=val1;tag3=val3;tag2=val2:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o.count;tag1=val1') !== -1, "Did not map 'fo/o;tag1=val1' => 'fo-o.count;tag1=val1'"); + test.ok(str.indexOf('b_ar.count;tag1=val1;tag2=val2') !== -1, "Did not map 'b ar;tag1=val1;tag2=val2' => 'b_ar.count;tag1=val1;tag2=val2'"); + test.ok(str.indexOf('foobar.count;tag1=val1;tag3=val3;tag2=val2') !== -1, "Did not map 'foo+bar;tag1=val1;tag3=val3;tag2=val2' => 'foobar.count;tag1=val1;tag3=val3;tag2=val2'"); + test.done(); + }); + }); + }); + }); + }); + }, + + dogstatsd_tags_are_supported: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c|#tag1:val1',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c|#tag1:val1,tag2:val2',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c|#tag1:val;1,tag3:val3,tag2:val2',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o.count;tag1=val1') !== -1, "Did not map 'fo/o:250|c|#tag1:val1' => 'fo-o.count;tag1=val1'"); + test.ok(str.indexOf('b_ar.count;tag1=val1;tag2=val2') !== -1, "Did not map 'b ar:250|c|#tag1:val1,tag2:val2' => 'b_ar.count;tag1=val1;tag2=val2'"); + test.ok(str.indexOf('foobar.count;tag1=val_1;tag3=val3;tag2=val2') !== -1, "Did not map 'foo+bar:250|c|#tag1:val;1,tag3:val3,tag2:val2' => 'foobar.count;tag1=val_1;tag3=val3;tag2=val2'"); + test.done(); + }); + }); + }); + }); + }); } } diff --git a/test/graphite_tests_filters.js b/test/graphite_tests_filters.js new file mode 100644 index 00000000..8de66d7c --- /dev/null +++ b/test/graphite_tests_filters.js @@ -0,0 +1,180 @@ +var fs = require('fs'), + net = require('net'), + temp = require('temp'), + spawn = require('child_process').spawn, + util = require('util'), + urlparse = require('url').parse, + _ = require('underscore'), + dgram = require('dgram'), + qsparse = require('querystring').parse, + http = require('http'); + + +var writeconfig = function(text, worker, cb, obj){ + temp.open({suffix: '-statsdconf.js'}, function(err, info) { + if (err) throw err; + fs.writeSync(info.fd, text); + fs.close(info.fd, function(err) { + if (err) throw err; + worker(info.path, cb, obj); + }); + }); +} + +var statsd_send = function(data,sock,host,port,cb){ + send_data = new Buffer(data); + sock.send(send_data,0,send_data.length,port,host,function(err,bytes){ + if (err) { + throw err; + } + cb(); + }); +} + +// keep collecting data until a specified timeout period has elapsed +// this will let us capture all data chunks so we don't miss one +var collect_for = function(server,timeout,cb){ + var received = []; + var in_flight = 0; + var timed_out = false; + var collector = function(req,res){ + in_flight += 1; + var body = ''; + req.on('data',function(data){ body += data; }); + req.on('end',function(){ + received = received.concat(body.split("\n")); + in_flight -= 1; + if((in_flight < 1) && timed_out){ + server.removeListener('request',collector); + cb(received); + } + }); + } + + setTimeout(function (){ + timed_out = true; + if((in_flight < 1)) { + server.removeListener('connection',collector); + cb(received); + } + },timeout); + + server.on('connection',collector); +} +module.exports = { + setUp: function (callback) { + this.testport = 31337; + this.myflush = 200; + var configfile = "{graphService: \"graphite\"\n\ + , batch: 200 \n\ + , flushInterval: " + this.myflush + " \n\ + , percentThreshold: 90\n\ + , calculatedTimerMetrics: ['count_ps', 'count', 'count_percent', 'mean_percent', 'histogram']\n\ + , histogram: [ { metric: \"a_test_value\", bins: [1000] } ]\n\ + , port: 8125\n\ + , dumpMessages: false \n\ + , debug: false\n\ + , graphite: { legacyNamespace: false }\n\ + , graphitePort: " + this.testport + "\n\ + , graphiteHost: \"127.0.0.1\"}"; + + this.acceptor = net.createServer(); + this.acceptor.listen(this.testport); + this.sock = dgram.createSocket('udp4'); + + this.server_up = true; + this.ok_to_die = false; + this.exit_callback_callback = process.exit; + + writeconfig(configfile,function(path, cb, obj){ + obj.path = path; + obj.server = spawn('node',['stats.js', path]); + obj.exit_callback = function (code) { + obj.server_up = false; + if(!obj.ok_to_die){ + console.log('node server unexpectedly quit with code: ' + code); + process.exit(1); + } + else { + obj.exit_callback_callback(); + } + }; + obj.server.on('exit', obj.exit_callback); + obj.server.stderr.on('data', function (data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + /* + obj.server.stdout.on('data', function (data) { + console.log('stdout: ' + data.toString().replace(/\n$/,'')); + }); + */ + obj.server.stdout.on('data', function (data) { + // wait until server is up before we finish setUp + if (data.toString().match(/server is up/)) { + cb(); + } + }); + + },callback,this); + }, + tearDown: function (callback) { + this.sock.close(); + this.acceptor.close(); + this.ok_to_die = true; + if(this.server_up){ + this.exit_callback_callback = callback; + this.server.kill(); + } else { + callback(); + } + }, + + timers_are_valid: function (test) { + test.expect(11); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection', function(c){ + statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + var numstat_test = function(post){ + var mykey = 'stats.statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(hashes,numstat_test), 'stats.statsd.numStats should be 5'); + + var testtimervalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.mean_90'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + var testtimerhistogramvalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be 1'); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); + + var count_test = function(post, metric){ + var mykey = 'stats.timers.a_test_value.' + metric; + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); + }; + test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); + test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); + test.equals(count_test(hashes, 'count_90'), 1, 'count_90 should be 1'); + test.equals(count_test(hashes, 'sum'), null, 'sum should be null'); + test.equals(count_test(hashes, 'sum_squares'), null, 'sum_squares should be null'); + test.equals(count_test(hashes, 'sum_90'), null, 'sum_90 should be null'); + test.equals(count_test(hashes, 'sum_squares_90'), null, 'sum_squares_90 should be null'); + test.done(); + }); + }); + }); + }, +} diff --git a/test/helpers_tests.js b/test/helpers_tests.js index d10da0ba..5dcd44aa 100644 --- a/test/helpers_tests.js +++ b/test/helpers_tests.js @@ -112,6 +112,7 @@ module.exports = { test.equals(helpers.is_valid_packet(['345345', 'ms', '@.']), false); test.equals(helpers.is_valid_packet(['345345', 'ms', '@.1.']), false); test.equals(helpers.is_valid_packet(['345345', 'ms', '@.1.2.3']), false); + test.equals(helpers.is_valid_packet(['345345', 'ms', '@-1.0']), false); test.done(); }, @@ -133,6 +134,12 @@ module.exports = { var res = helpers.is_valid_packet(['100', 'ms', '@0.1']); test.equals(res, true); test.done(); + }, + + correct_counter_with_float: function (test) { + var res = helpers.is_valid_packet(['1.0', 'c', '@0.1']); + test.equals(res, true); + test.done(); } }; diff --git a/test/process_metrics_tests.js b/test/process_metrics_tests.js index 6a5b1823..390f863c 100644 --- a/test/process_metrics_tests.js +++ b/test/process_metrics_tests.js @@ -11,6 +11,7 @@ module.exports = { var timer_counters = {}; var sets = {}; var pctThreshold = null; + var calculatedTimerMetrics = []; this.metrics = { counters: counters, @@ -25,14 +26,14 @@ module.exports = { counters_has_stats_count: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 1000, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 1000, this.time_stamp, function(){}); test.equal(2, this.metrics.counters['a']); test.done(); }, counters_has_correct_rate: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); test.equal(20, this.metrics.counter_rates['a']); test.done(); }, @@ -40,7 +41,7 @@ module.exports = { test.expect(1); this.metrics.timers['a'] = []; this.metrics.timer_counters['a'] = 0; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); //potentially a cleaner way to check this test.equal(undefined, this.metrics.counter_rates['a']); test.done(); @@ -49,7 +50,7 @@ module.exports = { test.expect(9); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(0, timer_data.std); test.equal(100, timer_data.upper); @@ -62,11 +63,49 @@ module.exports = { test.equal(100, timer_data.median); test.done(); }, - timers_multiple_times: function(test) { + timer_single_time_with_one_filter: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + let filter = ['upper', 'lower', 'count', 'count_ps', 'sum', 'sum_squares', 'mean', 'median'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(8, Object.keys(timer_data).length) + test.equal(null, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(1, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(100, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(100, timer_data.mean); + test.equal(100, timer_data.median); + test.done(); + }, + timer_single_time_multiple_filter: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + let filter = ['upper', 'lower', 'count_ps', 'sum_squares'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(4, Object.keys(timer_data).length) + test.equal(null, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(null, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(null, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(null, timer_data.mean); + test.equal(null, timer_data.median); + test.done(); + }, + timers_multiple_times: function(test) { test.expect(9); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(81.64965809277261, timer_data.std); test.equal(300, timer_data.upper); @@ -75,17 +114,36 @@ module.exports = { test.equal(30, timer_data.count_ps); test.equal(600, timer_data.sum); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares); + timer_data.sum_squares); + test.equal(200, timer_data.mean); + test.equal(200, timer_data.median); + test.done(); + }, + timers_multiple_times_with_calculated_timer_metrics: function(test) { + test.expect(9); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + let calculatedTimerMetrics = ['std', 'count', 'sum_squares', 'mean', 'median'] + pm.process_metrics(this.metrics, calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(81.64965809277261, timer_data.std); + test.equal(null, timer_data.upper); + test.equal(null, timer_data.lower); + test.equal(3, timer_data.count); + test.equal(null, timer_data.count_ps); + test.equal(null, timer_data.sum); + test.equal(100 * 100 + 200 * 200 + 300 * 300, + timer_data.sum_squares); test.equal(200, timer_data.mean); test.equal(200, timer_data.median); test.done(); }, - timers_single_time_single_percentile: function(test) { + timers_single_time_single_percentile: function(test) { test.expect(4); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); @@ -93,45 +151,94 @@ module.exports = { test.equal(100 * 100, timer_data.sum_squares_90); test.done(); }, - timers_single_time_multiple_percentiles: function(test) { - test.expect(9); + timers_single_time_single_percentile_with_calculated_timer_metrics: function(test) { + test.expect(4); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + this.metrics.pctThreshold = [90]; + pm.process_metrics(this.metrics, ['upper_percent', 'sum_squares_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.mean_90); + test.equal(100, timer_data.upper_90); + test.equal(null, timer_data.sum_90); + test.equal(100 * 100, timer_data.sum_squares_90); + test.done(); + }, + timers_single_time_multiple_percentiles: function(test) { + test.expect(10); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(1, timer_data.count_90); test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); test.equal(100, timer_data.sum_90); test.equal(100 * 100, timer_data.sum_squares_90); + test.equal(1, timer_data.count_80); test.equal(100, timer_data.mean_80); test.equal(100, timer_data.upper_80); test.equal(100, timer_data.sum_80); test.equal(100 * 100, timer_data.sum_squares_80); test.done(); }, - timers_multiple_times_single_percentiles: function(test) { + timers_single_time_multiple_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + this.metrics.pctThreshold = [90, 80]; + let calculatedTimerMetrics = ['mean_percent', 'sum_percent'] + pm.process_metrics(this.metrics, calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.count_90); + test.equal(100, timer_data.mean_90); + test.equal(null, timer_data.upper_90); + test.equal(100, timer_data.sum_90); + test.equal(null, timer_data.sum_squares_90); + test.equal(null, timer_data.count_80); + test.equal(100, timer_data.mean_80); + test.equal(null, timer_data.upper_80); + test.equal(100, timer_data.sum_80); + test.equal(null, timer_data.sum_squares_80); + test.done(); + }, + timers_multiple_times_single_percentiles: function(test) { test.expect(5); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(3, timer_data.count_90); test.equal(200, timer_data.mean_90); test.equal(300, timer_data.upper_90); test.equal(600, timer_data.sum_90); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares_90); + timer_data.sum_squares_90); + test.done(); + }, + timers_multiple_times_single_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(5); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + this.metrics.pctThreshold = [90]; + let filter = ['count_percent', 'mean_percent', 'upper_percent'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(3, timer_data.count_90); + test.equal(200, timer_data.mean_90); + test.equal(300, timer_data.upper_90); + test.equal(null, timer_data.sum_90); + test.equal(null, timer_data.sum_squares_90); test.done(); }, - timers_multiple_times_multiple_percentiles: function(test) { + timers_multiple_times_multiple_percentiles: function(test) { test.expect(11); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(3, timer_data.count); test.equal(3, timer_data.count_90); @@ -139,22 +246,45 @@ module.exports = { test.equal(300, timer_data.upper_90); test.equal(600, timer_data.sum_90); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares_90); + timer_data.sum_squares_90); test.equal(2, timer_data.count_80); test.equal(150, timer_data.mean_80); test.equal(200, timer_data.upper_80); test.equal(300, timer_data.sum_80); test.equal(100 * 100 + 200 * 200, - timer_data.sum_squares_80); + timer_data.sum_squares_80); + test.done(); + }, + timers_multiple_times_multiple_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(11); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + this.metrics.pctThreshold = [90, 80]; + pm.process_metrics(this.metrics, ['count_percent', 'sum_percent', 'sum_squares_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.count); + test.equal(3, timer_data.count_90); + test.equal(null, timer_data.mean_90); + test.equal(null, timer_data.upper_90); + test.equal(600, timer_data.sum_90); + test.equal(100 * 100 + 200 * 200 + 300 * 300, + timer_data.sum_squares_90); + + test.equal(2, timer_data.count_80); + test.equal(null, timer_data.mean_80); + test.equal(null, timer_data.upper_80); + test.equal(300, timer_data.sum_80); + test.equal(100 * 100 + 200 * 200, + timer_data.sum_squares_80); test.done(); }, - timers_sampled_times: function(test) { + timers_sampled_times: function(test) { test.expect(8); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 50; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(50, timer_data.count); test.equal(500, timer_data.count_ps); @@ -166,7 +296,45 @@ module.exports = { test.equal(300, timer_data.sum_80); test.done(); }, // check if the correct settings are being applied. as well as actual counts - timers_histogram: function (test) { + timers_histogram: function (test) { + test.expect(13); + this.metrics.timers['a'] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + this.metrics.timers['abc'] = [0.1234, 2.89, 4, 6, 8]; + this.metrics.timers['foo'] = [0, 2, 4, 6, 8]; + this.metrics.timers['barbazfoobar'] = [0, 2, 4, 6, 8]; + this.metrics.timers['bar.bazfoobar.abc'] = [0, 2, 4, 6, 8]; + this.metrics.timers['xyz'] = [0, 2, 4, 6, 8]; + this.metrics.histogram = [ { metric: 'foo', bins: [] }, + { metric: 'abcd', bins: [ 1, 5, 'inf'] }, + { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, + { metric: 'a', bins: [ 1, 2] } ]; + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data; + // nothing matches the 'abcd' calculatedTimerMetrics, so nothing has bin_5 + test.equal(undefined, timer_data['a']['histogram']['bin_5']); + test.equal(undefined, timer_data['abc']['histogram']['bin_5']); + + // check that 'a' got the right calculatedTimerMetrics and numbers + test.equal(0, timer_data['a']['histogram']['bin_1']); + test.equal(1, timer_data['a']['histogram']['bin_2']); + test.equal(undefined, timer_data['a']['histogram']['bin_inf']); + + // only 'abc' should have a bin_inf; also check all its counts, + // and make sure it has no other bins + test.equal(1, timer_data['abc']['histogram']['bin_1']); + test.equal(0, timer_data['abc']['histogram']['bin_2_21']); + test.equal(4, timer_data['abc']['histogram']['bin_inf']); + test.equal(3, _.size(timer_data['abc']['histogram'])); + + // these all have histograms disabled ('foo' explicitly, rest implicitly) + test.equal(undefined, timer_data['foo']['histogram']); + test.equal(undefined, timer_data['barbazfoobar']['histogram']); + test.equal(undefined, timer_data['bar.bazfoobar.abc']['histogram']); + test.equal(undefined, timer_data['xyz']['histogram']); + + test.done(); + }, + timers_histogram_with_calculated_timer_metrics: function (test) { test.expect(13); this.metrics.timers['a'] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; this.metrics.timers['abc'] = [0.1234, 2.89, 4, 6, 8]; @@ -175,16 +343,16 @@ module.exports = { this.metrics.timers['bar.bazfoobar.abc'] = [0, 2, 4, 6, 8]; this.metrics.timers['xyz'] = [0, 2, 4, 6, 8]; this.metrics.histogram = [ { metric: 'foo', bins: [] }, - { metric: 'abcd', bins: [ 1, 5, 'inf'] }, - { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, - { metric: 'a', bins: [ 1, 2] } ]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + { metric: 'abcd', bins: [ 1, 5, 'inf'] }, + { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, + { metric: 'a', bins: [ 1, 2] } ]; + pm.process_metrics(this.metrics, ['histogram'], 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data; - // nothing matches the 'abcd' config, so nothing has bin_5 + // nothing matches the 'abcd' calculatedTimerMetrics, so nothing has bin_5 test.equal(undefined, timer_data['a']['histogram']['bin_5']); test.equal(undefined, timer_data['abc']['histogram']['bin_5']); - // check that 'a' got the right config and numbers + // check that 'a' got the right calculatedTimerMetrics and numbers test.equal(0, timer_data['a']['histogram']['bin_1']); test.equal(1, timer_data['a']['histogram']['bin_2']); test.equal(undefined, timer_data['a']['histogram']['bin_inf']); @@ -204,41 +372,88 @@ module.exports = { test.done(); }, - timers_single_time_single_top_percentile: function(test) { + timers_single_time_single_top_percentile: function(test) { test.expect(3); this.metrics.timers['a'] = [100]; this.metrics.pctThreshold = [-10]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_top10); test.equal(100, timer_data.lower_top10); test.equal(100, timer_data.sum_top10); test.done(); }, - timers_multiple_times_single_top_percentile: function(test) { + timers_single_time_single_top_percentile_with_calculated_timer_metrics: function(test) { + test.expect(3); + this.metrics.timers['a'] = [100]; + this.metrics.pctThreshold = [-10]; + pm.process_metrics(this.metrics, ['lower_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.mean_top10); + test.equal(100, timer_data.lower_top10); + test.equal(null, timer_data.sum_top10); + test.done(); + }, + timers_multiple_times_single_top_percentile: function(test) { test.expect(3); this.metrics.timers['a'] = [10, 10, 10, 10, 10, 10, 10, 10, 100, 200]; this.metrics.pctThreshold = [-20]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(150, timer_data.mean_top20); test.equal(100, timer_data.lower_top20); test.equal(300, timer_data.sum_top20); test.done(); }, - statsd_metrics_exist: function(test) { + timers_multiple_times_single_top_percentile_with_calculated_timer_metrics: function(test) { + test.expect(3); + this.metrics.timers['a'] = [10, 10, 10, 10, 10, 10, 10, 10, 100, 200]; + this.metrics.pctThreshold = [-20]; + pm.process_metrics(this.metrics, ['mean_percent', 'sum_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(150, timer_data.mean_top20); + test.equal(null, timer_data.lower_top20); + test.equal(300, timer_data.sum_top20); + test.done(); + }, + statsd_metrics_exist: function(test) { test.expect(1); - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); statsd_metrics = this.metrics.statsd_metrics; test.notEqual(undefined, statsd_metrics["processing_time"]); test.done(); }, - timers_multiple_times_even: function(test) { + timers_multiple_times_even: function(test) { + test.expect(1); + this.metrics.timers['a'] = [300, 200, 400, 100]; + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(250, timer_data.median); + test.done(); + }, + timers_multiple_times_even_with_calculated_timer_metrics: function(test) { test.expect(1); this.metrics.timers['a'] = [300, 200, 400, 100]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, ['median'], 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(250, timer_data.median); test.done(); + }, + timers_with_invalid_filter: function(test) { + test.expect(9); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + pm.process_metrics(this.metrics, 'not a valid filter', 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(0, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(1, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(100, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(100, timer_data.mean); + test.equal(100, timer_data.median); + test.done(); } } diff --git a/test/repeater_tests.js b/test/repeater_tests.js new file mode 100644 index 00000000..1a1008d0 --- /dev/null +++ b/test/repeater_tests.js @@ -0,0 +1,171 @@ +var net = require('net'), + spawn = require('child_process').spawn, + fs = require('fs'), + temp = require('temp'), + dgram = require('dgram'), + EventEmitter = require('events').EventEmitter; + + + + +function log() { + //console.log.apply(console, arguments); +} + + +function StatsDWrapper(serverpath) { + var wrapper = function(port, message_callback) { + this.port = port || 9125; + this.statsd = require(serverpath); + this.message_callback = message_callback; + }; + + wrapper.prototype.start = function(cb) { + var self = this; + this.statsd.start({ port: this.port }, function(packet, rinfo) { + self.message_callback(packet, rinfo); + }); + + this.statsd.server.on('listening', function() { + cb(); + }); + }; + + return wrapper; +}; + +var TcpStatsD = StatsDWrapper('../servers/tcp'); +TcpStatsD.prototype.stop = function(cb) { + if(this.statsd.server) { + this.statsd.server.close(cb); + } +}; + +var UdpStatsD = StatsDWrapper('../servers/udp'); +UdpStatsD.prototype.stop = function(cb) { + if(this.statsd.server) { + this.statsd.server.close(); + cb(); + } +}; + + + +var RepeaterServer = function(port, server_port) { + this.port = port || 8125; + this.server_port = server_port || 9125; + this.config = { + repeater: [{ host: '127.0.0.1', port: this.server_port }], + repeaterProtocol: 'udp4', + server: './servers/udp', + port: this.port, + backends: [ './backends/repeater' ] + }; + + this.emitter = new EventEmitter(); +}; + +RepeaterServer.prototype.start = function(cb) { + this.repeater = require('../backends/repeater'); + this.repeater.init(0, this.config, this.emitter); + cb(); +}; + +RepeaterServer.prototype.send = function(stringval) { + this.emitter.emit('packet', new Buffer(stringval), {}); +}; + +RepeaterServer.prototype.stop = function(cb) { + this.repeater.stop(cb); +}; + + + +var ServerSet = function() { + this.servers = []; +}; +ServerSet.prototype.add = function() { + for(var i = 0; i < arguments.length; i++) { + this.servers.push(arguments[i]); + } +}; +ServerSet.prototype.start = function(cb) { + var self = this; + function start_server(i) { + if(i == self.servers.length) { + cb(); + } else { + self.servers[i].start(function() { + start_server(i + 1); + }); + } + } + start_server(0); +}; +ServerSet.prototype.stop = function(cb) { + var self = this; + function stop_server(i) { + if(i == self.servers.length) { + cb(); + } else { + self.servers[i].stop(function() { + stop_server(i + 1); + }); + } + } + stop_server(0); +}; + + + +module.exports = { + + setUp: function(cb) { + this.servers = new ServerSet(); + this.repeater = new RepeaterServer(); + this.servers.add(this.repeater); + cb(); + }, + + tearDown: function(cb) { + this.servers.stop(cb); + }, + + + repeater_works: function(test) { + test.expect(1); + var statsd = new UdpStatsD(9125, function(packet, rinfo) { + test.equal('foobar', packet.toString()); + test.done(); + }); + + this.servers.add(statsd); + + var repeater = this.repeater; + + this.servers.start(function() { + repeater.send('foobar'); + }); + }, + + tcp_repeater_works: function(test) { + test.expect(1); + + var statsd = new TcpStatsD(9125, function(packet, rinfo) { + test.equal('foobar\n', packet.toString()); + test.done(); + }); + + this.servers.add(statsd); + + var repeater = this.repeater; + repeater.config.repeaterProtocol = 'tcp'; + + this.servers.start(function() { + repeater.send('foobar'); + }); + } + +}; + + diff --git a/test/server_tests.js b/test/server_tests.js index b15e70d2..77bb7dac 100644 --- a/test/server_tests.js +++ b/test/server_tests.js @@ -1,5 +1,6 @@ var dgram = require('dgram'), - net = require('net'); + net = require('net'), + fs = require('fs'); var config = { address: '127.0.0.1', @@ -59,5 +60,24 @@ module.exports = { }); client.end(); }); + }, + unix_socket_data_received: function(test) { + test.expect(3); + var server = require('../servers/tcp'); + config.socket = './statsd_tmp.socket'; + var started = server.start(config, function (data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + fs.unlinkSync(config.socket); + config.socket = undefined; + test.done(); + }); + + test.ok(started); + + var client = net.connect(config.socket, function () { + client.write(msg); + client.end(); + }); } -} +}; diff --git a/test/set_tests.js b/test/set_tests.js index 47b645bc..99f20867 100644 --- a/test/set_tests.js +++ b/test/set_tests.js @@ -37,5 +37,18 @@ module.exports = { s.insert('b'); test.equal(2, s.values().length); test.done(); + }, + size_is_correct: function(test) { + test.expect(5); + var s = new set.Set(); + test.equal(0, s.size()); + s.insert('a'); + test.equal(1, s.size()); + s.insert('a'); + test.equal(1, s.size()); + s.insert('b'); + test.equal(2, s.size()); + test.equal(s.values().length, s.size()); + test.done(); } } diff --git a/utils/check_statsd_health b/utils/check_statsd_health new file mode 100644 index 00000000..b6ec96ea --- /dev/null +++ b/utils/check_statsd_health @@ -0,0 +1,62 @@ +#!/bin/bash + +# Check the status of a statsd or statsd proxy connecting directly to the +# management console. + +# This script can be used both for Nagios and Keepalived passing a parameter. +# The default behaviour is the Nagios one. + +OK=0; +CRITICAL=2; +UNKNOWN=3; +KEEPALIVED_CRITICAL=1; + +HOST="127.0.0.1" +PORT="8126" +MODE_KEEPALIVED=0 + +print_usage() { + echo "Usage: check_statsd_health [-H host] [-P port] [-k]" + echo "Options:" + echo " -H host Specify the host to check. [default: 127.0.0.1]" + echo " -P port Specify the port to check. [default: 8126]" + echo " -k Use exit status for Keepalived MISC_CHECK. [default: false]" + + if [[ "${MODE_KEEPALIVED}" -eq "1" ]]; then + exit ${KEEPALIVED_CRITICAL} + else + exit ${UNKNOWN} + fi +} + +while getopts ":H:P:k" opt; do + case $opt in + H) HOST="${OPTARG}";; + P) PORT="${OPTARG}";; + k) MODE_KEEPALIVED=1;; + + :) + echo "Missing mandatory value for option '-${OPTARG}'" >&2 + print_usage + ;; + + \?) + echo "Invalid option '${OPTARG}'" >&2 + print_usage + ;; + + esac +done + +HEALTH="$(echo -e "health\n" | nc "${HOST}" "${PORT}")" +echo "Statsd '${HOST}:${PORT}' responded: '${HEALTH}'" + +if [[ "${HEALTH}" == "health: up" ]]; then + exit ${OK} +else + if [[ "${MODE_KEEPALIVED}" -eq "1" ]]; then + exit ${KEEPALIVED_CRITICAL} + else + exit ${CRITICAL} + fi +fi