From 2d8aaf0ece9d0bdcc7dd49597d3244882a394ce8 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Mon, 24 Sep 2012 14:12:13 -0400 Subject: [PATCH 001/207] Stat key name sanitization is now configurable at the top-level. Setting keyNameSanitize to false pushes the requirement of sanitizing key names to the backends. This permits backends that have less strict character set requirements to take advantage of an expanded stat name character set. The default behavior remains the same as collisions in key name space are not handled if two different stat names map to the same sanitized key name. --- backends/graphite.js | 28 +++++++++++++++++++++++----- exampleConfig.js | 3 +++ stats.js | 20 ++++++++++++++++---- test/graphite_tests.js | 19 +++++++++++++++++++ 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 41b6e325..5d7d8bb3 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -34,6 +34,7 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -98,15 +99,27 @@ var flush_stats = function graphite_flush(ts, metrics) { var timer_data = metrics.timer_data; var statsd_metrics = metrics.statsd_metrics; + // Sanitize key for graphite if not done globally + function sk(key) { + if (globalKeySanitize) { + return key; + } else { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } + }; + for (key in counters) { - var namespace = counterNamespace.concat(key); var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate + var keyName = sk(key); + var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; if (flush_counts) { - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + statString += 'stats_counts.' + keyName + globalSuffix + value + ts_suffix; } } else { statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; @@ -119,8 +132,9 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(key); + var namespace = timerNamespace.concat(sk(key)); var the_key = namespace.join("."); + for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; @@ -138,13 +152,13 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(key); + var namespace = gaugesNamespace.concat(sk(key)); statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(key); + var namespace = setsNamespace.concat(sk(key)); statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; numStats += 1; } @@ -238,6 +252,10 @@ exports.init = function graphite_init(startup_time, config, events) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/exampleConfig.js b/exampleConfig.js index a5406c38..ff77fce0 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -52,6 +52,9 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index 628a3ce7..e8eef81f 100644 --- a/stats.js +++ b/stats.js @@ -28,6 +28,7 @@ var flushInterval, keyFlushInt, server, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; +var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -135,6 +136,16 @@ var stats = { } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } else { + return key; + } +} + // Global for the logger var l; @@ -156,6 +167,10 @@ config.configFile(process.argv[2], function (config, oldConfig) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } + if (server === undefined) { // key counting @@ -180,10 +195,7 @@ config.configFile(process.argv[2], function (config, oldConfig) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = bits.shift() - .replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + var key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { diff --git a/test/graphite_tests.js b/test/graphite_tests.js index b93a6aec..503b1388 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -358,5 +358,24 @@ module.exports = { }); }); }); + }, + + metric_names_are_sanitized: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); + test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); + test.ok(str.indexOf('foobar') !== -1, "Did not map 'foo+bar' => 'foobar'"); + test.done(); + }); + }); + }); + }); + }); } } From 0375990f8fcb0663bbedbc003867f06979305ca1 Mon Sep 17 00:00:00 2001 From: John Leach Date: Mon, 21 Jul 2014 11:49:08 +0100 Subject: [PATCH 002/207] Describe timer sample rate in metric_types docs. Fixes #435 Code does accept a sample rate for time metrics: https://github.com/etsy/statsd/blob/bad366cd438cd59b2064dd03ff3d415fc7db0c4b/stats.js#L206-L225 At least one client sends these (Openstack Swift) and at least one server implementation considers them malformed currently (collectd), presumbaly due to the bad docs :) --- docs/metric_types.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/metric_types.md b/docs/metric_types.md index 75b68d3c..ce370774 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -22,7 +22,7 @@ Tells StatsD that this counter is being sent sampled every 1/10th of the time. Timing ------ - glork:320|ms + glork:320|ms|@0.1 The glork took 320ms to complete this time. StatsD figures out percentiles, average (mean), standard deviation, sum, lower and upper bounds for the flush interval. @@ -67,6 +67,10 @@ Examples: [ { metric: 'foo', bins: [] }, { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] +Statsd also maintains a counter for each timer metric. The 3rd field +specifies the sample rate for this counter (in this example @0.1). The field +is optional and defaults to 1. + Note: * first match for a metric wins. From a7426b32d10f73ea638090480452a9068632868e Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:47:06 +0300 Subject: [PATCH 003/207] Fix merge conflicts with master --- stats.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/stats.js b/stats.js index dbd1c521..abd3fd6c 100644 --- a/stats.js +++ b/stats.js @@ -27,12 +27,9 @@ var flushInterval, keyFlushInt, serverLoaded, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; -<<<<<<< HEAD var old_timestamp = 0; var timestamp_lag_namespace; -======= var keyNameSanitize = true; ->>>>>>> pr/155 // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -194,15 +191,11 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; -<<<<<<< HEAD - if (!serverLoaded) { -======= if (config.keyNameSanitize !== undefined) { keyNameSanitize = config.keyNameSanitize; } - if (server === undefined) { ->>>>>>> pr/155 + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); From 494ed3029bad61ddfc76f0838b563cb17ba44168 Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:51:10 +0300 Subject: [PATCH 004/207] Fix merge conflicts with master --- backends/graphite.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index a0295e71..09e7b48f 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -32,11 +32,8 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; -<<<<<<< HEAD var prefixStats; -======= var globalKeySanitize = true; ->>>>>>> pr/155 // set up namespaces var legacyNamespace = true; From dc5bd4247c6aabc39ef474e71326bc4fae65197f Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 16:31:42 +0300 Subject: [PATCH 005/207] increase timeout due to sporadic failures --- test/graphite_tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/graphite_tests.js b/test/graphite_tests.js index b6774322..971af4a5 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -366,7 +366,7 @@ module.exports = { statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ - collect_for(me.acceptor, me.myflush, function(strings){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ var str = strings.join(); test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); From a1b9b2169d26cfdd449c9279128edfa835a299f3 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Tue, 2 Sep 2014 20:02:13 +0000 Subject: [PATCH 006/207] bumping for v0.7.2 patch release Let's see if travis will upload a release to npm for us. --- Changelog.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 7210191b..5c62ab01 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # Changelog +## v0.7.2 (09/02/2014) +- Fixes to detecting valid packets + ## v0.7.1 (02/06/2014) - move contributing information into CONTRIBUTING.md - Updates winser to v0.1.6 diff --git a/package.json b/package.json index 48eb0e51..62482098 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "type": "git", "url": "https://github.com/etsy/statsd.git" }, - "version": "0.7.1", + "version": "0.7.2", "dependencies": { }, "devDependencies": { From 523ae8330d0390081346d9dd2b355126e2a8019e Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Sun, 27 Jul 2014 11:33:52 +0400 Subject: [PATCH 007/207] Added IDEA project files to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3c3629e6..7d618a48 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ node_modules + +# WebStorm / IntelliJ IDEA project files +.idea +*.iml From 2006226adf1ba29c4e1dea39384f2ed379993742 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 18:53:17 +0400 Subject: [PATCH 008/207] Removed unused variable --- backends/graphite.js | 1 - 1 file changed, 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index f988702a..8a9adbfa 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -27,7 +27,6 @@ var flush_counts; // prefix configuration var globalPrefix; -var prefixPersecond; var prefixCounter; var prefixTimer; var prefixGauge; From d47e6d71d116ba8c1e40c0e6490ef28861896c70 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 18:58:30 +0400 Subject: [PATCH 009/207] Fix `prefixStats` global variable leaking `prefixStats` is leaked through stats.js and was in global scope of event emitter without failing graphite backend by pure accident. Now it is read from config. --- backends/graphite.js | 3 +++ stats.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index 8a9adbfa..34cd299d 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -32,6 +32,7 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; +var prefixStats; // set up namespaces var legacyNamespace = true; @@ -188,6 +189,7 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixSet = config.graphite.prefixSet; globalSuffix = config.graphite.globalSuffix; legacyNamespace = config.graphite.legacyNamespace; + prefixStats = config.prefixStats; // set defaults for prefixes & suffix globalPrefix = globalPrefix !== undefined ? globalPrefix : "stats"; @@ -195,6 +197,7 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixTimer = prefixTimer !== undefined ? prefixTimer : "timers"; prefixGauge = prefixGauge !== undefined ? prefixGauge : "gauges"; prefixSet = prefixSet !== undefined ? prefixSet : "sets"; + prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; legacyNamespace = legacyNamespace !== undefined ? legacyNamespace : true; // In order to unconditionally add this string, it either needs to be diff --git a/stats.js b/stats.js index ccfe6768..ed7b599c 100644 --- a/stats.js +++ b/stats.js @@ -151,7 +151,7 @@ config.configFile(process.argv[2], function (config, oldConfig) { l = new logger.Logger(config.log || {}); // setup config for stats prefix - prefixStats = config.prefixStats; + var prefixStats = config.prefixStats; prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; //setup the names for the stats stored in counters{} bad_lines_seen = prefixStats + ".bad_lines_seen"; From 7e8efc2c856a4ee5bb4ee1b412d2fed01c3a3be5 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 19:47:27 +0400 Subject: [PATCH 010/207] Removed unused parameter --- stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats.js b/stats.js index ed7b599c..bea7a07e 100644 --- a/stats.js +++ b/stats.js @@ -143,7 +143,7 @@ var stats = { // Global for the logger var l; -config.configFile(process.argv[2], function (config, oldConfig) { +config.configFile(process.argv[2], function (config) { conf = config; process_mgmt.init(config); From be5c23404d7270e1d3d4342daa8c72dd3824316f Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Thu, 11 Sep 2014 18:41:35 -0400 Subject: [PATCH 011/207] fix typo in Gauges section --- docs/metric_types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metric_types.md b/docs/metric_types.md index 75b68d3c..e839cd07 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -82,7 +82,7 @@ StatsD now also supports gauges, arbitrary values, which can be recorded. gaugor:333|g If the gauge is not updated at the next flush, it will send the previous value. You can opt to send -no metric at all for this gauge, by setting `config.deleteGauge` +no metric at all for this gauge, by setting `config.deleteGauges` Adding a sign to the gauge value will change the value, rather than setting it. From 88c8fce6a8b056ca9fd0173a8700f6db3be01e93 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 21:33:32 +0000 Subject: [PATCH 012/207] Convert listen server to loadable module, defaulting to udp --- servers/udp.js | 8 ++++++++ stats.js | 28 +++++++++++++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 servers/udp.js diff --git a/servers/udp.js b/servers/udp.js new file mode 100644 index 00000000..b03bc32b --- /dev/null +++ b/servers/udp.js @@ -0,0 +1,8 @@ +var dgram = require('dgram'); + +exports.init = function(config, callback){ + var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + var server = dgram.createSocket(udp_version, callback); + server.bind(config.port || 8125, config.address || undefined); + return true; +}; diff --git a/stats.js b/stats.js index bea7a07e..3a615f73 100644 --- a/stats.js +++ b/stats.js @@ -1,7 +1,6 @@ /*jshint node:true, laxcomma:true */ -var dgram = require('dgram') - , util = require('util') +var util = require('util') , net = require('net') , config = require('./lib/config') , helpers = require('./lib/helpers') @@ -24,7 +23,7 @@ var sets = {}; var counter_rates = {}; var timer_data = {}; var pctThreshold = null; -var flushInterval, keyFlushInt, server, mgmtServer; +var flushInterval, keyFlushInt, serverLoaded, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; @@ -46,6 +45,21 @@ function loadBackend(config, name) { } } +// Load and init the server from the servers/ directory. +function startServer(config, name, callback) { + var servermod = require(name); + + if (config.debug) { + l.log("Loading server: " + name, 'DEBUG'); + } + + var ret = servermod.init(config, callback); + if (!ret) { + l.log("Failed to load server: " + name); + process.exit(1); + } +} + // global for conf var conf; @@ -162,13 +176,14 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (server === undefined) { + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); - var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; - server = dgram.createSocket(udp_version, function (msg, rinfo) { + // The default server is UDP + var server = config.server || './servers/udp' + serverLoaded = startServer(config, server, function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; var packet_data = msg.toString(); @@ -355,7 +370,6 @@ config.configFile(process.argv[2], function (config) { }); }); - server.bind(config.port || 8125, config.address || undefined); mgmtServer.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); util.log("server is up"); From 9eab749f782907fcfbcecfc224851ab3ddefa3a7 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 21:33:54 +0000 Subject: [PATCH 013/207] Add tcp loadable server module --- servers/tcp.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 servers/tcp.js diff --git a/servers/tcp.js b/servers/tcp.js new file mode 100644 index 00000000..e094a327 --- /dev/null +++ b/servers/tcp.js @@ -0,0 +1,16 @@ +var net = require('net'); + +exports.init = function(config, callback){ + var server = net.createServer(function(stream) { + stream.setEncoding('ascii'); + + stream.on('data', function(data) { + var rinfo = stream.address(); + rinfo.size = data.length; + callback(data, rinfo); + }); + }); + + server.listen(config.port || 8125, config.address || undefined); + return true; +}; From 15bf64ad43b8a43da68b139cadb5e3b7b6cf74be Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 23:28:40 +0000 Subject: [PATCH 014/207] Represent remote address in tcp server rinfo, not local --- servers/tcp.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index e094a327..c1ee3af7 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -1,13 +1,18 @@ var net = require('net'); +function rinfo(tcpstream, data) { + this.address = tcpstream.remoteAddress; + this.port = tcpstream.remotePort; + this.family = tcpstream.address().family; + this.size = data.length; +} + exports.init = function(config, callback){ var server = net.createServer(function(stream) { stream.setEncoding('ascii'); stream.on('data', function(data) { - var rinfo = stream.address(); - rinfo.size = data.length; - callback(data, rinfo); + callback(data, new rinfo(stream, data)); }); }); From f2102aeb7130c3dc378d0431da499568b62aac12 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:26:47 +0000 Subject: [PATCH 015/207] Rename server init to 'start' to reflect reality --- servers/tcp.js | 2 +- servers/udp.js | 2 +- stats.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index c1ee3af7..3c2e651b 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -7,7 +7,7 @@ function rinfo(tcpstream, data) { this.size = data.length; } -exports.init = function(config, callback){ +exports.start = function(config, callback){ var server = net.createServer(function(stream) { stream.setEncoding('ascii'); diff --git a/servers/udp.js b/servers/udp.js index b03bc32b..a0a3072b 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,6 +1,6 @@ var dgram = require('dgram'); -exports.init = function(config, callback){ +exports.start = function(config, callback){ var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; var server = dgram.createSocket(udp_version, callback); server.bind(config.port || 8125, config.address || undefined); diff --git a/stats.js b/stats.js index 3a615f73..280f3eb7 100644 --- a/stats.js +++ b/stats.js @@ -53,7 +53,7 @@ function startServer(config, name, callback) { l.log("Loading server: " + name, 'DEBUG'); } - var ret = servermod.init(config, callback); + var ret = servermod.start(config, callback); if (!ret) { l.log("Failed to load server: " + name); process.exit(1); From 6f7f566751a0a52594a918812b447c02ea1c6b79 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:27:54 +0000 Subject: [PATCH 016/207] Add basic test for each of udp and tcp server module --- test/server_tests.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 test/server_tests.js diff --git a/test/server_tests.js b/test/server_tests.js new file mode 100644 index 00000000..b426ba92 --- /dev/null +++ b/test/server_tests.js @@ -0,0 +1,42 @@ +var dgram = require('dgram'), + net = require('net'); + +var config = { + address: '127.0.0.1', + port: 8125 +}; +var msg = "This is a test\r\n"; + +module.exports = { + udp_data_received: function(test) { + test.expect(3); + var server = require('../servers/udp'); + var started = server.start(config, function(data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + test.done(); + }); + test.ok(started); + + var buf = new Buffer(msg); + var sock = dgram.createSocket('udp4'); + sock.send(buf, 0, buf.length, config.port, config.address, function(err, bytes) { + sock.close(); + }); + }, + tcp_data_received: function(test) { + test.expect(3); + var server = require('../servers/tcp'); + var started = server.start(config, function(data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + test.done(); + }); + test.ok(started); + + var client = net.connect(config.port, config.address, function() { + client.write(msg); + client.end(); + }); + } +} From 6984d5d37dede44aaa7b06c2b46f74bc0d1ebfa7 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:35:26 +0000 Subject: [PATCH 017/207] Update docs and example config to reflect new tcp server option --- README.md | 9 +++++---- exampleConfig.js | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9f14b964..74b129ca 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] ====== A network daemon that runs on the [Node.js][node] platform and -listens for statistics, like counters and timers, sent over [UDP][udp] -and sends aggregates to one or more pluggable backend services (e.g., +listens for statistics, like counters and timers, sent over [UDP][udp] or +[TCP][tcp] and sends aggregates to one or more pluggable backend services (e.g., [Graphite][graphite]). We ([Etsy][etsy]) [blogged][blog post] about how it works and why we created it. @@ -46,12 +46,12 @@ Installation and Configuration Usage ------- -The basic line protocol expects metrics to be sent via UDP in the format: +The basic line protocol expects metrics to be sent in the format: :| So the simplest way to send in metrics from your command line if you have -StatsD running on localhost would be: +StatsD running with the default UDP server on localhost would be: echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 @@ -104,6 +104,7 @@ Meta [counting-timing]: http://code.flickr.com/blog/2008/10/27/counting-timing/ [Flicker-StatsD]: https://github.com/iamcal/Flickr-StatsD [udp]: http://en.wikipedia.org/wiki/User_Datagram_Protocol +[tcp]: http://en.wikipedia.org/wiki/Transmission_Control_Protocol [docs_metric_types]: https://github.com/etsy/statsd/blob/master/docs/metric_types.md [docs_graphite]: https://github.com/etsy/statsd/blob/master/docs/graphite.md [docs_backend]: https://github.com/etsy/statsd/blob/master/docs/backend.md diff --git a/exampleConfig.js b/exampleConfig.js index 174dd094..b0053a59 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -15,10 +15,14 @@ Optional Variables: the default graphite backend will be loaded. * example for console and graphite: [ "./backends/console", "./backends/graphite" ] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + * example for tcp server: + "./servers/tcp" debug: debug flag [default: false] - address: address to listen on over UDP [default: 0.0.0.0] + address: address to listen on [default: 0.0.0.0] address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on over UDP [default: 8125] + port: port to listen for messages on [default: 8125] mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] mgmt_port: port to run the management TCP interface on [default: 8126] From f630e15f04451ade70b5c028796344e1b23debd6 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 21:16:11 +0000 Subject: [PATCH 018/207] Add brief docs around server callback function sig --- stats.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stats.js b/stats.js index 280f3eb7..eb57671f 100644 --- a/stats.js +++ b/stats.js @@ -46,6 +46,10 @@ function loadBackend(config, name) { } // Load and init the server from the servers/ directory. +// The callback mimics the dgram 'message' event parameters (msg, rinfo) +// msg: the message received by the server. may contain more than one metric +// rinfo: contains remote address information and message length +// (attributes are .address, .port, .family, .size - you're welcome) function startServer(config, name, callback) { var servermod = require(name); From 4e7a60049e70b7c227f833d251f913d6e1edf8f7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 17 Sep 2014 10:04:39 -0700 Subject: [PATCH 019/207] Add servers directory to debian package --- debian/statsd.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/statsd.install b/debian/statsd.install index 174d59e2..21b225b5 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -2,4 +2,5 @@ stats.js /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends lib/*.js /usr/share/statsd/lib +servers/*.js /usr/share/statsd/servers debian/localConfig.js /etc/statsd From c7e6421d1e7b284a94372acf455a2e18b4d997a8 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Mon, 24 Sep 2012 14:12:13 -0400 Subject: [PATCH 020/207] Stat key name sanitization is now configurable at the top-level. Setting keyNameSanitize to false pushes the requirement of sanitizing key names to the backends. This permits backends that have less strict character set requirements to take advantage of an expanded stat name character set. The default behavior remains the same as collisions in key name space are not handled if two different stat names map to the same sanitized key name. --- backends/graphite.js | 28 ++++++++-- config.js | 113 +++++++++++++++++++++++++++++++++++++++++ exampleConfig.js | 3 ++ stats.js | 21 ++++++-- test/graphite_tests.js | 19 +++++++ test/stam | 1 + 6 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 config.js create mode 100644 test/stam diff --git a/backends/graphite.js b/backends/graphite.js index 34cd299d..09e7b48f 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,6 +33,7 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -97,15 +98,27 @@ var flush_stats = function graphite_flush(ts, metrics) { var timer_data = metrics.timer_data; var statsd_metrics = metrics.statsd_metrics; + // Sanitize key for graphite if not done globally + function sk(key) { + if (globalKeySanitize) { + return key; + } else { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } + }; + for (key in counters) { - var namespace = counterNamespace.concat(key); var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate + var keyName = sk(key); + var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; if (flush_counts) { - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + statString += 'stats_counts.' + keyName + globalSuffix + value + ts_suffix; } } else { statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; @@ -118,8 +131,9 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(key); + var namespace = timerNamespace.concat(sk(key)); var the_key = namespace.join("."); + for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; @@ -137,13 +151,13 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(key); + var namespace = gaugesNamespace.concat(sk(key)); statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(key); + var namespace = setsNamespace.concat(sk(key)); statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; numStats += 1; } @@ -239,6 +253,10 @@ exports.init = function graphite_init(startup_time, config, events, logger) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/config.js b/config.js new file mode 100644 index 00000000..7e03fe4d --- /dev/null +++ b/config.js @@ -0,0 +1,113 @@ +/* +Graphite Required Variables: + +(Leave these unset to avoid sending stats to Graphite. + Set debug flag and leave these unset to run in 'dry' debug mode - + useful for testing statsd clients without a Graphite server.) + + graphiteHost: hostname or IP of Graphite server + graphitePort: port of Graphite server + +Optional Variables: + + backends: an array of backends to load. Each backend must exist + by name in the directory backends/. If not specified, + the default graphite backend will be loaded. + * example for console and graphite: + [ "./backends/console", "./backends/graphite" ] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + * example for tcp server: + "./servers/tcp" + debug: debug flag [default: false] + address: address to listen on [default: 0.0.0.0] + address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] + port: port to listen for messages on [default: 8125] + mgmt_address: address to run the management TCP interface on + [default: 0.0.0.0] + mgmt_port: port to run the management TCP interface on [default: 8126] + title: Allows for overriding the process title. [default: statsd] + if set to false, will not override the process title and let the OS set it. + The length of the title has to be less than or equal to the binary name + cli arguments + NOTE: This does not work on Mac's with node versions prior to v0.10 + + healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] + dumpMessages: log all incoming messages + flushInterval: interval (in ms) to flush metrics to each backend + percentThreshold: for time information, calculate the Nth percentile(s) + (can be a single value or list of floating-point values) + negative values mean to use "top" Nth percentile(s) values + [%, default: 90] + flush_counts: send stats_counts metrics [default: true] + + keyFlush: log the most frequently sent keys [object, default: undefined] + interval: how often to log frequent keys [ms, default: 0] + percent: percentage of frequent keys to log [%, default: 100] + log: location of log file for frequent keys [default: STDOUT] + deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers + as opposed to sending 0. For gauges, this unsets the gauge (instead of sending + the previous value). Can be individually overriden. [default: false] + deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] + deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] + deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] + deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] + prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] + applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. + + console: + prettyprint: whether to prettyprint the console backend + output [true or false, default: true] + + log: log settings [object, default: undefined] + backend: where to log: stdout or syslog [string, default: stdout] + application: name of the application for syslog [string, default: statsd] + level: log level for [node-]syslog [string, default: LOG_INFO] + + graphite: + legacyNamespace: use the legacy namespace [default: true] + globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] + prefixCounter: graphite prefix for counter metrics [default: "counters"] + prefixTimer: graphite prefix for timer metrics [default: "timers"] + prefixGauge: graphite prefix for gauge metrics [default: "gauges"] + prefixSet: graphite prefix for set metrics [default: "sets"] + globalSuffix: global suffix to use for sending stats to graphite [default: ""] + This is particularly useful for sending per host stats by + settings this value to: require('os').hostname().split('.')[0] + + repeater: an array of hashes of the for host: and port: + that details other statsd servers to which the received + packets should be "repeated" (duplicated to). + e.g. [ { host: '10.10.10.10', port: 8125 }, + { host: 'observer', port: 88125 } ] + + repeaterProtocol: whether to use udp4 or udp6 for repeaters. + ["udp4" or "udp6", default: "udp4"] + + histogram: for timers, an array of mappings of strings (to match metrics) and + corresponding ordered non-inclusive upper limits of bins. + For all matching metrics, histograms are maintained over + time by writing the frequencies for all bins. + 'inf' means infinity. A lower limit of 0 is assumed. + default: [], meaning no histograms for any timer. + First match wins. examples: + * histogram to only track render durations, with unequal + class intervals and catchall for outliers: + [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] + * histogram for all timers except 'foo' related, + equal class interval and catchall for outliers: + [ { metric: 'foo', bins: [] }, + { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] + + automaticConfigReload: whether to watch the config file and reload it when it + changes. The default is true. Set this to false to disable. +*/ +{ + graphitePort: 2003 +, graphiteHost: "127.0.0.1" +, port: 8125 +, keyNameSanitize: false +, backends: [ "./backends/graphite", "./backends/console" ] +} diff --git a/exampleConfig.js b/exampleConfig.js index b0053a59..598f2ec3 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,6 +53,9 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index eb57671f..b8762b6d 100644 --- a/stats.js +++ b/stats.js @@ -29,6 +29,7 @@ var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; var old_timestamp = 0; var timestamp_lag_namespace; +var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -158,6 +159,16 @@ var stats = { } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } else { + return key; + } +} + // Global for the logger var l; @@ -180,8 +191,11 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (!serverLoaded) { + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); @@ -205,10 +219,7 @@ config.configFile(process.argv[2], function (config) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = bits.shift() - .replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + var key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { diff --git a/test/graphite_tests.js b/test/graphite_tests.js index a034f316..971af4a5 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -358,5 +358,24 @@ module.exports = { }); }); }); + }, + + metric_names_are_sanitized: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); + test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); + test.ok(str.indexOf('foobar') !== -1, "Did not map 'foo+bar' => 'foobar'"); + test.done(); + }); + }); + }); + }); + }); } } diff --git a/test/stam b/test/stam new file mode 100644 index 00000000..6b1f07d5 --- /dev/null +++ b/test/stam @@ -0,0 +1 @@ +stats.counters.statsd.bad_lines_seen.rate 0 1411387496,stats.counters.statsd.bad_lines_seen.count 0 1411387496,stats.counters.statsd.packets_received.rate 15 1411387496,stats.counters.statsd.packets_received.count 3 1411387496,stats.counters.fo-o.rate 1250 1411387496,stats.counters.fo-o.count 250 1411387496,stats.counters.b_ar.rate 1250 1411387496,stats.counters.b_ar.count 250 1411387496,stats.counters.foobar.rate 1250 1411387496,stats.counters.foobar.count 250 1411387496,stats.gauges.statsd.timestamp_lag -0.2 1411387496,stats.statsd.numStats 6 1411387496,stats.statsd.graphiteStats.calculationtime 0 1411387496,stats.statsd.processing_time 0 1411387496,stats.statsd.graphiteStats.last_exception 1411387495 1411387496,stats.statsd.graphiteStats.last_flush 1411387496 1411387496,stats.statsd.graphiteStats.flush_time 1 1411387496,stats.statsd.graphiteStats.flush_length 587 1411387496, \ No newline at end of file From 55be359068e80d8d7635c14219fa856801edfeca Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:47:06 +0300 Subject: [PATCH 021/207] merge from PR155 --- config.js | 113 ------------------------------------------------------ test/stam | 1 - 2 files changed, 114 deletions(-) delete mode 100644 config.js delete mode 100644 test/stam diff --git a/config.js b/config.js deleted file mode 100644 index 7e03fe4d..00000000 --- a/config.js +++ /dev/null @@ -1,113 +0,0 @@ -/* -Graphite Required Variables: - -(Leave these unset to avoid sending stats to Graphite. - Set debug flag and leave these unset to run in 'dry' debug mode - - useful for testing statsd clients without a Graphite server.) - - graphiteHost: hostname or IP of Graphite server - graphitePort: port of Graphite server - -Optional Variables: - - backends: an array of backends to load. Each backend must exist - by name in the directory backends/. If not specified, - the default graphite backend will be loaded. - * example for console and graphite: - [ "./backends/console", "./backends/graphite" ] - server: the server to load. The server must exist by name in the directory - servers/. If not specified, the default udp server will be loaded. - * example for tcp server: - "./servers/tcp" - debug: debug flag [default: false] - address: address to listen on [default: 0.0.0.0] - address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on [default: 8125] - mgmt_address: address to run the management TCP interface on - [default: 0.0.0.0] - mgmt_port: port to run the management TCP interface on [default: 8126] - title: Allows for overriding the process title. [default: statsd] - if set to false, will not override the process title and let the OS set it. - The length of the title has to be less than or equal to the binary name + cli arguments - NOTE: This does not work on Mac's with node versions prior to v0.10 - - healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] - dumpMessages: log all incoming messages - flushInterval: interval (in ms) to flush metrics to each backend - percentThreshold: for time information, calculate the Nth percentile(s) - (can be a single value or list of floating-point values) - negative values mean to use "top" Nth percentile(s) values - [%, default: 90] - flush_counts: send stats_counts metrics [default: true] - - keyFlush: log the most frequently sent keys [object, default: undefined] - interval: how often to log frequent keys [ms, default: 0] - percent: percentage of frequent keys to log [%, default: 100] - log: location of log file for frequent keys [default: STDOUT] - deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers - as opposed to sending 0. For gauges, this unsets the gauge (instead of sending - the previous value). Can be individually overriden. [default: false] - deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] - deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] - deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] - deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] - prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] - applies to both legacy and new namespacing - keyNameSanitize: sanitize all stat names on ingress [default: true] - If disabled, it is up to the backends to sanitize keynames - as appropriate per their storage requirements. - - console: - prettyprint: whether to prettyprint the console backend - output [true or false, default: true] - - log: log settings [object, default: undefined] - backend: where to log: stdout or syslog [string, default: stdout] - application: name of the application for syslog [string, default: statsd] - level: log level for [node-]syslog [string, default: LOG_INFO] - - graphite: - legacyNamespace: use the legacy namespace [default: true] - globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] - prefixCounter: graphite prefix for counter metrics [default: "counters"] - prefixTimer: graphite prefix for timer metrics [default: "timers"] - prefixGauge: graphite prefix for gauge metrics [default: "gauges"] - prefixSet: graphite prefix for set metrics [default: "sets"] - globalSuffix: global suffix to use for sending stats to graphite [default: ""] - This is particularly useful for sending per host stats by - settings this value to: require('os').hostname().split('.')[0] - - repeater: an array of hashes of the for host: and port: - that details other statsd servers to which the received - packets should be "repeated" (duplicated to). - e.g. [ { host: '10.10.10.10', port: 8125 }, - { host: 'observer', port: 88125 } ] - - repeaterProtocol: whether to use udp4 or udp6 for repeaters. - ["udp4" or "udp6", default: "udp4"] - - histogram: for timers, an array of mappings of strings (to match metrics) and - corresponding ordered non-inclusive upper limits of bins. - For all matching metrics, histograms are maintained over - time by writing the frequencies for all bins. - 'inf' means infinity. A lower limit of 0 is assumed. - default: [], meaning no histograms for any timer. - First match wins. examples: - * histogram to only track render durations, with unequal - class intervals and catchall for outliers: - [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] - * histogram for all timers except 'foo' related, - equal class interval and catchall for outliers: - [ { metric: 'foo', bins: [] }, - { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] - - automaticConfigReload: whether to watch the config file and reload it when it - changes. The default is true. Set this to false to disable. -*/ -{ - graphitePort: 2003 -, graphiteHost: "127.0.0.1" -, port: 8125 -, keyNameSanitize: false -, backends: [ "./backends/graphite", "./backends/console" ] -} diff --git a/test/stam b/test/stam deleted file mode 100644 index 6b1f07d5..00000000 --- a/test/stam +++ /dev/null @@ -1 +0,0 @@ -stats.counters.statsd.bad_lines_seen.rate 0 1411387496,stats.counters.statsd.bad_lines_seen.count 0 1411387496,stats.counters.statsd.packets_received.rate 15 1411387496,stats.counters.statsd.packets_received.count 3 1411387496,stats.counters.fo-o.rate 1250 1411387496,stats.counters.fo-o.count 250 1411387496,stats.counters.b_ar.rate 1250 1411387496,stats.counters.b_ar.count 250 1411387496,stats.counters.foobar.rate 1250 1411387496,stats.counters.foobar.count 250 1411387496,stats.gauges.statsd.timestamp_lag -0.2 1411387496,stats.statsd.numStats 6 1411387496,stats.statsd.graphiteStats.calculationtime 0 1411387496,stats.statsd.processing_time 0 1411387496,stats.statsd.graphiteStats.last_exception 1411387495 1411387496,stats.statsd.graphiteStats.last_flush 1411387496 1411387496,stats.statsd.graphiteStats.flush_time 1 1411387496,stats.statsd.graphiteStats.flush_length 587 1411387496, \ No newline at end of file From 0eb2517b04d645fe58a5d65c32b6f63a0eca9658 Mon Sep 17 00:00:00 2001 From: shaylang Date: Tue, 23 Sep 2014 18:43:08 +0300 Subject: [PATCH 022/207] Metric name sanitization moved to graphite backend. it is now the responsibility of each backend developer to take care for sanitization --- backends/graphite.js | 9 --------- exampleConfig.js | 3 --- stats.js | 16 +--------------- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 09e7b48f..20b73f03 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,7 +33,6 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; -var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -100,13 +99,9 @@ var flush_stats = function graphite_flush(ts, metrics) { // Sanitize key for graphite if not done globally function sk(key) { - if (globalKeySanitize) { - return key; - } else { return key.replace(/\s+/g, '_') .replace(/\//g, '-') .replace(/[^a-zA-Z_\-0-9\.]/g, ''); - } }; for (key in counters) { @@ -253,10 +248,6 @@ exports.init = function graphite_init(startup_time, config, events, logger) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; - if (config.keyNameSanitize !== undefined) { - globalKeySanitize = config.keyNameSanitize; - } - flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/exampleConfig.js b/exampleConfig.js index 598f2ec3..b0053a59 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,9 +53,6 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing - keyNameSanitize: sanitize all stat names on ingress [default: true] - If disabled, it is up to the backends to sanitize keynames - as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index e515afea..4f9f21c3 100644 --- a/stats.js +++ b/stats.js @@ -29,7 +29,6 @@ var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; var old_timestamp = 0; var timestamp_lag_namespace; -var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -159,16 +158,6 @@ var stats = { } }; -function sanitizeKeyName(key) { - if (keyNameSanitize) { - return key.replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); - } else { - return key; - } -} - // Global for the logger var l; @@ -191,9 +180,6 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (config.keyNameSanitize !== undefined) { - keyNameSanitize = config.keyNameSanitize; - } if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); @@ -218,7 +204,7 @@ config.configFile(process.argv[2], function (config) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = sanitizeKeyName(bits.shift()); + var key = bits.shift(); if (keyFlushInterval > 0) { if (! keyCounter[key]) { From b7b8c9a92dd7dbc09c1df61ebd2565f8bc01d7cc Mon Sep 17 00:00:00 2001 From: Wagner Date: Wed, 1 Oct 2014 18:21:01 -0300 Subject: [PATCH 023/207] Adding proxy.js to debian package --- debian/statsd.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/statsd.install b/debian/statsd.install index 21b225b5..1a97930f 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -1,4 +1,5 @@ stats.js /usr/share/statsd +proxy.js /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends lib/*.js /usr/share/statsd/lib From 115103b333e34887203ea34183bbb327fa5d14cb Mon Sep 17 00:00:00 2001 From: Wagner Date: Wed, 1 Oct 2014 18:21:19 -0300 Subject: [PATCH 024/207] Update statsd.install --- debian/statsd.install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/statsd.install b/debian/statsd.install index 1a97930f..6f986858 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -1,4 +1,4 @@ -stats.js /usr/share/statsd +stats.js /usr/share/statsd proxy.js /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends From 8d042866beeb467a69c98d17cdf7e0adfd7890f1 Mon Sep 17 00:00:00 2001 From: Wagner Date: Wed, 1 Oct 2014 18:23:08 -0300 Subject: [PATCH 025/207] removing duplicated libs from debian package --- debian/statsd.install | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/statsd.install b/debian/statsd.install index 6f986858..15b9ddaf 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -2,6 +2,5 @@ stats.js /usr/share/statsd proxy.js /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends -lib/*.js /usr/share/statsd/lib servers/*.js /usr/share/statsd/servers debian/localConfig.js /etc/statsd From 8ddd61b34d52ad2fbb253c5bc2a38f58a509a678 Mon Sep 17 00:00:00 2001 From: Adam Dubiel Date: Mon, 27 Oct 2014 13:27:07 +0100 Subject: [PATCH 026/207] Validate negative sampling --- lib/helpers.js | 13 +++++++++++-- test/helpers_tests.js | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index 98c6225a..52ba3a3f 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -11,6 +11,15 @@ function isNumber(str) { return Boolean(str && !isNaN(str)); } +function isValidSampleRate(str) { + var validSampleRate = false; + if(str.length > 1 && str[0] === '@') { + var numberStr = str.substring(1); + validSampleRate = isNumber(numberStr) && numberStr[0] != '-'; + } + return validSampleRate; +} + function is_valid_packet(fields) { // test for existing metrics type @@ -19,8 +28,8 @@ function is_valid_packet(fields) { } // filter out malformed sample rates - if (fields[2] !== undefined) { - if (fields[2].length <= 1 || fields[2][0] != '@' || !isNumber(fields[2].substring(1))) { + if(fields[2] !== undefined) { + if(!isValidSampleRate(fields[2])) { return false; } } diff --git a/test/helpers_tests.js b/test/helpers_tests.js index d10da0ba..cb769842 100644 --- a/test/helpers_tests.js +++ b/test/helpers_tests.js @@ -112,6 +112,7 @@ module.exports = { test.equals(helpers.is_valid_packet(['345345', 'ms', '@.']), false); test.equals(helpers.is_valid_packet(['345345', 'ms', '@.1.']), false); test.equals(helpers.is_valid_packet(['345345', 'ms', '@.1.2.3']), false); + test.equals(helpers.is_valid_packet(['345345', 'ms', '@-1.0']), false); test.done(); }, From e74872cb91318020124e24c25f71b15ea663f18b Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 1 Oct 2014 20:14:15 -0700 Subject: [PATCH 027/207] Fix prettyprint option for console backend Pretty was using an invalid invocation of util.inspect(). Perhaps it was valid pre-v0.10, but inspect now takes an options object. --- backends/console.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/console.js b/backends/console.js index b3a6065a..e435aa59 100644 --- a/backends/console.js +++ b/backends/console.js @@ -33,7 +33,7 @@ ConsoleBackend.prototype.flush = function(timestamp, metrics) { }; if(this.config.prettyprint) { - console.log(util.inspect(out, false, 5, true)); + console.log(util.inspect(out, {depth: 5, colors: true})); } else { console.log(out); } From 3dadbd686ba730c6f9a1daafae4174fc9465ccb4 Mon Sep 17 00:00:00 2001 From: Chao Wang Date: Sat, 8 Nov 2014 20:01:00 +0800 Subject: [PATCH 028/207] Add node-bell to statsd backend list --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index c36ae8df..6806325f 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -32,6 +32,7 @@ queues and third-party services. ## Available Third-party backends - [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) +- [node-bell](https://github.com/eleme/node-bell) - [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) - [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) - [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) From d58948d784c3ff9f93926f46dcceefd967243c37 Mon Sep 17 00:00:00 2001 From: Cyril Rohr Date: Wed, 26 Mar 2014 16:27:14 +0000 Subject: [PATCH 029/207] Enable packaging at https://packager.io/gh/pkgr/statsd --- .pkgr.yml | 8 ++++++++ packager/Procfile | 1 + packager/postinst | 16 ++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 .pkgr.yml create mode 100644 packager/Procfile create mode 100755 packager/postinst diff --git a/.pkgr.yml b/.pkgr.yml new file mode 100644 index 00000000..b4acb42b --- /dev/null +++ b/.pkgr.yml @@ -0,0 +1,8 @@ +targets: + ubuntu-14.04: + ubuntu-12.04: + debian-7: + centos-6: +before: + - mv packager/Procfile . +after_install: ./packager/postinst diff --git a/packager/Procfile b/packager/Procfile new file mode 100644 index 00000000..64e1781e --- /dev/null +++ b/packager/Procfile @@ -0,0 +1 @@ +web: node stats.js ${STATSD_CONFIG:="config.js"} diff --git a/packager/postinst b/packager/postinst new file mode 100755 index 00000000..6b5ade03 --- /dev/null +++ b/packager/postinst @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +APP_NAME="statsd" +CLI="$APP_NAME" +APP_USER=$(${CLI} config:get APP_USER) +APP_GROUP=$(${CLI} config:get APP_GROUP) +APP_CONFIG="/etc/${APP_NAME}/config.js" + +[ -f "$APP_CONFIG" ] || cp /opt/${APP_NAME}/exampleConfig.js $APP_CONFIG +chown $APP_USER.$APP_GROUP $APP_CONFIG +ln -f -s $APP_CONFIG /opt/$APP_NAME/config.js +chmod 0640 $APP_CONFIG + +${CLI} scale web=1 || true From 96fcefeb97ff9189a8047eec9cb10685a07f99a7 Mon Sep 17 00:00:00 2001 From: Callum Macdonald Date: Mon, 29 Dec 2014 17:43:53 +0100 Subject: [PATCH 030/207] Added InfluxDB (assuming alphabetical order) --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index c36ae8df..81f432a4 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -36,6 +36,7 @@ queues and third-party services. - [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) - [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) - [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) +- [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) - [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) - [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) - [librato-backend](https://github.com/librato/statsd-librato-backend) From a4273e179694029ab29feef5fe161ff6ac2cf8bf Mon Sep 17 00:00:00 2001 From: Yahong Gu Date: Mon, 26 Jan 2015 09:05:14 -0500 Subject: [PATCH 031/207] add netuitive backend to backends.md --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 81f432a4..e27130c3 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -42,6 +42,7 @@ queues and third-party services. - [librato-backend](https://github.com/librato/statsd-librato-backend) - [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) - [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) +- [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) - [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) - [socket.io-backend](https://github.com/Chatham/statsd-socket.io) - [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) From 917a3e4276daf2587c30fcfcce65b0066e776626 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Fri, 6 Feb 2015 19:00:57 +0000 Subject: [PATCH 032/207] Backend key name sanitization can be controlled with config flag This reverts commit 0eb2517b04d645fe58a5d65c32b6f63a0eca9658. --- backends/graphite.js | 9 +++++++++ exampleConfig.js | 3 +++ stats.js | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index 20b73f03..09e7b48f 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,6 +33,7 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -99,9 +100,13 @@ var flush_stats = function graphite_flush(ts, metrics) { // Sanitize key for graphite if not done globally function sk(key) { + if (globalKeySanitize) { + return key; + } else { return key.replace(/\s+/g, '_') .replace(/\//g, '-') .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } }; for (key in counters) { @@ -248,6 +253,10 @@ exports.init = function graphite_init(startup_time, config, events, logger) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/exampleConfig.js b/exampleConfig.js index b0053a59..598f2ec3 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,6 +53,9 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index 1cd73038..9758a285 100644 --- a/stats.js +++ b/stats.js @@ -29,6 +29,7 @@ var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; var old_timestamp = 0; var timestamp_lag_namespace; +var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -160,6 +161,16 @@ var stats = { } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } else { + return key; + } +} + // Global for the logger var l; @@ -184,6 +195,9 @@ config.configFile(process.argv[2], function (config) { counters[packets_received] = 0; counters[metrics_received] = 0; + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); @@ -210,7 +224,7 @@ config.configFile(process.argv[2], function (config) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = bits.shift(); + var key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { From 80a7166ccbd6fbfe700adfe34255fc8913d52f4a Mon Sep 17 00:00:00 2001 From: sysadminmike Date: Fri, 6 Mar 2015 16:01:28 +0000 Subject: [PATCH 033/207] add couchdb to backends --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 04c75252..7039ec55 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -33,6 +33,7 @@ queues and third-party services. ## Available Third-party backends - [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) - [node-bell](https://github.com/eleme/node-bell) +- [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) - [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) - [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) - [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) From 40647cb6e474e3c47e27bf8944e421647286b6dd Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 6 Mar 2015 11:18:08 -0500 Subject: [PATCH 034/207] update hashring dependency to make it work on v0.12 related to #492 and #496 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 62482098..751c26b9 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "optionalDependencies": { "node-syslog":"1.1.7", - "hashring":"1.0.1", + "hashring":"3.1.0", "winser": "=0.1.6" }, "engines": { From 4a555df3c4f8abde146d154408f856423b049f1f Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 6 Mar 2015 11:19:12 -0500 Subject: [PATCH 035/207] use a node-syslog version that compiles on v0.12 related to #496 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 751c26b9..4a91b582 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "temp": "0.4.x" }, "optionalDependencies": { - "node-syslog":"1.1.7", + "node-syslog":"schamane/node-syslog#30d44555", "hashring":"3.1.0", "winser": "=0.1.6" }, From 20e353840528104750729bd8008a418a5b595b6a Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 6 Mar 2015 11:19:59 -0500 Subject: [PATCH 036/207] remove support for v0.8.x fixes #494 and #490 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7cb41cb5..6aef008c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: node_js node_js: -- '0.8' - '0.10' script: ./run_tests.sh notifications: From fd705016fa78a5c40bc143b3cac7a2081568b61f Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 6 Mar 2015 11:22:19 -0500 Subject: [PATCH 037/207] run tests on v0.12.x fixes #496 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6aef008c..2ff67195 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: node_js node_js: - '0.10' +- '0.12' script: ./run_tests.sh notifications: email: false From 9d92f894700c7a49d2ef5e6afb67512198da7f13 Mon Sep 17 00:00:00 2001 From: tihonove Date: Fri, 3 Apr 2015 13:17:48 +0500 Subject: [PATCH 038/207] remove readFileSync inside readFile while reading config --- lib/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config.js b/lib/config.js index b61bf3b7..d537f61e 100644 --- a/lib/config.js +++ b/lib/config.js @@ -16,7 +16,7 @@ var Configurator = function (file) { if (err) { throw err; } old_config = self.config; - self.config = eval('config = ' + fs.readFileSync(file)); + self.config = eval('config = ' + data); self.emit('configChanged', self.config); }); }; From a07296c018c57f7c357cb7a7b93faac389a65c6b Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sun, 12 Apr 2015 11:00:45 -0400 Subject: [PATCH 039/207] add license information into package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 4a91b582..81cc1ea5 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "statsd", "description": "A simple, lightweight network daemon to collect metrics over UDP", "author": "Etsy", + "license" : "MIT", "scripts": { "test": "./run_tests.sh", "start": "node stats.js config.js", From ed9ac23b1173fe9bd7d33ba8ec7c337ff99efff1 Mon Sep 17 00:00:00 2001 From: Joseph Hughes Date: Thu, 16 Apr 2015 13:05:23 -0600 Subject: [PATCH 040/207] update package --- debian/changelog | 11 +++ debian/control | 2 +- debian/postinst | 6 +- debian/proxyConfig.js | 11 +++ debian/statsd.init | 169 ------------------------------------ debian/statsd.install | 2 + debian/statsd_proxy.upstart | 28 ++++++ 7 files changed, 55 insertions(+), 174 deletions(-) create mode 100644 debian/proxyConfig.js delete mode 100644 debian/statsd.init create mode 100644 debian/statsd_proxy.upstart diff --git a/debian/changelog b/debian/changelog index a1340bd1..3aa0e8e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +statsd (0.7.2) unstable; urgency=low + + * Align version number to git tag versions + * Add statsd_proxy upstart + * Install node dependencies on package installation + * Add proxyConfig.js + * Remove init script that does not work in favor of just having upstart + * Fix node version dependency + +-- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 + statsd (0.6.0-1) unstable; urgency=low * Non-maintainer upload. diff --git a/debian/control b/debian/control index 53dff2d2..fdedea94 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 8.0.0) Package: statsd Architecture: all -Depends: nodejs (>= 0.6), adduser +Depends: nodejs (>= 0.8), adduser Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. diff --git a/debian/postinst b/debian/postinst index e92c3698..ed3b4bba 100755 --- a/debian/postinst +++ b/debian/postinst @@ -4,6 +4,8 @@ set -e if [ "$1" = configure ]; then # Automatically added by dh_installinit + (cd /usr/share/statsd && /usr/bin/npm install) + if ! getent passwd _statsd > /dev/null; then adduser --system --quiet --home /nonexistent --no-create-home \ --shell /bin/false --force-badname --group --gecos "StatsD User" _statsd @@ -13,8 +15,4 @@ if [ "$1" = configure ]; then dpkg-statoverride --update --add _statsd _statsd 0755 /var/run/statsd fi - if [ -x "/etc/init.d/statsd" ]; then - update-rc.d statsd defaults >/dev/null - invoke-rc.d statsd start || exit $? - fi fi diff --git a/debian/proxyConfig.js b/debian/proxyConfig.js new file mode 100644 index 00000000..b1ab9444 --- /dev/null +++ b/debian/proxyConfig.js @@ -0,0 +1,11 @@ +{ +nodes: [ +{host: 'localhost', port: 8125, adminport: 8126}, +], +udp_version: 'udp4', +host: '0.0.0.0', +port: 8127, +forkCount: 0, +checkInterval: 1000, +cacheSize: 10000 +} diff --git a/debian/statsd.init b/debian/statsd.init deleted file mode 100644 index 960aefea..00000000 --- a/debian/statsd.init +++ /dev/null @@ -1,169 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: statsd -# Required-Start: $remote_fs $network $local_fs -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -### END INIT INFO - -# Do NOT "set -e" - -PATH=$PATH:/usr/local/bin:/usr/bin:/bin -NODE_BIN=$(which nodejs||which node) - -if [ ! -x "$NODE_BIN" ]; then - echo "Can't find executable nodejs or node in PATH=$PATH" - exit 1 -fi - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="StatsD" -NAME=statsd -USER=_statsd -DAEMON=$NODE_BIN -DAEMON_ARGS="/usr/share/statsd/stats.js /etc/statsd/localConfig.js" -PIDFILE=/var/run/$NAME/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -CHDIR="/usr/share/statsd" - -# Exit if the package is not installed -# [ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. -. /lib/lsb/init-functions - -# Create PIDDIR on runtime -if [ ! -d /var/run/$NAME ]; -then - mkdir /var/run/$NAME - chown $USER /var/run/$NAME -fi - -# Create LOGDIR on runtime -mkdir -p /var/log/$NAME - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet -m --pidfile $PIDFILE --startas $DAEMON --chuid $USER:$USER --background --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet -m --pidfile $PIDFILE --startas $DAEMON --chuid $USER:$USER --background --no-close --chdir $CHDIR -- \ - $DAEMON_ARGS >> /var/log/$NAME/$NAME.log 2> /var/log/$NAME/stderr.log \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=0/0/KILL/5 --pidfile $PIDFILE - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON - [ "$?" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME - return 0 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - status) - status_of_proc -p $PIDFILE $DAEMON "$NAME" && exit 0 || exit $? - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|restart|status|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/debian/statsd.install b/debian/statsd.install index 15b9ddaf..7933ef02 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -1,6 +1,8 @@ stats.js /usr/share/statsd proxy.js /usr/share/statsd +package.json /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends servers/*.js /usr/share/statsd/servers debian/localConfig.js /etc/statsd +debian/proxyConfig.js /etc/statsd diff --git a/debian/statsd_proxy.upstart b/debian/statsd_proxy.upstart new file mode 100644 index 00000000..2a04d718 --- /dev/null +++ b/debian/statsd_proxy.upstart @@ -0,0 +1,28 @@ +# statsd - Network daemon for aggregating statistics +# +# This is a network service that receives metric data via UDP from other +# applications. It aggregates this data and flushes it to a storage backend +# (typically Graphite) at regular intervals. +# +description "Network daemon for aggregating statistics" +author "Etsy" + +start on (local-filesystems and net-device-up IFACE!=lo) + +setuid _statsd +setgid _statsd + +respawn +respawn limit 10 5 + +chdir /usr/share/statsd + +pre-start script + NODE_BIN=$(which nodejs || which node) + [ -n $NODE_BIN ] || { stop; exit 0; } +end script + +script + NODE_BIN=$(which nodejs || which node) + exec $NODE_BIN proxy.js /etc/statsd/proxyConfig.js +end script From e5137e6a98a62f8cb1a80c2f8849f99a32cf6402 Mon Sep 17 00:00:00 2001 From: Joseph Hughes Date: Thu, 16 Apr 2015 13:45:16 -0600 Subject: [PATCH 041/207] fix typo --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 3aa0e8e3..7101d071 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,7 +7,7 @@ statsd (0.7.2) unstable; urgency=low * Remove init script that does not work in favor of just having upstart * Fix node version dependency --- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 + -- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 statsd (0.6.0-1) unstable; urgency=low From 08554bdb5f7f2ec977624489021f479c117b279c Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Mon, 4 May 2015 12:12:44 +0100 Subject: [PATCH 042/207] Support loading multiple servers. Example config looks something like this: { servers: [ {server: './servers/tcp', address: '0.0.0.0', port: 6125} , {server: './servers/udp', port: 6125} ] } --- exampleConfig.js | 16 ++++++++++++---- stats.js | 20 ++++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/exampleConfig.js b/exampleConfig.js index 598f2ec3..2a3d2eaf 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -15,14 +15,22 @@ Optional Variables: the default graphite backend will be loaded. * example for console and graphite: [ "./backends/console", "./backends/graphite" ] - server: the server to load. The server must exist by name in the directory + + servers: an array of server configurations. + If not specified, the server, address, + address_ipv6, and port top-level configuration + options are used to configure a single server for + backwards-compatibility + Each server configuration supports the following keys: + server: the server to load. The server must exist by name in the directory servers/. If not specified, the default udp server will be loaded. * example for tcp server: "./servers/tcp" + address: address to listen on [default: 0.0.0.0] + address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] + port: port to listen for messages on [default: 8125] + debug: debug flag [default: false] - address: address to listen on [default: 0.0.0.0] - address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on [default: 8125] mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] mgmt_port: port to run the management TCP interface on [default: 8126] diff --git a/stats.js b/stats.js index 9758a285..9e10f1a8 100644 --- a/stats.js +++ b/stats.js @@ -23,7 +23,7 @@ var sets = {}; var counter_rates = {}; var timer_data = {}; var pctThreshold = null; -var flushInterval, keyFlushInt, serverLoaded, mgmtServer; +var flushInterval, keyFlushInt, serversLoaded, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; @@ -198,13 +198,12 @@ config.configFile(process.argv[2], function (config) { if (config.keyNameSanitize !== undefined) { keyNameSanitize = config.keyNameSanitize; } - if (!serverLoaded) { + if (!serversLoaded) { + // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); - // The default server is UDP - var server = config.server || './servers/udp' - serverLoaded = startServer(config, server, function (msg, rinfo) { + var handlePacket = function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; var packet_data = msg.toString(); @@ -279,7 +278,16 @@ config.configFile(process.argv[2], function (config) { } stats.messages.last_msg_seen = Math.round(new Date().getTime() / 1000); - }); + } + + // If config.servers isn't specified, use the top-level config for backwards-compatibility + var server_config = config.servers || [config] + for (var i = 0; i < server_config.length; i++) { + // The default server is UDP + var server = server_config[i].server || './servers/udp' + startServer(server_config[i], server, handlePacket) + } + serversLoaded = true mgmtServer = net.createServer(function(stream) { stream.setEncoding('ascii'); From cca35ef397fb372aa281080bc7615592fe81a335 Mon Sep 17 00:00:00 2001 From: Steve Mardenfeld Date: Tue, 12 May 2015 10:31:20 -0700 Subject: [PATCH 043/207] store set values as boolean and not extra values --- lib/set.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/set.js b/lib/set.js index b48a7a7f..0e7fbe71 100644 --- a/lib/set.js +++ b/lib/set.js @@ -14,7 +14,7 @@ Set.prototype = { }, insert: function(value) { if (value) { - this.store[value] = value; + this.store[value] = true; } }, clear: function() { From e2aec69a89ae45fa47e879be4d001c81623a2ef6 Mon Sep 17 00:00:00 2001 From: Steve Mardenfeld Date: Tue, 12 May 2015 10:51:30 -0700 Subject: [PATCH 044/207] add size to set --- backends/graphite.js | 2 +- lib/set.js | 3 +++ test/set_tests.js | 13 +++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index 09e7b48f..2e7ae9af 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -158,7 +158,7 @@ var flush_stats = function graphite_flush(ts, metrics) { for (key in sets) { var namespace = setsNamespace.concat(sk(key)); - statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; + statString += namespace.join(".") + '.count' + globalSuffix + sets[key].size() + ts_suffix; numStats += 1; } diff --git a/lib/set.js b/lib/set.js index 0e7fbe71..b8ea3da0 100644 --- a/lib/set.js +++ b/lib/set.js @@ -26,6 +26,9 @@ Set.prototype = { values.push(value); } return values; + }, + size: function() { + return Object.keys(this.store).length; } }; diff --git a/test/set_tests.js b/test/set_tests.js index 47b645bc..99f20867 100644 --- a/test/set_tests.js +++ b/test/set_tests.js @@ -37,5 +37,18 @@ module.exports = { s.insert('b'); test.equal(2, s.values().length); test.done(); + }, + size_is_correct: function(test) { + test.expect(5); + var s = new set.Set(); + test.equal(0, s.size()); + s.insert('a'); + test.equal(1, s.size()); + s.insert('a'); + test.equal(1, s.size()); + s.insert('b'); + test.equal(2, s.size()); + test.equal(s.values().length, s.size()); + test.done(); } } From fe0d7da477befd8419f4312c4c13fd6b77205e07 Mon Sep 17 00:00:00 2001 From: Dannel Jurado Date: Wed, 13 May 2015 16:59:29 -0400 Subject: [PATCH 045/207] Pin to new node-syslog release that fixes #510 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 81cc1ea5..b2cae50a 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "temp": "0.4.x" }, "optionalDependencies": { - "node-syslog":"schamane/node-syslog#30d44555", + "node-syslog":"1.2.0", "hashring":"3.1.0", "winser": "=0.1.6" }, From ab6d2651503c8a5a8fd704f8f30ebb8094184f8e Mon Sep 17 00:00:00 2001 From: Joseph Hughes Date: Thu, 11 Jun 2015 11:18:44 -0600 Subject: [PATCH 046/207] update node version --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index fdedea94..143c8e30 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 8.0.0) Package: statsd Architecture: all -Depends: nodejs (>= 0.8), adduser +Depends: nodejs (>= 0.10), adduser Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. From 715c9188193612183f341cbe48c4ad1cc318e453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denny=20Sch=C3=A4fer?= Date: Mon, 15 Jun 2015 12:30:29 +0200 Subject: [PATCH 047/207] Add additional dh_installinit rules for put statsd proxy upstart script into the deb package --- debian/rules | 4 ++++ debian/{statsd_proxy.upstart => statsd-proxy.upstart} | 0 2 files changed, 4 insertions(+) rename debian/{statsd_proxy.upstart => statsd-proxy.upstart} (100%) diff --git a/debian/rules b/debian/rules index 2d33f6ac..7e0e37ad 100755 --- a/debian/rules +++ b/debian/rules @@ -2,3 +2,7 @@ %: dh $@ + +override_dh_installinit: + dh_installinit --name=statsd -- defaults + dh_installinit --name=statsd-proxy -- defaults diff --git a/debian/statsd_proxy.upstart b/debian/statsd-proxy.upstart similarity index 100% rename from debian/statsd_proxy.upstart rename to debian/statsd-proxy.upstart From 44767449055ebfa1f317906535aeab6a6aab7f75 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Thu, 18 Jun 2015 15:26:42 -0700 Subject: [PATCH 048/207] basic end-to-end test of repeater --- servers/udp.js | 11 ++- test/repeater_tests.js | 182 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 test/repeater_tests.js diff --git a/servers/udp.js b/servers/udp.js index a0a3072b..6f4b1a59 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,8 +1,15 @@ -var dgram = require('dgram'); +var dgram = require('dgram'), + Logger = require('../lib/logger').Logger; -exports.start = function(config, callback){ +exports.start = function(config, callback) { + var l = new Logger(config.log || {}); var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; var server = dgram.createSocket(udp_version, callback); + server.bind(config.port || 8125, config.address || undefined); + server.on('listening', function() { + l.log('server is listening'); + }); + return true; }; diff --git a/test/repeater_tests.js b/test/repeater_tests.js new file mode 100644 index 00000000..94f406ce --- /dev/null +++ b/test/repeater_tests.js @@ -0,0 +1,182 @@ +var net = require('net'), + spawn = require('child_process').spawn, + fs = require('fs'), + temp = require('temp'), + dgram = require('dgram'); + + +var writeconfig = function(text) { + var info = temp.openSync({ suffix: '-statsdconf.js' }); + fs.writeSync(info.fd, text); + return info.path; +}; + + +function log() { + //console.log.apply(console, arguments); +} + + +var FakeStatsDServer = function() { + this.port = 9125; +}; + +FakeStatsDServer.prototype.start = function(cb) { + this.sock = dgram.createSocket('udp4'); + + var self = this; + this.sock.on('listening', function() { + log('Fake statsd server listening on', self.port); + cb(); + }); + + this.sock.bind(this.port); +}; + +FakeStatsDServer.prototype.stop = function(cb) { + this.sock.close(); + cb(); +}; + +FakeStatsDServer.prototype.collect = function(timeout, cb) { + var sock = this.sock; + + var messages = []; + function onmsg(msg) { + log('Received %s', msg.toString()); + messages.push(msg.toString()); + } + + sock.on('message', onmsg); + + setTimeout(function() { + sock.removeListener('message', onmsg); + cb(messages); + }, timeout); +}; + + +var StatsDClient = function(port, host) { + this.host = host || '127.0.0.1'; + this.port = port || 8125; +}; + +StatsDClient.prototype.send = function(data, cb) { + var buf = new Buffer(data); + var sock = dgram.createSocket('udp4'); + sock.send(buf, 0, buf.length, this.port, this.host, function(err, bytes) { + if(err) { + throw err; + } + sock.close(); + cb(); + }); +}; + + +var RepeaterServer = function(port, server_port) { + this.port = port || 8125; + this.server_port = server_port || 9125; +}; + +RepeaterServer.prototype.start = function(cb) { + var config_path = writeconfig( + "{ " + + "repeater: [{ host: '127.0.0.1', port: " + this.server_port + "}] " + + ", repeaterProtocol: 'udp4' " + + ", server: './servers/udp' " + + ", dumpMessages: true " + + ", port: " + this.port + " " + + ", backends: [ './backends/repeater' ] " + + "}" + ); + log('Wrote config file %s', config_path); + log('Starting repeater listening on', this.port, + 'forwarding to', this.server_port); + + this.server_up = true; + this.ok_to_die = false; + var self = this; + var r = spawn('node', ['stats.js', config_path]); + + r.on('exit', function(code) { + self.server_up = false; + if(!self.ok_to_die) { + console.log('node server unexpectedly quit with code:', code); + process.exit(); + } + self.exit_callback(); + }); + + r.stderr.on('data', function(data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + + r.stdout.on('data', function (data) { + if (data.toString().match(/server is listening/)) { + log('Repeater server is up'); + cb(); + } + }); + + this.repeater = r; +}; + +RepeaterServer.prototype.stop = function(cb) { + this.ok_to_die = true; + if(this.server_up) { + this.exit_callback = cb; + this.repeater.kill(); + } else { + cb(); + } +}; + + + +module.exports = { + + setUp: function(cb) { + this.server = new FakeStatsDServer(); + this.repeater = new RepeaterServer(); + + var repeater = this.repeater; + this.server.start(function() { + repeater.start(cb); + }); + }, + + tearDown: function(cb) { + var repeater = this.repeater; + this.server.stop(function() { + repeater.stop(cb); + }); + }, + + fake_server_is_working: function(test) { + test.expect(1); + var server = this.server; + var client = new StatsDClient(this.server.port, 'localhost'); + + client.send('foobar', function() { + server.collect(100, function(messages) { + test.equal(messages[0], 'foobar'); + test.done(); + }); + }); + }, + + repeater_works: function(test) { + test.expect(1); + var server = this.server; + var client = new StatsDClient(this.repeater.port, '127.0.0.1'); + + client.send('foobar', function() { + server.collect(200, function(messages) { + test.equal(messages[0], 'foobar'); + test.done(); + }); + }); + } + +}; \ No newline at end of file From 7435d2cb5dd836b2cb783a4c3e72f0af7b348377 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Thu, 18 Jun 2015 17:15:38 -0700 Subject: [PATCH 049/207] removing superfluous test of fake statsd backend --- test/repeater_tests.js | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/test/repeater_tests.js b/test/repeater_tests.js index 94f406ce..71449b8e 100644 --- a/test/repeater_tests.js +++ b/test/repeater_tests.js @@ -153,18 +153,6 @@ module.exports = { }); }, - fake_server_is_working: function(test) { - test.expect(1); - var server = this.server; - var client = new StatsDClient(this.server.port, 'localhost'); - - client.send('foobar', function() { - server.collect(100, function(messages) { - test.equal(messages[0], 'foobar'); - test.done(); - }); - }); - }, repeater_works: function(test) { test.expect(1); @@ -172,7 +160,7 @@ module.exports = { var client = new StatsDClient(this.repeater.port, '127.0.0.1'); client.send('foobar', function() { - server.collect(200, function(messages) { + server.collect(100, function(messages) { test.equal(messages[0], 'foobar'); test.done(); }); From ef8560baede19bc437c3d61d24dd8c37d2fb06b9 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Thu, 18 Jun 2015 20:54:23 -0700 Subject: [PATCH 050/207] cleaner config for test repeater server --- test/repeater_tests.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/repeater_tests.js b/test/repeater_tests.js index 71449b8e..c11e6a94 100644 --- a/test/repeater_tests.js +++ b/test/repeater_tests.js @@ -77,19 +77,17 @@ StatsDClient.prototype.send = function(data, cb) { var RepeaterServer = function(port, server_port) { this.port = port || 8125; this.server_port = server_port || 9125; + this.config = { + repeater: [{ host: '127.0.0.1', port: this.server_port }], + repeaterProtocol: 'udp4', + server: './servers/udp', + port: this.port, + backends: [ './backends/repeater' ] + }; }; RepeaterServer.prototype.start = function(cb) { - var config_path = writeconfig( - "{ " - + "repeater: [{ host: '127.0.0.1', port: " + this.server_port + "}] " - + ", repeaterProtocol: 'udp4' " - + ", server: './servers/udp' " - + ", dumpMessages: true " - + ", port: " + this.port + " " - + ", backends: [ './backends/repeater' ] " - + "}" - ); + var config_path = writeconfig(JSON.stringify(this.config)); log('Wrote config file %s', config_path); log('Starting repeater listening on', this.port, 'forwarding to', this.server_port); From ce1551497a5b7c7e01df587c6905fe7a1a16974f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denny=20Sch=C3=A4fer?= Date: Mon, 15 Jun 2015 22:52:54 +0200 Subject: [PATCH 051/207] Update debian changelog regarding pull request #517 from xofer --- debian/changelog | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7101d071..b40f9fa6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,40 @@ -statsd (0.7.2) unstable; urgency=low - - * Align version number to git tag versions - * Add statsd_proxy upstart - * Install node dependencies on package installation - * Add proxyConfig.js - * Remove init script that does not work in favor of just having upstart - * Fix node version dependency - - -- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 +statsd (0.7.2-1) unstable; urgency=low + + * Fixes to detecting valid packets + + -- Unknown Author <> Tue, 2 Sep 2014 01:00:00 +0000 + +statsd (0.7.1-1) unstable; urgency=low + + * move contributing information into CONTRIBUTING.md + * Updates winser to v0.1.6 + * examples: python: added efficiency note + * python: examples: fixed doctests for Python 3 + * Standardized debian log locations + * Enhancement: consume logger in graphite and repeater backends + * Enhancement: update backend documentation + * Enhancement: inject logger object into backend + * Send STDOUT and STDERR to the appropriate files + + -- Unknown Author <> Thu, 6 Feb 2014 01:00:00 +0000 + +statsd (v0.7.0-1) unstable; urgency=low + + * added cluster proxy + * measure and graph timestamp generation lag + * added median calculation for timers + * support for top percentiles for timers + * drop support for node v0.6.x + * support for setting the process title + * functionality for optionally omitting stats_counts metrics + * improved functionality to delete counters from the management console + * updates to Debian packaging + * added a clojure example client + * cleaned up the Go example client + * increased test coverage + * documentation updates + + -- Unknown Author <> Fri, 5 Dec 2014 01:00:00 +0000 statsd (0.6.0-1) unstable; urgency=low From d5fc1e31b65090928062a156cab24cf59487b8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denny=20Sch=C3=A4fer?= Date: Fri, 19 Jun 2015 12:12:24 +0200 Subject: [PATCH 052/207] Update changelog regarding pull request #506 from joshughes --- debian/changelog | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b40f9fa6..40deac9d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,14 @@ statsd (0.7.2-1) unstable; urgency=low * Fixes to detecting valid packets + * Align version number to git tag versions + * Add statsd_proxy upstart + * Install node dependencies on package installation + * Add proxyConfig.js + * Remove init script that does not work in favor of just having upstart + * Fix node version dependency - -- Unknown Author <> Tue, 2 Sep 2014 01:00:00 +0000 + -- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 statsd (0.7.1-1) unstable; urgency=low From 979c13479ff4bc6dd3125a61f45bfa14cf326393 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Fri, 19 Jun 2015 11:20:45 -0700 Subject: [PATCH 053/207] refactoring tests --- test/repeater_tests.js | 63 ++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/test/repeater_tests.js b/test/repeater_tests.js index c11e6a94..2798b432 100644 --- a/test/repeater_tests.js +++ b/test/repeater_tests.js @@ -131,36 +131,69 @@ RepeaterServer.prototype.stop = function(cb) { }; +var ServerSet = function() { + this.servers = []; +}; +ServerSet.prototype.add = function() { + for(var i = 0; i < arguments.length; i++) { + this.servers.push(arguments[i]); + } +}; +ServerSet.prototype.start = function(cb) { + var self = this; + function start_server(i) { + if(i == self.servers.length) { + cb(); + } else { + self.servers[i].start(function() { + start_server(i + 1); + }); + } + } + start_server(0); +}; +ServerSet.prototype.stop = function(cb) { + var self = this; + function stop_server(i) { + if(i == self.servers.length) { + cb(); + } else { + self.servers[i].stop(function() { + stop_server(i + 1); + }); + } + } + stop_server(0); +}; + + module.exports = { setUp: function(cb) { - this.server = new FakeStatsDServer(); + this.servers = new ServerSet(); this.repeater = new RepeaterServer(); - - var repeater = this.repeater; - this.server.start(function() { - repeater.start(cb); - }); + this.servers.add(this.repeater); + cb(); }, tearDown: function(cb) { - var repeater = this.repeater; - this.server.stop(function() { - repeater.stop(cb); - }); + this.servers.stop(cb); }, repeater_works: function(test) { test.expect(1); - var server = this.server; + var statsd = new FakeStatsDServer(); + this.servers.add(statsd); var client = new StatsDClient(this.repeater.port, '127.0.0.1'); - client.send('foobar', function() { - server.collect(100, function(messages) { - test.equal(messages[0], 'foobar'); - test.done(); + this.servers.start(function(){ + client.send('foobar', function() { + statsd.collect(100, function(messages) { + test.equal(messages[0], 'foobar'); + test.done(); + }); }); }); } From 13a78b6844aef621799cbebf0a7c45a97d960790 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Fri, 19 Jun 2015 14:39:06 -0700 Subject: [PATCH 054/207] tcp repeater working. much improved, nonpolling, nonspawning tests --- backends/repeater.js | 129 ++++++++++++++++++++++++++--- package.json | 1 + servers/tcp.js | 4 +- servers/udp.js | 4 +- test/repeater_tests.js | 182 +++++++++++++++++------------------------ 5 files changed, 200 insertions(+), 120 deletions(-) diff --git a/backends/repeater.js b/backends/repeater.js index 5251e589..ccbf75eb 100644 --- a/backends/repeater.js +++ b/backends/repeater.js @@ -2,43 +2,152 @@ var util = require('util') , dgram = require('dgram') - , logger = require('../lib/logger'); + , logger = require('../lib/logger') + , Pool = require('generic-pool').Pool + , net = require('net'); + var l; var debug; +var instance; + +function logerror(err) { + if(err && debug) { + l.log(err); + } +} -function RepeaterBackend(startupTime, config, emitter){ + + +function UDPRepeaterBackend(startupTime, config, emitter) { var self = this; this.config = config.repeater || []; this.sock = (config.repeaterProtocol == 'udp6') ? dgram.createSocket('udp6') : dgram.createSocket('udp4'); + // Attach DNS error handler this.sock.on('error', function (err) { if (debug) { l.log('Repeater error: ' + err); } }); + // attach emitter.on('packet', function(packet, rinfo) { self.process(packet, rinfo); }); } -RepeaterBackend.prototype.process = function(packet, rinfo) { + +UDPRepeaterBackend.prototype.process = function(packet, rinfo) { var self = this; - hosts = self.config; + var hosts = self.config; for(var i=0; i Date: Fri, 19 Jun 2015 14:41:47 -0700 Subject: [PATCH 055/207] updating documentation --- exampleConfig.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exampleConfig.js b/exampleConfig.js index 598f2ec3..d0a37d21 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -83,8 +83,8 @@ Optional Variables: e.g. [ { host: '10.10.10.10', port: 8125 }, { host: 'observer', port: 88125 } ] - repeaterProtocol: whether to use udp4 or udp6 for repeaters. - ["udp4" or "udp6", default: "udp4"] + repeaterProtocol: whether to use udp4, udp6, or tcp for repeaters. + ["udp4," "udp6", or "tcp" default: "udp4"] histogram: for timers, an array of mappings of strings (to match metrics) and corresponding ordered non-inclusive upper limits of bins. From de2c38ea1baaa3945fb019833aeb81e5fdc8e5f8 Mon Sep 17 00:00:00 2001 From: Dan McKinley Date: Fri, 19 Jun 2015 14:46:20 -0700 Subject: [PATCH 056/207] removing superfluous logging setup in udp server --- servers/udp.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/servers/udp.js b/servers/udp.js index 4c18e6f5..345cc567 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,8 +1,6 @@ -var dgram = require('dgram'), - Logger = require('../lib/logger').Logger; +var dgram = require('dgram'); exports.start = function(config, callback) { - var l = new Logger(config.log || {}); var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; var server = dgram.createSocket(udp_version, callback); From 4e5cc127cbee5dcb7e267085ab9a0f5afff24066 Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Mon, 22 Jun 2015 00:42:22 +0300 Subject: [PATCH 057/207] IncrementByValue method added --- examples/go/statsd.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/go/statsd.go b/examples/go/statsd.go index 962e567a..4bd94b3c 100644 --- a/examples/go/statsd.go +++ b/examples/go/statsd.go @@ -96,6 +96,18 @@ func (client *StatsdClient) Increment(stat string) { client.UpdateStats(stats, 1, 1) } +// Increments one stat counter by value provided without sampling +// +// Usage: +// +// import "statsd" +// client := statsd.New('localhost', 8125) +// client.Increment('foo.bar', 5) +func (client *StatsdClient) IncrementByValue(stat string, val int) { + stats := []string{stat} + client.UpdateStats(stats, val, 1) +} + // Increments one stat counter with sampling // // Usage: From 5feea5e637baa2c694f3a5448a0f3ca6cc52b137 Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Mon, 22 Jun 2015 00:51:07 +0300 Subject: [PATCH 058/207] IncrementByValue method added --- examples/go/statsd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/go/statsd.go b/examples/go/statsd.go index 4bd94b3c..ef34e9e5 100644 --- a/examples/go/statsd.go +++ b/examples/go/statsd.go @@ -102,7 +102,7 @@ func (client *StatsdClient) Increment(stat string) { // // import "statsd" // client := statsd.New('localhost', 8125) -// client.Increment('foo.bar', 5) +// client.IncrementByValue('foo.bar', 5) func (client *StatsdClient) IncrementByValue(stat string, val int) { stats := []string{stat} client.UpdateStats(stats, val, 1) From 1fb5ba91aae79f5be9b6460eb4cfa440fa2da9f9 Mon Sep 17 00:00:00 2001 From: Vivien Barousse Date: Mon, 22 Jun 2015 12:43:41 +0100 Subject: [PATCH 059/207] Explicitly close UDP sockets in Ruby examples Not closing the sockets explicitely causes them to hang around longer than expected, and can cause issues when the number of open sockets exceeds system limits. --- examples/ruby_example.rb | 1 + examples/ruby_example2.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/ruby_example.rb b/examples/ruby_example.rb index ec52f098..cc436685 100644 --- a/examples/ruby_example.rb +++ b/examples/ruby_example.rb @@ -68,6 +68,7 @@ def self.send(data, sample_rate=1) value = data[stat] sock.send("#{stat}:#{value}", 0, @@config[:host], @@config[:port]) end + sock.close end end end diff --git a/examples/ruby_example2.rb b/examples/ruby_example2.rb index 0281a99f..a52f3183 100644 --- a/examples/ruby_example2.rb +++ b/examples/ruby_example2.rb @@ -77,6 +77,7 @@ def self.send(data, sample_rate=1) send_data = "%s:%s" % [stat, val] udp.send send_data, 0, host, port end + udp.close rescue => e puts e.message end From a607becc310e5eda5e802b1fe410b193e2272db7 Mon Sep 17 00:00:00 2001 From: rikanulo Date: Tue, 23 Jun 2015 18:09:27 +0300 Subject: [PATCH 060/207] New ATSD backend added A new backend for Axibase Time-Series Database (ATSD) is added. --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 7039ec55..52e092f8 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -32,6 +32,7 @@ queues and third-party services. ## Available Third-party backends - [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) +- [atsd-backend](https://github.com/axibase/atsd-statsd-backend) - [node-bell](https://github.com/eleme/node-bell) - [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) - [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) From fa001a503159e5ed89461a2a1b0e3f1303e50694 Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Wed, 1 Jul 2015 14:49:01 +0200 Subject: [PATCH 061/207] Add npm to deb package dependencies npm is used in debian/postinst script. --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 143c8e30..c19608f4 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 8.0.0) Package: statsd Architecture: all -Depends: nodejs (>= 0.10), adduser +Depends: nodejs (>= 0.10), adduser, npm Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. From 77ca6ccd3ebaa22f532fe663c51c0508c6ff4cc7 Mon Sep 17 00:00:00 2001 From: Richard Hightower Date: Wed, 1 Jul 2015 17:26:04 -0700 Subject: [PATCH 062/207] Update StatsdClient.java --- examples/StatsdClient.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/StatsdClient.java b/examples/StatsdClient.java index 3b31ff67..c6202cdc 100644 --- a/examples/StatsdClient.java +++ b/examples/StatsdClient.java @@ -67,6 +67,10 @@ public StatsdClient(String host, int port) throws UnknownHostException, IOExcept public StatsdClient(InetAddress host, int port) throws IOException { _address = new InetSocketAddress(host, port); _channel = DatagramChannel.open(); + /* Put this in non-blocking mode so send does not block forever. */ + _channel.configureBlocking(false); + /* Increase the size of the output buffer so that the size is larger than our buffer size. */ + _channel.setOption(StandardSocketOptions.SO_SNDBUF, 4096); setBufferSize((short) 1500); } @@ -250,6 +254,7 @@ public synchronized boolean flush() { } } catch (IOException e) { + /* This would be a good place to close the channel down and recreate it. */ log.error( String.format("Could not send stat %s to host %s:%d", sendBuffer.toString(), _address.getHostName(), _address.getPort()), e); From 159cb6902ab4fa51e2ff112123f3ef89368672e2 Mon Sep 17 00:00:00 2001 From: Aron Atkins Date: Tue, 9 Jul 2013 12:22:16 -0400 Subject: [PATCH 063/207] Separate the collection of all metrics from the serialization of those metrics. Looking ahead to supporting the graphite pickle format. --- backends/graphite.js | 100 ++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 2e7ae9af..0b6ff717 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -45,11 +45,12 @@ var setsNamespace = []; var graphiteStats = {}; -var post_stats = function graphite_post_stats(statString) { +var post_stats = function graphite_post_stats(stats) { var last_flush = graphiteStats.last_flush || 0; var last_exception = graphiteStats.last_exception || 0; var flush_time = graphiteStats.flush_time || 0; var flush_length = graphiteStats.flush_length || 0; + if (graphiteHost) { try { var graphite = net.createConnection(graphitePort, graphiteHost); @@ -59,34 +60,61 @@ var post_stats = function graphite_post_stats(statString) { } }); graphite.on('connect', function() { - var ts = Math.round(new Date().getTime() / 1000); - var ts_suffix = ' ' + ts + "\n"; + var ts = Math.round(Date.now() / 1000); var namespace = globalNamespace.concat(prefixStats).join("."); - statString += namespace + '.graphiteStats.last_exception' + globalSuffix + last_exception + ts_suffix; - statString += namespace + '.graphiteStats.last_flush' + globalSuffix + last_flush + ts_suffix; - statString += namespace + '.graphiteStats.flush_time' + globalSuffix + flush_time + ts_suffix; - statString += namespace + '.graphiteStats.flush_length' + globalSuffix + flush_length + ts_suffix; + stats.add(namespace + '.graphiteStats.last_exception' + globalSuffix, last_exception, ts); + stats.add(namespace + '.graphiteStats.last_flush' + globalSuffix, last_flush , ts); + stats.add(namespace + '.graphiteStats.flush_time' + globalSuffix, flush_time , ts); + stats.add(namespace + '.graphiteStats.flush_length' + globalSuffix, flush_length , ts); + var stats_payload = stats.toText(); var starttime = Date.now(); - this.write(statString); + this.write(stats_payload); this.end(); + graphiteStats.flush_time = (Date.now() - starttime); - graphiteStats.flush_length = statString.length; - graphiteStats.last_flush = Math.round(new Date().getTime() / 1000); + graphiteStats.flush_length = stats_payload.length; + graphiteStats.last_flush = Math.round(Date.now() / 1000); }); } catch(e){ if (debug) { l.log(e); } - graphiteStats.last_exception = Math.round(new Date().getTime() / 1000); + graphiteStats.last_exception = Math.round(Date.now() / 1000); } } }; +// A single measurement for sending to graphite. +function Metric(key, value, ts) { + var m = this; + this.key = key; + this.value = value; + this.ts = ts; + + // return a string representation of this metric appropriate + // for sending to the graphite collector. does not include + // a trailing newline. + this.toText = function() { + return m.key + " " + m.value + " " + m.ts; + }; +} + +// A collection of measurements for sending to graphite. +function Stats() { + var s = this; + this.metrics = []; + this.add = function(key, value, ts) { + s.metrics.push(new Metric(key, value, ts)); + }; + + this.toText = function() { + return s.metrics.map(function(m) { return m.toText(); }).join('\n') + '\n'; + }; +} + var flush_stats = function graphite_flush(ts, metrics) { - var ts_suffix = ' ' + ts + "\n"; var starttime = Date.now(); - var statString = ''; var numStats = 0; var key; var timer_data_key; @@ -109,6 +137,11 @@ var flush_stats = function graphite_flush(ts, metrics) { } }; + // Flatten all the different types of metrics into a single + // collection so we can allow serialization to either the graphite + // text and pickle formats. + var stats = new Stats(); + for (key in counters) { var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate @@ -116,14 +149,14 @@ var flush_stats = function graphite_flush(ts, metrics) { var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { - statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; + stats.add(namespace.join(".") + globalSuffix, valuePerSecond, ts); if (flush_counts) { - statString += 'stats_counts.' + keyName + globalSuffix + value + ts_suffix; + stats.add('stats_counts.' + keyName + globalSuffix, value, ts); } } else { - statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; + stats.add(namespace.concat('rate').join(".") + globalSuffix, valuePerSecond, ts); if (flush_counts) { - statString += namespace.concat('count').join(".") + globalSuffix + value + ts_suffix; + stats.add(namespace.concat('count').join(".") + globalSuffix, value, ts); } } @@ -136,14 +169,14 @@ var flush_stats = function graphite_flush(ts, metrics) { for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { - statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; + stats.add(the_key + '.' + timer_data_key + globalSuffix, timer_data[key][timer_data_key], ts); } else { for (var timer_data_sub_key in timer_data[key][timer_data_key]) { if (debug) { l.log(timer_data[key][timer_data_key][timer_data_sub_key].toString()); } - statString += the_key + '.' + timer_data_key + '.' + timer_data_sub_key + globalSuffix + - timer_data[key][timer_data_key][timer_data_sub_key] + ts_suffix; + stats.add(the_key + '.' + timer_data_key + '.' + timer_data_sub_key + globalSuffix, + timer_data[key][timer_data_key][timer_data_sub_key], ts); } } } @@ -152,32 +185,32 @@ var flush_stats = function graphite_flush(ts, metrics) { for (key in gauges) { var namespace = gaugesNamespace.concat(sk(key)); - statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; + stats.add(namespace.join(".") + globalSuffix, gauges[key], ts); numStats += 1; } for (key in sets) { var namespace = setsNamespace.concat(sk(key)); - statString += namespace.join(".") + '.count' + globalSuffix + sets[key].size() + ts_suffix; + stats.add(namespace.join(".") + '.count' + globalSuffix, sets[key].size(), ts); numStats += 1; } - var namespace = globalNamespace.concat(prefixStats); if (legacyNamespace === true) { - statString += prefixStats + '.numStats' + globalSuffix + numStats + ts_suffix; - statString += 'stats.' + prefixStats + '.graphiteStats.calculationtime' + globalSuffix + (Date.now() - starttime) + ts_suffix; + stats.add(prefixStats + '.numStats' + globalSuffix, numStats, ts); + stats.add('stats.' + prefixStats + '.graphiteStats.calculationtime' + globalSuffix, (Date.now() - starttime), ts); for (key in statsd_metrics) { - statString += 'stats.' + prefixStats + '.' + key + globalSuffix + statsd_metrics[key] + ts_suffix; + stats.add('stats.' + prefixStats + '.' + key + globalSuffix, statsd_metrics[key], ts); } } else { - statString += namespace.join(".") + '.numStats' + globalSuffix + numStats + ts_suffix; - statString += namespace.join(".") + '.graphiteStats.calculationtime' + globalSuffix + (Date.now() - starttime) + ts_suffix; + var namespace = globalNamespace.concat(prefixStats); + stats.add(namespace.join(".") + '.numStats' + globalSuffix, numStats, ts); + stats.add(namespace.join(".") + '.graphiteStats.calculationtime' + globalSuffix, (Date.now() - starttime) , ts); for (key in statsd_metrics) { var the_key = namespace.concat(key); - statString += the_key.join(".") + globalSuffix + statsd_metrics[key] + ts_suffix; + stats.add(the_key.join(".") + globalSuffix,+ statsd_metrics[key], ts); } } - post_stats(statString); + post_stats(stats); if (debug) { l.log("numStats: " + numStats); @@ -214,10 +247,9 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; legacyNamespace = legacyNamespace !== undefined ? legacyNamespace : true; - // In order to unconditionally add this string, it either needs to be - // a single space if it was unset, OR surrounded by a . and a space if - // it was set. - globalSuffix = globalSuffix !== undefined ? '.' + globalSuffix + ' ' : ' '; + // In order to unconditionally add this string, it either needs to be an + // empty string if it was unset, OR prefixed by a . if it was set. + globalSuffix = globalSuffix !== undefined ? '.' + globalSuffix : ''; if (legacyNamespace === false) { if (globalPrefix !== "") { From 736118f172d1fdb7da3585f07322e9b2133e50ba Mon Sep 17 00:00:00 2001 From: Aron Atkins Date: Tue, 9 Jul 2013 14:56:26 -0400 Subject: [PATCH 064/207] pickling configs. stub pickle serialization. --- backends/graphite.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 0b6ff717..eba98cfa 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -11,7 +11,11 @@ * This backend supports the following config options: * * graphiteHost: Hostname of graphite server. - * graphitePort: Port to contact graphite server at. + * graphitePort: Port for the graphite text collector. Defaults to 2003. + * graphitePicklePort: Port for the graphite pickle collector. Defaults to 2004. + * graphiteProtocol: Either 'text' or 'pickle'. Defaults to 'text'. + * + * If graphiteHost is not specified, metrics are processed but discarded. */ var net = require('net'); @@ -23,6 +27,8 @@ var debug; var flushInterval; var graphiteHost; var graphitePort; +var graphitePicklePort; +var graphiteProtocol; var flush_counts; // prefix configuration @@ -53,7 +59,8 @@ var post_stats = function graphite_post_stats(stats) { if (graphiteHost) { try { - var graphite = net.createConnection(graphitePort, graphiteHost); + var port = graphiteProtocol == 'pickle' ? graphitePicklePort : graphitePort; + var graphite = net.createConnection(port, graphiteHost); graphite.addListener('error', function(connectionException){ if (debug) { l.log(connectionException); @@ -66,7 +73,7 @@ var post_stats = function graphite_post_stats(stats) { stats.add(namespace + '.graphiteStats.last_flush' + globalSuffix, last_flush , ts); stats.add(namespace + '.graphiteStats.flush_time' + globalSuffix, flush_time , ts); stats.add(namespace + '.graphiteStats.flush_length' + globalSuffix, flush_length , ts); - var stats_payload = stats.toText(); + var stats_payload = graphiteProtocol == 'pickle' ? stats.toPickle() : stats.toText(); var starttime = Date.now(); this.write(stats_payload); @@ -111,6 +118,10 @@ function Stats() { this.toText = function() { return s.metrics.map(function(m) { return m.toText(); }).join('\n') + '\n'; }; + + this.toPickle = function() { + return '\n'; + }; } var flush_stats = function graphite_flush(ts, metrics) { @@ -227,7 +238,9 @@ exports.init = function graphite_init(startup_time, config, events, logger) { debug = config.debug; l = logger; graphiteHost = config.graphiteHost; - graphitePort = config.graphitePort; + graphitePort = config.graphitePort || 2003; + graphitePicklePort = config.graphitePicklePort || 2004; + graphiteProtocol = config.graphiteProtocol || 'text'; config.graphite = config.graphite || {}; globalPrefix = config.graphite.globalPrefix; prefixCounter = config.graphite.prefixCounter; From 9d2e0cdc330c2f5a4dbe6a6f06422d6b17236f3c Mon Sep 17 00:00:00 2001 From: Aron Atkins Date: Tue, 9 Jul 2013 16:25:29 -0400 Subject: [PATCH 065/207] update example configuration with pickling goodness. --- exampleConfig.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/exampleConfig.js b/exampleConfig.js index 2a6fca06..6605ca03 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -1,15 +1,17 @@ /* -Graphite Required Variables: +Graphite Required Variable: -(Leave these unset to avoid sending stats to Graphite. - Set debug flag and leave these unset to run in 'dry' debug mode - +(Leave this unset to avoid sending stats to Graphite. + Set debug flag and leave this unset to run in 'dry' debug mode - useful for testing statsd clients without a Graphite server.) graphiteHost: hostname or IP of Graphite server - graphitePort: port of Graphite server Optional Variables: + graphitePort: port for the graphite text collector [default: 2003] + graphitePicklePort: port for the graphite pickle collector [default: 2004] + graphiteProtocol: either 'text' or 'pickle' [default: text] backends: an array of backends to load. Each backend must exist by name in the directory backends/. If not specified, the default graphite backend will be loaded. From b5218765607df7570324b12809426cff2de1d248 Mon Sep 17 00:00:00 2001 From: Aron Atkins Date: Wed, 10 Jul 2013 11:54:11 -0400 Subject: [PATCH 066/207] pickling support. --- backends/graphite.js | 25 +++- docs/graphite_pickle.md | 81 ++++++++++++ exampleConfig.js | 2 +- test/graphite_pickle_tests.js | 227 ++++++++++++++++++++++++++++++++++ 4 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 docs/graphite_pickle.md create mode 100644 test/graphite_pickle_tests.js diff --git a/backends/graphite.js b/backends/graphite.js index eba98cfa..de23a852 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -92,6 +92,15 @@ var post_stats = function graphite_post_stats(stats) { } }; +// Minimally necessary pickle opcodes. +var MARK = '(', + STOP = '.', + LONG = 'L', + STRING = 'S', + APPEND = 'a', + LIST = 'l', + TUPLE = 't'; + // A single measurement for sending to graphite. function Metric(key, value, ts) { var m = this; @@ -105,6 +114,10 @@ function Metric(key, value, ts) { this.toText = function() { return m.key + " " + m.value + " " + m.ts; }; + + this.toPickle = function() { + return MARK + STRING + '\'' + m.key + '\'\n' + MARK + LONG + m.ts + 'L\n' + STRING + '\'' + m.value + '\'\n' + TUPLE + TUPLE + APPEND; + }; } // A collection of measurements for sending to graphite. @@ -120,7 +133,17 @@ function Stats() { }; this.toPickle = function() { - return '\n'; + var body = MARK + LIST + s.metrics.map(function(m) { return m.toPickle(); }).join('') + STOP; + + // The first four bytes of the graphite pickle format + // contain the length of the rest of the payload. + // We use Buffer because this is binary data. + var buf = new Buffer(4 + body.length); + + buf.writeUInt32BE(body.length,0); + buf.write(body,4); + + return buf; }; } diff --git a/docs/graphite_pickle.md b/docs/graphite_pickle.md new file mode 100644 index 00000000..2151f59f --- /dev/null +++ b/docs/graphite_pickle.md @@ -0,0 +1,81 @@ +Pickling for Graphite +===================== + +The graphite statsd backend can optionally be configured to use pickle +for its over-the-wire protocol. + +```javascript + { graphiteHost: "your.graphite.host", + graphiteProtocol: "pickle" } +``` + +The default is to use the graphite text protocol, which can require +more CPU processing by the graphite endpoint. + +The message format expected by the graphite pickle endpoint consists +of a header and payload. + +The Payload +----------- + +The message payload is a list of tuples. Each tuple contains the measurement +for a single metric name. The measurement is encoded as a second, +nested tuple containing timestamp and measured value. + +This ends up looking like: + +```python +[ ( "path.to.metric.name", ( timestamp, "value" ) ), + ( "path.to.another.name", ( timestamp, "value" ) ) ] +``` + +The graphite receiver `carbon.protocols.MetricPickleReceiver` coerces +both the timestamp and measured value into `float`. + +The timestamp must be seconds since epoch encoded as a number. + +The measured value is encoded as a string. This may change in the +future. + +We have chosen to not implement pickle's object memoization. This +simplifies what is sent across the wire. It is not likely any +optimization would result within a single poll cycle. + +Here is some Python code showing how a given set of metrics can be +serialized in a more simple way. + +```python +import pickle + +metrics = [ ( "a.b.c", ( 1234L, "5678" ) ), ( "d.e.f.g", ( 1234L, "9012" ) ) ] +pickle.dumps(metrics) +# "(lp0\n(S'a.b.c'\np1\n(L1234L\nS'5678'\np2\ntp3\ntp4\na(S'd.e.f.g'\np5\n(L1234L\nS'9012'\np6\ntp7\ntp8\na." + +payload = "(l(S'a.b.c'\n(L1234L\nS'5678'\ntta(S'd.e.f.g'\n(L1234L\nS'9012'\ntta." +pickle.loads(payload) +# [('a.b.c', (1234L, '5678')), ('d.e.f.g', (1234L, '9012'))] +``` + +The trailing `L` for long fields is unnecessary, but we are adding the +character to match Python pickle output. It's a side-effect of +`repr(long(1234))`. + +The Header +---------- + +The message header is a 32-bit integer sent over the wire as +four-bytes. This integer must describe the length of the pickled +payload. + +Here is some sample code showing how to construct the message header +containing the payload length. + +```python +import struct + +payload_length = 81 +header = struct.pack("!L", payload_length) +# '\x00\x00\x00Q' +``` + +The `Q` character is equivalent to `\x81` (ASCII encoding). diff --git a/exampleConfig.js b/exampleConfig.js index 6605ca03..3aa7f573 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -11,7 +11,7 @@ Optional Variables: graphitePort: port for the graphite text collector [default: 2003] graphitePicklePort: port for the graphite pickle collector [default: 2004] - graphiteProtocol: either 'text' or 'pickle' [default: text] + graphiteProtocol: either 'text' or 'pickle' [default: 'text'] backends: an array of backends to load. Each backend must exist by name in the directory backends/. If not specified, the default graphite backend will be loaded. diff --git a/test/graphite_pickle_tests.js b/test/graphite_pickle_tests.js new file mode 100644 index 00000000..8a9fda2a --- /dev/null +++ b/test/graphite_pickle_tests.js @@ -0,0 +1,227 @@ +var fs = require('fs'), + net = require('net'), + temp = require('temp'), + cp = require('child_process'), + util = require('util'), + urlparse = require('url').parse, + _ = require('underscore'), + dgram = require('dgram'), + qsparse = require('querystring').parse, + http = require('http'); + +var spawn = cp.spawn; + +var writeconfig = function(text,worker,cb,obj){ + temp.open({suffix: '-statsdconf.js'}, function(err, info) { + if (err) throw err; + fs.writeSync(info.fd, text); + fs.close(info.fd, function(err) { + if (err) throw err; + worker(info.path,cb,obj); + }); + }); +}; + +var statsd_send = function(data,sock,host,port,cb){ + send_data = new Buffer(data); + sock.send(send_data,0,send_data.length,port,host,function(err,bytes){ + if (err) { + throw err; + } + cb(); + }); +}; + +// keep collecting data until a specified timeout period has elapsed +// this will let us capture all data chunks so we don't miss one +var collect_for = function(server,timeout,cb){ + // We have binary data arriving over the wire. Avoid strings. + var received = new Buffer(0); + var in_flight = 0; + var timed_out = false; + var collector = function(req,res){ + in_flight += 1; + req.on('data',function(data){ received = Buffer.concat([received,data]); }); + req.on('end',function(){ + in_flight -= 1; + if((in_flight < 1) && timed_out){ + server.removeListener('request',collector); + cb(received); + } + }); + }; + + setTimeout(function (){ + timed_out = true; + if((in_flight < 1)) { + server.removeListener('connection',collector); + cb(received); + } + },timeout); + + server.on('connection',collector); +}; + +// A python script that converts from the graphite pickle-based +// wire protocol into JSON written to stdout. +var script = + "import sys\n" + + "import cPickle\n" + + "import struct\n" + + "import json\n" + + "payload = open(sys.argv[1], 'rb').read()\n" + + "pack_format = '!L'\n" + + "header_length = struct.calcsize(pack_format)\n" + + "payload_length, = struct.unpack(pack_format, payload[:header_length])\n" + + "batch_length = header_length + payload_length\n" + + "metrics = cPickle.loads(payload[header_length:batch_length])\n" + + "print json.dumps(metrics)\n"; + +// Write our binary payload and unpickling script to disk +// then process the unserialized results. +var unpickle = function(payload, cb) { + temp.open({suffix: '-payload.pickle'}, function(err, payload_info) { + if (err) throw err; + + // the header may contain null characters. explicit length is necessary. + var len = fs.writeSync(payload_info.fd, payload, 0, payload.length); + fs.close(payload_info.fd, function(err) { + if (err) throw err; + + temp.open({suffix:'-unpickle.py'}, function(err, unpickle_info) { + if (err) throw err; + + fs.writeSync(unpickle_info.fd, script); + fs.close(unpickle_info.fd, function(err) { + if (err) throw err; + + var cmd = 'python ' + unpickle_info.path + ' ' + payload_info.path; + var python = cp.exec(cmd, function(err, stdout, stderr) { + if (err) throw err; + var metrics = JSON.parse(stdout); + // Transform the output into the same list of dictionaries + // used by the other graphite_* tests so our tests look + // the same. + var hashes = _.map(metrics, function(m) { + var data = {}; + data[m[0]] = m[1][1]; + return data; + }); + cb(hashes); + }); + }); + }); + }); + }); +}; + +module.exports = { + setUp: function (callback) { + this.testport = 31337; + this.myflush = 200; + var configfile = "{graphService: \"graphite\"\n\ + , batch: 200 \n\ + , flushInterval: " + this.myflush + " \n\ + , percentThreshold: 90\n\ + , histogram: [ { metric: \"a_test_value\", bins: [1000] } ]\n\ + , port: 8125\n\ + , dumpMessages: false \n\ + , debug: false\n\ + , graphite: { legacyNamespace: false }\n\ + , graphitePicklePort: " + this.testport + "\n\ + , graphiteHost: \"127.0.0.1\"\n\ + , graphiteProtocol: \"pickle\"}"; + + this.acceptor = net.createServer(); + this.acceptor.listen(this.testport); + this.sock = dgram.createSocket('udp4'); + + this.server_up = true; + this.ok_to_die = false; + this.exit_callback_callback = process.exit; + + writeconfig(configfile,function(path,cb,obj){ + obj.path = path; + obj.server = spawn('node',['stats.js', path]); + obj.exit_callback = function (code) { + obj.server_up = false; + if(!obj.ok_to_die){ + console.log('node server unexpectedly quit with code: ' + code); + process.exit(1); + } + else { + obj.exit_callback_callback(); + } + }; + obj.server.on('exit', obj.exit_callback); + obj.server.stderr.on('data', function (data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + /* + obj.server.stdout.on('data', function (data) { + console.log('stdout: ' + data.toString().replace(/\n$/,'')); + }); + */ + obj.server.stdout.on('data', function (data) { + // wait until server is up before we finish setUp + if (data.toString().match(/server is up/)) { + cb(); + } + }); + + },callback,this); + }, + + tearDown: function (callback) { + this.sock.close(); + this.acceptor.close(); + this.ok_to_die = true; + if(this.server_up){ + this.exit_callback_callback = callback; + this.server.kill(); + } else { + callback(); + } + }, + + timers_are_valid: function (test) { + test.expect(6); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection',function(c){ + statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(payload){ + test.ok(payload.length > 0,'should receive some data'); + unpickle(payload, function(hashes) { + var numstat_test = function(post){ + var mykey = 'stats.statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(hashes,numstat_test), 'stats.statsd.numStats should be 5'); + + var testtimervalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.mean_90'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + var testtimerhistogramvalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be ' + 1); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); + + var count_test = function(post, metric){ + var mykey = 'stats.timers.a_test_value.' + metric; + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); + }; + test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); + test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); + + test.done(); + }); + }); + }); + }); + } +}; From f5b66c59f32c1cfcb39e594fc2f670d691264443 Mon Sep 17 00:00:00 2001 From: Aron Atkins Date: Fri, 10 Jul 2015 08:27:13 -0400 Subject: [PATCH 067/207] Fix variable names in test assertion message. --- test/graphite_tests.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/graphite_tests.js b/test/graphite_tests.js index ec6a13a7..67f5b9a4 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -219,12 +219,12 @@ module.exports = { var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; return _.include(_.keys(post),mykey) && (post[mykey] == 1); }; - test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.mean should be ' + 1); - test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean should be ' + testvalue); + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be ' + 1); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); var count_test = function(post, metric){ var mykey = 'stats.timers.a_test_value.' + metric; - return _.first(_.filter(_.pluck(post, mykey), function (e) { return e })); + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); }; test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); From 9ece81e85be5ee3c90c18cd7f387168265adbf25 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Wed, 22 Jul 2015 12:52:26 -0400 Subject: [PATCH 068/207] add a unit test for counters as floats this is just to clarify that counters can be floats. This might not make sense in the general case, but we are liberal in what we accept. And even if this ends up being a problem, we at least have a unit test to document it. --- test/helpers_tests.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/helpers_tests.js b/test/helpers_tests.js index cb769842..5dcd44aa 100644 --- a/test/helpers_tests.js +++ b/test/helpers_tests.js @@ -134,6 +134,12 @@ module.exports = { var res = helpers.is_valid_packet(['100', 'ms', '@0.1']); test.equals(res, true); test.done(); + }, + + correct_counter_with_float: function (test) { + var res = helpers.is_valid_packet(['1.0', 'c', '@0.1']); + test.equals(res, true); + test.done(); } }; From 45ee8884dcbeeebd123346c53f7e445d83df67e4 Mon Sep 17 00:00:00 2001 From: rolf Date: Wed, 29 Jul 2015 13:23:34 +0200 Subject: [PATCH 069/207] Adding systemd initscript --- debian/statsd.systemd | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 debian/statsd.systemd diff --git a/debian/statsd.systemd b/debian/statsd.systemd new file mode 100644 index 00000000..60a61263 --- /dev/null +++ b/debian/statsd.systemd @@ -0,0 +1,11 @@ +[Unit] +Description=monitoring daemon, that aggregates events received by udp in 10 second intervals +Documentation=https://github.com/etsy/statsd/ +Wants=network.target + +[Service] +Type=simple +User=_statsd +ExecStart=/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/localConfig.js + +[Install] From 04e90eed34936ac8e3349e684037fdf896d6c897 Mon Sep 17 00:00:00 2001 From: rolf Date: Wed, 29 Jul 2015 13:37:33 +0200 Subject: [PATCH 070/207] Adding statsd-proxy service --- debian/statsd-proxy.service | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 debian/statsd-proxy.service diff --git a/debian/statsd-proxy.service b/debian/statsd-proxy.service new file mode 100644 index 00000000..e138151a --- /dev/null +++ b/debian/statsd-proxy.service @@ -0,0 +1,11 @@ +[Unit] +Description=Network daemon for aggregating statistics +Documentation=https://github.com/etsy/statsd/ +Wants=network.target + +[Service] +Type=simple +User=_statsd +ExecStart=/usr/bin/nodejs /usr/share/statsd/proxy.js /etc/statsd/proxyConfig.js + +[Install] From f32e670fefeef2e50586895a8e64ec40cc096fd8 Mon Sep 17 00:00:00 2001 From: rolf Date: Wed, 29 Jul 2015 13:37:49 +0200 Subject: [PATCH 071/207] Renaming and changing desc --- debian/statsd.service | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 debian/statsd.service diff --git a/debian/statsd.service b/debian/statsd.service new file mode 100644 index 00000000..1e94f0d2 --- /dev/null +++ b/debian/statsd.service @@ -0,0 +1,11 @@ +[Unit] +Description=Network daemon for aggregating statistics +Documentation=https://github.com/etsy/statsd/ +Wants=network.target + +[Service] +Type=simple +User=_statsd +ExecStart=/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/localConfig.js + +[Install] From 74a1eb9f3d92f2fe6205048c394fb91c477ec3db Mon Sep 17 00:00:00 2001 From: rolf Date: Wed, 29 Jul 2015 13:44:36 +0200 Subject: [PATCH 072/207] Deleting statsd.systemd --- debian/statsd.systemd | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 debian/statsd.systemd diff --git a/debian/statsd.systemd b/debian/statsd.systemd deleted file mode 100644 index 60a61263..00000000 --- a/debian/statsd.systemd +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=monitoring daemon, that aggregates events received by udp in 10 second intervals -Documentation=https://github.com/etsy/statsd/ -Wants=network.target - -[Service] -Type=simple -User=_statsd -ExecStart=/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/localConfig.js - -[Install] From 9ee156dd04e245b11c12727ff79f876925275eb9 Mon Sep 17 00:00:00 2001 From: Andrew Dittes Date: Mon, 10 Aug 2015 15:00:37 -0700 Subject: [PATCH 073/207] add statsd-jut-backend --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 52e092f8..97e591d5 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -41,6 +41,7 @@ queues and third-party services. - [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) - [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) - [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) +- [jut-backend](https://github.com/jut-io/statsd-jut-backend) - [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) - [librato-backend](https://github.com/librato/statsd-librato-backend) - [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) From 19e5b5a36e90345b84835eb504e2b1b7e6b03e0d Mon Sep 17 00:00:00 2001 From: hit9 Date: Sat, 29 Aug 2015 16:00:31 +0800 Subject: [PATCH 074/207] Upgrade hashring to 3.2 (pure js version) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bc62e0cd..bd423eba 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ }, "optionalDependencies": { "node-syslog":"1.2.0", - "hashring":"3.1.0", + "hashring":"3.2.0", "winser": "=0.1.6" }, "engines": { From 132c59357e6d06836f0343e92cbf96509f207314 Mon Sep 17 00:00:00 2001 From: Bartosz Cisek Date: Fri, 25 Sep 2015 15:40:48 +0200 Subject: [PATCH 075/207] partable bash shebang --- examples/statsd-client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/statsd-client.sh b/examples/statsd-client.sh index 9b236cc1..51b2bf98 100755 --- a/examples/statsd-client.sh +++ b/examples/statsd-client.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Very simple bash client to send metrics to a statsd server # Example with gauge: ./statsd-client.sh 'my_metric:100|g' From ef92150159ebd02909c08aa8b8463ed62defbb07 Mon Sep 17 00:00:00 2001 From: till Date: Fri, 23 Oct 2015 02:06:00 +0200 Subject: [PATCH 076/207] Fix: only install production dependencies --- debian/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/postinst b/debian/postinst index ed3b4bba..31efff95 100755 --- a/debian/postinst +++ b/debian/postinst @@ -4,7 +4,7 @@ set -e if [ "$1" = configure ]; then # Automatically added by dh_installinit - (cd /usr/share/statsd && /usr/bin/npm install) + (cd /usr/share/statsd && /usr/bin/npm install --production) if ! getent passwd _statsd > /dev/null; then adduser --system --quiet --home /nonexistent --no-create-home \ From f5bd683bebaa3a5d142fab0be896f5aed8021e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Flaud=C3=ADsio=20Tolentino?= Date: Fri, 23 Oct 2015 12:34:17 -0200 Subject: [PATCH 077/207] Minor font style improvements --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed176338..23ac959a 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,10 @@ Installation and Configuration * Install node.js * Clone the project - * Create a config file from exampleConfig.js and put it somewhere - * Start the Daemon: + * Create a config file from `exampleConfig.js` and put it somewhere + * Start the Daemon: + `node stats.js /path/to/config` - node stats.js /path/to/config - Usage ------- The basic line protocol expects metrics to be sent in the format: From d550cad3f393b74f92576e14f740944bb47a2e73 Mon Sep 17 00:00:00 2001 From: Peter Peresini Date: Sat, 7 Nov 2015 00:08:59 +0100 Subject: [PATCH 078/207] Fix date in debian changelog Otherwise we get a perl date parsing error while trying to do dpkg-buildpackage --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 7101d071..6f5ac1f0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,7 +7,7 @@ statsd (0.7.2) unstable; urgency=low * Remove init script that does not work in favor of just having upstart * Fix node version dependency - -- Joseph Hughes Thurs, 16 Apr 2014 13:03:10 +0200 + -- Joseph Hughes Thu, 16 Apr 2014 13:03:10 +0200 statsd (0.6.0-1) unstable; urgency=low From fb87925ad60f26c1cd7222eaf94324c39e9e432c Mon Sep 17 00:00:00 2001 From: Bertrand Paquet Date: Wed, 11 Nov 2015 20:53:35 +0100 Subject: [PATCH 079/207] Force statsd restart when updating package --- packager/postinst | 1 + 1 file changed, 1 insertion(+) diff --git a/packager/postinst b/packager/postinst index 6b5ade03..3ba76a2d 100755 --- a/packager/postinst +++ b/packager/postinst @@ -13,4 +13,5 @@ chown $APP_USER.$APP_GROUP $APP_CONFIG ln -f -s $APP_CONFIG /opt/$APP_NAME/config.js chmod 0640 $APP_CONFIG +${CLI} scale web=0 || true ${CLI} scale web=1 || true From 493e2ca4269a1351ff9fc655b45f5b1c0337ce78 Mon Sep 17 00:00:00 2001 From: Bertrand Paquet Date: Wed, 11 Nov 2015 20:54:34 +0100 Subject: [PATCH 080/207] Remove all default dependencies in generated package Remove all default dependencies in generated package like mysql, postgres, libxml ... --- .pkgr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pkgr.yml b/.pkgr.yml index b4acb42b..b412d0d2 100644 --- a/.pkgr.yml +++ b/.pkgr.yml @@ -1,3 +1,4 @@ +default_dependencies: false targets: ubuntu-14.04: ubuntu-12.04: From 87908bf56c528e42daab3d8059ed887e0a3c0a15 Mon Sep 17 00:00:00 2001 From: mpsss Date: Wed, 18 Nov 2015 13:41:19 +0800 Subject: [PATCH 081/207] change subconfig(at statsd.js 330:15) to local variable --- stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats.js b/stats.js index 9e10f1a8..69f7d299 100644 --- a/stats.js +++ b/stats.js @@ -315,7 +315,7 @@ config.configFile(process.argv[2], function (config) { stream.write(prop + ": " + config[prop] + "\n"); continue; } - subconfig = config[prop]; + var subconfig = config[prop]; for (var subprop in subconfig) { if (!subconfig.hasOwnProperty(subprop)) { continue; From 8c267f3812b8de30f3dc741d3377aad86f5616a8 Mon Sep 17 00:00:00 2001 From: Martin Camitz Date: Fri, 4 Dec 2015 19:33:37 +0100 Subject: [PATCH 082/207] Added aws-cloudwatch-backend to list --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 97e591d5..cba05480 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -33,6 +33,7 @@ queues and third-party services. ## Available Third-party backends - [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) - [atsd-backend](https://github.com/axibase/atsd-statsd-backend) +- [aws-cloudwatch-backend](https://github.com/camitz/aws-cloudwatch-statsd-backend) - [node-bell](https://github.com/eleme/node-bell) - [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) - [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) From 97ffdcb8d6f049033356d7e8c2bb7eea80aab481 Mon Sep 17 00:00:00 2001 From: AQNOUCH Mohammed Date: Fri, 1 Jan 2016 01:29:47 +0000 Subject: [PATCH 083/207] Updated copyright to 2016 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4199459a..f86a9a68 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2010-2014 Etsy + Copyright (c) 2010-2016 Etsy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation From 9d417b0c083f87b2204addaac88a93ea83d8abef Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 10:16:34 +0100 Subject: [PATCH 084/207] systemd improvements * Allow to manually enable the systemd service to autostart at boot * Added a restart on failure with a delay to keep the service running --- debian/statsd-proxy.service | 3 +++ debian/statsd.service | 3 +++ 2 files changed, 6 insertions(+) diff --git a/debian/statsd-proxy.service b/debian/statsd-proxy.service index e138151a..a832b061 100644 --- a/debian/statsd-proxy.service +++ b/debian/statsd-proxy.service @@ -7,5 +7,8 @@ Wants=network.target Type=simple User=_statsd ExecStart=/usr/bin/nodejs /usr/share/statsd/proxy.js /etc/statsd/proxyConfig.js +Restart=on-failure +RestartSec=10 [Install] +WantedBy=multi-user.target diff --git a/debian/statsd.service b/debian/statsd.service index 1e94f0d2..c672fcaf 100644 --- a/debian/statsd.service +++ b/debian/statsd.service @@ -7,5 +7,8 @@ Wants=network.target Type=simple User=_statsd ExecStart=/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/localConfig.js +Restart=on-failure +RestartSec=10 [Install] +WantedBy=multi-user.target From f23313c481b722532d95512a6709d2e2e938d077 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 15:25:29 +0100 Subject: [PATCH 085/207] Fix systemd build * Added required build dependency * Added systemd option to dh * Renamed systemd service files to follow systemd standards --- debian/control | 2 +- debian/rules | 2 +- debian/{statsd-proxy.service => statsd.statsd-proxy.service} | 0 debian/{statsd.service => statsd.statsd.service} | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename debian/{statsd-proxy.service => statsd.statsd-proxy.service} (100%) rename debian/{statsd.service => statsd.statsd.service} (100%) diff --git a/debian/control b/debian/control index c19608f4..03e84f1b 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: devel Priority: optional Maintainer: Stephen Koenig Standards-Version: 3.9.3 -Build-Depends: debhelper (>= 8.0.0) +Build-Depends: debhelper (>= 8.0.0), dh-systemd (>= 1.5) Package: statsd Architecture: all diff --git a/debian/rules b/debian/rules index 7e0e37ad..88df0139 100755 --- a/debian/rules +++ b/debian/rules @@ -1,7 +1,7 @@ #!/usr/bin/make -f %: - dh $@ + dh $@ --with systemd override_dh_installinit: dh_installinit --name=statsd -- defaults diff --git a/debian/statsd-proxy.service b/debian/statsd.statsd-proxy.service similarity index 100% rename from debian/statsd-proxy.service rename to debian/statsd.statsd-proxy.service diff --git a/debian/statsd.service b/debian/statsd.statsd.service similarity index 100% rename from debian/statsd.service rename to debian/statsd.statsd.service From 3fc4a58c32d06dd818602625f58e10db78687f0a Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 15:26:36 +0100 Subject: [PATCH 086/207] Added message after installation * Added a message post-installation that remember the user to enable the statsd or statsd-proxy services if they need to be started at boot. --- debian/postinst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/postinst b/debian/postinst index 31efff95..9e1c11c7 100755 --- a/debian/postinst +++ b/debian/postinst @@ -15,4 +15,13 @@ if [ "$1" = configure ]; then dpkg-statoverride --update --add _statsd _statsd 0755 /var/run/statsd fi + cat << EOF +Available systemd services are disabled by default: + * statsd + * statsd-proxy + +To enable one of them use: + systemctl enable NAME +EOF + fi From 74f875ee3dcdf7cba08254896403ef3d05e251aa Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Tue, 2 Feb 2016 15:43:49 +0100 Subject: [PATCH 087/207] Allow to use unix sockets in tcp servers. * Add socket config option to specify path where socket will be stored. * Add socket_mod config option to set correct permission on socket file. * Remove socket on process exit. --- servers/tcp.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index 8e55d0de..489b0942 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -1,4 +1,5 @@ var net = require('net'); +var fs = require('fs'); function rinfo(tcpstream, data) { this.address = tcpstream.remoteAddress; @@ -23,8 +24,15 @@ exports.start = function(config, callback) { }); }); - server.listen(config.port || 8125, config.address || undefined); - this.server = server; + server.on('listening', (e) => { + config.socket && config.socket_mod && fs.chmod(config.socket, config.socket_mod); + }); + process.on('exit', (e) => { + config.socket && fs.unlinkSync(config.socket); + }) + + server.listen(config.socket || config.port || 8125, config.address || undefined); + this.server = server; return true; }; From e722d92c140e3be3d5e9171ff51b74f47522c27f Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Tue, 2 Feb 2016 16:17:39 +0100 Subject: [PATCH 088/207] Document `socket` and `socket_mod` server options. --- exampleConfig.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/exampleConfig.js b/exampleConfig.js index 2a6fca06..913630a8 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -29,6 +29,10 @@ Optional Variables: address: address to listen on [default: 0.0.0.0] address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] port: port to listen for messages on [default: 8125] + socket: (only for tcp servers) path to unix domain socket which will be used to receive + metrics [default: undefinded] + socket_mod: (only for tcp servers) file mode which should be applied to unix domain socket, relevant + only if socket option is used [default: undefined] debug: debug flag [default: false] mgmt_address: address to run the management TCP interface on From 5e74003d83df962d8e0404cbdbaea702e4ad9298 Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Tue, 2 Feb 2016 17:09:28 +0100 Subject: [PATCH 089/207] Do not use lambda expression. --- servers/tcp.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index 489b0942..8427e794 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -24,13 +24,13 @@ exports.start = function(config, callback) { }); }); - server.on('listening', (e) => { + server.on('listening', function() { config.socket && config.socket_mod && fs.chmod(config.socket, config.socket_mod); }); - process.on('exit', (e) => { + process.on('exit', function() { config.socket && fs.unlinkSync(config.socket); - }) + }); server.listen(config.socket || config.port || 8125, config.address || undefined); this.server = server; From 1e0c5cbd8056345d629c780c7d23afc77f57d253 Mon Sep 17 00:00:00 2001 From: Gabor Almer Date: Tue, 9 Feb 2016 11:01:24 +0100 Subject: [PATCH 090/207] add newer nodejs versions to travis --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ff67195..b031a5ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ language: node_js node_js: -- '0.10' -- '0.12' + - '0.10' + - '0.12' + - '4' + - 'stable' script: ./run_tests.sh notifications: email: false From b4d21f1107308da0aa0bdf3c6585bf5210e9ee6c Mon Sep 17 00:00:00 2001 From: Gabor Almer Date: Tue, 9 Feb 2016 11:01:59 +0100 Subject: [PATCH 091/207] use newer nodeunit version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bd423eba..793c717a 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "generic-pool": "2.2.0" }, "devDependencies": { - "nodeunit": "0.7.x", + "nodeunit": "0.9.x", "underscore": "1.4.x", "temp": "0.4.x" }, From bf3da7464b339818bd56acc88ec859037e664bed Mon Sep 17 00:00:00 2001 From: Gabor Almer Date: Tue, 9 Feb 2016 11:03:59 +0100 Subject: [PATCH 092/207] use modern-syslog instead of node-syslog, because node-syslog is not supported anymore --- lib/logger.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index fea607b9..1f102673 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -8,7 +8,7 @@ var Logger = function (config) { this.util = require('util'); } else { if (this.backend == 'syslog') { - this.util = require('node-syslog'); + this.util = require('modern-syslog'); this.util.init(config.application || 'statsd', this.util.LOG_PID | this.util.LOG_ODELAY, this.util.LOG_LOCAL0); } else { throw "Logger: Should be 'stdout' or 'syslog'."; diff --git a/package.json b/package.json index 793c717a..6023aa70 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "temp": "0.4.x" }, "optionalDependencies": { - "node-syslog":"1.2.0", + "modern-syslog":"1.1.2", "hashring":"3.2.0", "winser": "=0.1.6" }, From 0a20119ad8c75f81dde7916b3b25e56027f21ec3 Mon Sep 17 00:00:00 2001 From: Tyler Nisonoff Date: Fri, 12 Feb 2016 18:04:29 -0800 Subject: [PATCH 093/207] Allow the proxy to listen over TCP --- exampleProxyConfig.js | 7 +++++++ proxy.js | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/exampleProxyConfig.js b/exampleProxyConfig.js index fcc8e98e..951fb767 100644 --- a/exampleProxyConfig.js +++ b/exampleProxyConfig.js @@ -15,6 +15,12 @@ Optional Variables: checkInterval: health status check interval [default: 10000] cacheSize: size of the cache to store for hashring key lookups [default: 10000] forkCount: number of child processes (cluster module), number or 'auto' for utilize all cpus [default:0] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + Note: This will still send to the backends via udp regardless of the + server type for the proxy + * example for tcp server: + "./servers/tcp" */ { @@ -23,6 +29,7 @@ nodes: [ {host: '127.0.0.1', port: 8129, adminport: 8130}, {host: '127.0.0.1', port: 8131, adminport: 8132} ], +server: './servers/udp', udp_version: 'udp4', host: '0.0.0.0', port: 8125, diff --git a/proxy.js b/proxy.js index 7053f6b3..507aa941 100644 --- a/proxy.js +++ b/proxy.js @@ -62,7 +62,10 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { // Setup the udp listener - var server = dgram.createSocket(udp_version, function (msg, rinfo) { + // + var server_config = config.server || './servers/udp' + var servermod = require(server_config) + var server = servermod.start(config, function (msg, rinfo) { // Convert the raw packet to a string (defaults to UTF8 encoding) var packet_data = msg.toString(); // If the packet contains a \n then it contains multiple metrics @@ -108,9 +111,6 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { } }); - // Bind the listening udp server to the configured port and host - server.bind(config.port, config.host || undefined); - // Set the interval for healthchecks setInterval(doHealthChecks, config.checkInterval || 10000); From dd0b7ee2958f764a18a604010fa5f33819bc8015 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Fri, 18 Mar 2016 18:26:40 +0000 Subject: [PATCH 094/207] proxy now uses config.address_ipv6 for ip version configuration related to pr #566 --- exampleProxyConfig.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exampleProxyConfig.js b/exampleProxyConfig.js index 951fb767..ef7f26b1 100644 --- a/exampleProxyConfig.js +++ b/exampleProxyConfig.js @@ -10,8 +10,8 @@ Required Variables: Optional Variables: - udp_version: defines if the address is an IPv4 or IPv6 address ['udp4' or 'udp6', default: 'udp4'] host: address to listen on over UDP [default: 0.0.0.0] + address_ipv6: defines if the listen address is an IPv4 or IPv6 address [true or false, default: false] checkInterval: health status check interval [default: 10000] cacheSize: size of the cache to store for hashring key lookups [default: 10000] forkCount: number of child processes (cluster module), number or 'auto' for utilize all cpus [default:0] @@ -30,7 +30,7 @@ nodes: [ {host: '127.0.0.1', port: 8131, adminport: 8132} ], server: './servers/udp', -udp_version: 'udp4', + host: '0.0.0.0', port: 8125, forkCount: 0, From 102bf5e56a8b9f61d89aec306059f61490d95213 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 10:28:49 +0100 Subject: [PATCH 095/207] Fix JSHint warnings Fix JSHint warnings --- lib/helpers.js | 2 +- proxy.js | 19 ++++++++++++------- stats.js | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index 52ba3a3f..c0ab9646 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -49,6 +49,6 @@ function is_valid_packet(fields) { return true; } -}; +} exports.is_valid_packet = is_valid_packet; diff --git a/proxy.js b/proxy.js index 507aa941..791bbadf 100644 --- a/proxy.js +++ b/proxy.js @@ -1,3 +1,5 @@ +/*jshint node:true, laxcomma:true */ + var dgram = require('dgram') , net = require('net') , events = require('events') @@ -26,7 +28,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { var logPrefix = "[" + process.pid + "] "; var log = function(msg, type) { l.log(logPrefix + msg, type); - } + }; if (forkCount > 1 && cluster.isMaster) { @@ -68,15 +70,18 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { var server = servermod.start(config, function (msg, rinfo) { // Convert the raw packet to a string (defaults to UTF8 encoding) var packet_data = msg.toString(); + var current_metric + , bits + , key; // If the packet contains a \n then it contains multiple metrics if (packet_data.indexOf("\n") > -1) { var metrics; metrics = packet_data.split("\n"); // Loop through the metrics and split on : to get mertric name for hashing for (var midx in metrics) { - var current_metric = metrics[midx]; - var bits = current_metric.split(':'); - var key = bits.shift(); + current_metric = metrics[midx]; + bits = current_metric.split(':'); + key = bits.shift(); if (current_metric !== '') { var new_msg = new Buffer(current_metric); packet.emit('send', key, new_msg); @@ -85,9 +90,9 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { } else { // metrics needs to be an array to fake it for single metric packets - var current_metric = packet_data; - var bits = current_metric.split(':'); - var key = bits.shift(); + current_metric = packet_data; + bits = current_metric.split(':'); + key = bits.shift(); if (current_metric !== '') { packet.emit('send', key, msg); } diff --git a/stats.js b/stats.js index 9e10f1a8..a6a0b479 100644 --- a/stats.js +++ b/stats.js @@ -206,11 +206,12 @@ config.configFile(process.argv[2], function (config) { var handlePacket = function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; + var metrics; var packet_data = msg.toString(); if (packet_data.indexOf("\n") > -1) { - var metrics = packet_data.split("\n"); + metrics = packet_data.split("\n"); } else { - var metrics = [ packet_data ] ; + metrics = [ packet_data ] ; } for (var midx in metrics) { @@ -278,16 +279,16 @@ config.configFile(process.argv[2], function (config) { } stats.messages.last_msg_seen = Math.round(new Date().getTime() / 1000); - } + }; // If config.servers isn't specified, use the top-level config for backwards-compatibility - var server_config = config.servers || [config] + var server_config = config.servers || [config]; for (var i = 0; i < server_config.length; i++) { // The default server is UDP - var server = server_config[i].server || './servers/udp' - startServer(server_config[i], server, handlePacket) + var server = server_config[i].server || './servers/udp'; + startServer(server_config[i], server, handlePacket); } - serversLoaded = true + serversLoaded = true; mgmtServer = net.createServer(function(stream) { stream.setEncoding('ascii'); @@ -431,8 +432,8 @@ config.configFile(process.argv[2], function (config) { config.flushInterval = flushInterval; if (config.backends) { - for (var i = 0; i < config.backends.length; i++) { - loadBackend(config, config.backends[i]); + for (var j = 0; j < config.backends.length; j++) { + loadBackend(config, config.backends[j]); } } else { // The default backend is graphite From 2cbcf2096282e79ba69b981f7ffe31d33e4ed411 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 14:51:29 +0100 Subject: [PATCH 096/207] Allow additional log levels for syslog * When using syslog, automatically translates stdout types to syslog levels. * Added some levels to existing logs. --- lib/logger.js | 17 ++++++++++------- proxy.js | 12 ++++++------ stats.js | 6 +++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 1f102673..56e27ee8 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -24,15 +24,18 @@ Logger.prototype = { } this.util.log(type + ": " + msg); } else { + var level; if (!type) { - type = this.level; - if (!this.util[type]) { - throw "Undefined log level: " + type; - } - } else if (type == 'debug') { - type = "LOG_DEBUG"; + level = this.level; + } else { + level = "LOG_" + type.toUpperCase(); } - this.util.log(this.util[type], msg); + + if (!this.util[level]) { + throw "Undefined log level: " + level; + } + + this.util.log(this.util[level], msg); } } }; diff --git a/proxy.js b/proxy.js index 791bbadf..8b8acc0c 100644 --- a/proxy.js +++ b/proxy.js @@ -33,7 +33,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { if (forkCount > 1 && cluster.isMaster) { logPrefix += "[master] "; - log("forking " + forkCount + " childs"); + log("forking " + forkCount + " childs", "INFO"); for (var i = 0; i < forkCount; i++) { cluster.fork(); @@ -107,7 +107,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { // break the retreived host to pass to the send function if (statsd_host === undefined) { - log('Warning: No backend statsd nodes available!'); + log('Warning: No backend statsd nodes available!', 'WARNING'); } else { var host_config = statsd_host.split(':'); @@ -143,7 +143,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { node_status[node_id]++; } if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.'); + log('Removing node ' + node_id + ' from the ring.', 'WARNING'); ring.remove(node_id); } } else { @@ -151,7 +151,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { if (node_status[node_id] > 0) { var new_server = {}; new_server[node_id] = 100; - log('Adding node ' + node_id + ' to the ring.'); + log('Adding node ' + node_id + ' to the ring.', 'WARNING'); ring.add(new_server); } } @@ -166,11 +166,11 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { node_status[node_id]++; } if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.'); + log('Removing node ' + node_id + ' from the ring.', 'WARNING'); ring.remove(node_id); } } else { - log('Error during healthcheck on node ' + node_id + ' with ' + e.code); + log('Error during healthcheck on node ' + node_id + ' with ' + e.code, 'ERROR'); } }); } diff --git a/stats.js b/stats.js index a6a0b479..02fbdebb 100644 --- a/stats.js +++ b/stats.js @@ -41,7 +41,7 @@ function loadBackend(config, name) { var ret = backendmod.init(startup_time, config, backendEvents, l); if (!ret) { - l.log("Failed to load backend: " + name); + l.log("Failed to load backend: " + name, "ERROR"); process.exit(1); } } @@ -60,7 +60,7 @@ function startServer(config, name, callback) { var ret = servermod.start(config, callback); if (!ret) { - l.log("Failed to load server: " + name); + l.log("Failed to load server: " + name, "ERROR"); process.exit(1); } } @@ -421,7 +421,7 @@ config.configFile(process.argv[2], function (config) { mgmtServer.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); - util.log("server is up"); + util.log("server is up", "INFO"); pctThreshold = config.percentThreshold || 90; if (!Array.isArray(pctThreshold)) { From c83a663f040143d9a44bc537dba7eb08bcbc9813 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 14:56:22 +0100 Subject: [PATCH 097/207] Management server for statsd-proxy * Extracted the management server into lib/mgmt_server.js * Moved the input handling into the management server * Extracted the write configuration into lib/helpers.js * Added the management server to proxy.js * Refactored the management server in stats.js * Handling the health status change across all forks --- lib/config.js | 3 +- lib/helpers.js | 20 +++++ lib/mgmt_server.js | 24 +++++ proxy.js | 214 ++++++++++++++++++++++++++++++++------------- stats.js | 56 ++++-------- 5 files changed, 216 insertions(+), 101 deletions(-) create mode 100644 lib/mgmt_server.js diff --git a/lib/config.js b/lib/config.js index b61bf3b7..7895ec37 100644 --- a/lib/config.js +++ b/lib/config.js @@ -10,7 +10,7 @@ var Configurator = function (file) { var oldConfig = {}; this.updateConfig = function () { - util.log('reading config file: ' + file); + util.log('[' + process.pid + '] reading config file: ' + file); fs.readFile(file, function (err, data) { if (err) { throw err; } @@ -40,4 +40,3 @@ exports.configFile = function(file, callbackFunc) { callbackFunc(config.config, config.oldConfig); }); }; - diff --git a/lib/helpers.js b/lib/helpers.js index c0ab9646..e2686554 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -52,3 +52,23 @@ function is_valid_packet(fields) { } exports.is_valid_packet = is_valid_packet; + +exports.writeConfig = function(config, stream) { + stream.write("\n"); + for (var prop in config) { + if (!config.hasOwnProperty(prop)) { + continue; + } + if (typeof config[prop] !== 'object') { + stream.write(prop + ": " + config[prop] + "\n"); + continue; + } + subconfig = config[prop]; + for (var subprop in subconfig) { + if (!subconfig.hasOwnProperty(subprop)) { + continue; + } + stream.write(prop + " > " + subprop + ": " + subconfig[subprop] + "\n"); + } + } +}; diff --git a/lib/mgmt_server.js b/lib/mgmt_server.js new file mode 100644 index 00000000..c89e32bc --- /dev/null +++ b/lib/mgmt_server.js @@ -0,0 +1,24 @@ +/*jshint node:true, laxcomma:true */ + +var net = require('net'); + +exports.start = function(config, on_data_callback, on_error_callback) { + var server = net.createServer(function(stream) { + stream.setEncoding('ascii'); + + stream.on('data', function(data) { + var cmdline = data.trim().split(" "); + var cmd = cmdline.shift(); + + on_data_callback(cmd, cmdline, stream); + }); + + stream.on('error', function(err) { + on_error_callback(err, stream); + }); + }); + + server.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); + + return true; +}; diff --git a/proxy.js b/proxy.js index 8b8acc0c..5dddca02 100644 --- a/proxy.js +++ b/proxy.js @@ -6,11 +6,16 @@ var dgram = require('dgram') , logger = require('./lib/logger') , hashring = require('hashring') , cluster = require('cluster') + , helpers = require('./lib/helpers') + , mgmt_server = require('./lib/mgmt_server') , configlib = require('./lib/config'); var packet = new events.EventEmitter(); +var startup_time = Math.round(new Date().getTime() / 1000); var node_status = []; +var workers = []; // Keep track of all forked childs var node_ring = {}; +var servers_loaded; var config; var l; // logger @@ -30,20 +35,52 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { l.log(logPrefix + msg, type); }; + var healthStatus = configlib.healthStatus || 'up'; + var broadcastMsg = function(msg) { + for (var i = 0; i < workers.length; i++) { + workers[i].send(msg); + } + }; - if (forkCount > 1 && cluster.isMaster) { - logPrefix += "[master] "; - log("forking " + forkCount + " childs", "INFO"); + if (forkCount > 1) { + if (cluster.isMaster) { + var worker; + logPrefix += "[master] "; + log("forking " + forkCount + " childs", "INFO"); - for (var i = 0; i < forkCount; i++) { - cluster.fork(); - } + for (var i = 0; i < forkCount; i++) { + worker = cluster.fork(); + worker.on('message', broadcastMsg); + } - cluster.on('exit', function(worker, code, signal) { - log('worker ' + worker.process.pid + ' died with exit code:' + code + " restarting", 'ERROR'); - cluster.fork(); - }); - return; + cluster.on('online', function(worker) { + log('worker ' + worker.process.pid + ' is online', 'INFO'); + workers.push(worker); + }); + + cluster.on('exit', function(worker, code, signal) { + log('worker ' + worker.process.pid + ' died with exit code:' + code + " restarting", 'ERROR'); + + // Remove died worker from the array + for (var i = 0; i < workers.length; i++) { + if (workers[i].process.pid == worker.process.pid) { + workers.splice(i, 1); + } + } + + worker = cluster.fork(); + worker.on('message', broadcastMsg); + }); + + return; + + } else { + process.on('message', function(msg) { + if (msg.healthStatus) { + healthStatus = msg.healthStatus; + } + }); + } } //load the node_ring object with the available nodes and a weight of 100 @@ -59,65 +96,124 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { 'replicas': 0 }); - // Do an initial rount of health checks prior to starting up the server - doHealthChecks(); - - - // Setup the udp listener - // - var server_config = config.server || './servers/udp' - var servermod = require(server_config) - var server = servermod.start(config, function (msg, rinfo) { - // Convert the raw packet to a string (defaults to UTF8 encoding) - var packet_data = msg.toString(); - var current_metric - , bits - , key; - // If the packet contains a \n then it contains multiple metrics - if (packet_data.indexOf("\n") > -1) { - var metrics; - metrics = packet_data.split("\n"); - // Loop through the metrics and split on : to get mertric name for hashing - for (var midx in metrics) { - current_metric = metrics[midx]; + if (!servers_loaded) { + // Do an initial rount of health checks prior to starting up the server + doHealthChecks(); + + // Setup the udp listener + var server_config = config.server || './servers/udp'; + var servermod = require(server_config); + var server = servermod.start(config, function (msg, rinfo) { + // Convert the raw packet to a string (defaults to UTF8 encoding) + var packet_data = msg.toString(); + var current_metric + , bits + , key; + // If the packet contains a \n then it contains multiple metrics + if (packet_data.indexOf("\n") > -1) { + var metrics; + metrics = packet_data.split("\n"); + // Loop through the metrics and split on : to get mertric name for hashing + for (var midx in metrics) { + current_metric = metrics[midx]; + bits = current_metric.split(':'); + key = bits.shift(); + if (current_metric !== '') { + var new_msg = new Buffer(current_metric); + packet.emit('send', key, new_msg); + } + } + + } else { + // metrics needs to be an array to fake it for single metric packets + current_metric = packet_data; bits = current_metric.split(':'); key = bits.shift(); if (current_metric !== '') { - var new_msg = new Buffer(current_metric); - packet.emit('send', key, new_msg); + packet.emit('send', key, msg); } } + }); - } else { - // metrics needs to be an array to fake it for single metric packets - current_metric = packet_data; - bits = current_metric.split(':'); - key = bits.shift(); - if (current_metric !== '') { - packet.emit('send', key, msg); + var client = dgram.createSocket(udp_version); + // Listen for the send message, and process the metric key and msg + packet.on('send', function(key, msg) { + // retreives the destination for this key + var statsd_host = ring.get(key); + + // break the retreived host to pass to the send function + if (statsd_host === undefined) { + log('Warning: No backend statsd nodes available!', 'WARNING'); + } else { + var host_config = statsd_host.split(':'); + + // Send the mesg to the backend + client.send(msg, 0, msg.length, host_config[1], host_config[0]); } - } - }); + }); - var client = dgram.createSocket(udp_version); - // Listen for the send message, and process the metric key and msg - packet.on('send', function(key, msg) { - // retreives the destination for this key - var statsd_host = ring.get(key); + // Bind the listening udp server to the configured port and host + server.bind(config.port, config.host || undefined); - // break the retreived host to pass to the send function - if (statsd_host === undefined) { - log('Warning: No backend statsd nodes available!', 'WARNING'); - } else { - var host_config = statsd_host.split(':'); + mgmt_server.start( + config, + function(cmd, parameters, stream) { + switch(cmd) { + case "help": + stream.write("Commands: config, health, status, quit\n\n"); + break; - // Send the mesg to the backend - client.send(msg, 0, msg.length, host_config[1], host_config[0]); - } - }); + case "config": + helpers.writeConfig(config, stream); + break; - // Set the interval for healthchecks - setInterval(doHealthChecks, config.checkInterval || 10000); + case "health": + if (parameters.length > 0) { + var cmdaction = parameters[0].toLowerCase(); + if (cmdaction === 'up') { + healthStatus = 'up'; + process.send({ healthStatus: healthStatus }); + } else if (cmdaction === 'down') { + healthStatus = 'down'; + process.send({ healthStatus: healthStatus }); + } + } + stream.write("health: " + healthStatus + "\n"); + break; + + case "status": + var now = Math.round(new Date().getTime() / 1000); + var uptime = now - startup_time; + + stream.write("uptime: " + uptime + "\n"); + + stream.write("nodes: "); + ring.servers.forEach(function(server, index, array) { + stream.write(server.string + " "); + }); + stream.write("\n"); + break; + + case "quit": + stream.end(); + break; + + default: + stream.write("ERROR\n"); + break; + } + }, + function(err, stream) { + log("MGMT: Caught " + err + ", Moving on", "WARNING"); + } + ); + + servers_loaded = true; + log("server is up", "INFO"); + + // Set the interval for healthchecks + setInterval(doHealthChecks, config.checkInterval || 10000); + } // Perform health check on all nodes function doHealthChecks() { diff --git a/stats.js b/stats.js index 02fbdebb..157313e6 100644 --- a/stats.js +++ b/stats.js @@ -1,7 +1,6 @@ /*jshint node:true, laxcomma:true */ var util = require('util') - , net = require('net') , config = require('./lib/config') , helpers = require('./lib/helpers') , fs = require('fs') @@ -10,6 +9,7 @@ var util = require('util') , set = require('./lib/set') , pm = require('./lib/process_metrics') , process_mgmt = require('./lib/process_mgmt') + , mgmt_server = require('./lib/mgmt_server') , mgmt = require('./lib/mgmt_console'); @@ -288,47 +288,22 @@ config.configFile(process.argv[2], function (config) { var server = server_config[i].server || './servers/udp'; startServer(server_config[i], server, handlePacket); } - serversLoaded = true; - - mgmtServer = net.createServer(function(stream) { - stream.setEncoding('ascii'); - - stream.on('error', function(err) { - l.log('Caught ' + err +', Moving on'); - }); - - stream.on('data', function(data) { - var cmdline = data.trim().split(" "); - var cmd = cmdline.shift(); + mgmt_server.start( + config, + function(cmd, parameters, stream) { switch(cmd) { case "help": stream.write("Commands: stats, counters, timers, gauges, delcounters, deltimers, delgauges, health, config, quit\n\n"); break; case "config": - stream.write("\n"); - for (var prop in config) { - if (!config.hasOwnProperty(prop)) { - continue; - } - if (typeof config[prop] !== 'object') { - stream.write(prop + ": " + config[prop] + "\n"); - continue; - } - subconfig = config[prop]; - for (var subprop in subconfig) { - if (!subconfig.hasOwnProperty(subprop)) { - continue; - } - stream.write(prop + " > " + subprop + ": " + subconfig[subprop] + "\n"); - } - } + helpers.writeConfig(config, stream); break; case "health": - if (cmdline.length > 0) { - var cmdaction = cmdline[0].toLowerCase(); + if (parameters.length > 0) { + var cmdaction = parameters[0].toLowerCase(); if (cmdaction === 'up') { healthStatus = 'up'; } else if (cmdaction === 'down') { @@ -396,15 +371,15 @@ config.configFile(process.argv[2], function (config) { break; case "delcounters": - mgmt.delete_stats(counters, cmdline, stream); + mgmt.delete_stats(counters, parameters, stream); break; case "deltimers": - mgmt.delete_stats(timers, cmdline, stream); + mgmt.delete_stats(timers, parameters, stream); break; case "delgauges": - mgmt.delete_stats(gauges, cmdline, stream); + mgmt.delete_stats(gauges, parameters, stream); break; case "quit": @@ -415,12 +390,13 @@ config.configFile(process.argv[2], function (config) { stream.write("ERROR\n"); break; } + }, + function(err, stream) { + l.log('MGMT: Caught ' + err +', Moving on', 'WARNING'); + } + ); - }); - }); - - mgmtServer.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); - + serversLoaded = true; util.log("server is up", "INFO"); pctThreshold = config.percentThreshold || 90; From e9f59f09aa8b32ebcd6ef02b002d319a65aaca74 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 14:56:38 +0100 Subject: [PATCH 098/207] Updated documentation * Updated documentation for the admin interface. --- docs/admin_interface.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/admin_interface.md b/docs/admin_interface.md index 853d5a00..8c2b84d0 100644 --- a/docs/admin_interface.md +++ b/docs/admin_interface.md @@ -1,20 +1,29 @@ TCP Stats Interface -------------------- +=================== A really simple TCP management interface is available by default on port 8126 or overriden in the configuration file. Inspired by the memcache stats approach this can be used to monitor a live statsd server. You can interact with the management server by telnetting to port 8126, the following commands are -available: +available based on the running server. + +Common commands +--------------- + +* health [up|down] - a way to get/set the health status of statsd. Alone will get you the current health status. Passing a second command will set the status to the new value. Accepted values are _up_ and _down_. +* config - a dump of the current configuration +* quit - close the connection from the server side + +Statsd specific commands +------------------------ * stats - some stats about the running server * counters - a dump of all the current counters * gauges - a dump of all the current gauges * timers - a dump of the current timers * delcounters - delete a counter or folder of counters -* delgauges - delete a gauge or folder of gauges +* delgauges - delete a gauge or folder of gauges * deltimers - delete a timer or folder of timers -* health - a way to set the health status of statsd The stats output currently will give you: @@ -31,7 +40,7 @@ Or you can use the del command to delete a folder of metrics like this : #to delete counters sandbox.test.* echo "delcounters sandbox.test.*" | nc 127.0.0.1 8126 - + Each backend will also publish a set of statistics, prefixed by its module name. @@ -55,3 +64,12 @@ The health output: * using health up or health down, you can change the current health status. * the healthStatus configuration option allows you to set the default health status at start. +Statsd Proxy specific commands +------------------------------ + +* status - the status of the current server + +The __status__ output currently will give you: + +* uptime: the number of seconds elapsed since statsd proxy started +* nodes: a space separated list of host:port for each active node in the ring From 331e8ba8ac9809f52fc01cebc6782119726c9977 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Thu, 14 Jan 2016 14:56:59 +0100 Subject: [PATCH 099/207] Nagios and Keepalived check script * Added check script that can be used by Nagios and Keepalived to monitor a statsd or statsd-proxy instance through the management interface. --- utils/check_statsd_health | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 utils/check_statsd_health diff --git a/utils/check_statsd_health b/utils/check_statsd_health new file mode 100644 index 00000000..962bc455 --- /dev/null +++ b/utils/check_statsd_health @@ -0,0 +1,62 @@ +#!/bin/bash + +# Check the status of a statsd or statsd proxy connecting directly to the +# management console. + +# This script can be used both for Nagios and Keepalived passing a parameter. +# The default behaviour is the Nagios one. + +OK=0; +CRITICAL=2; +UNKNOWN=3; +KEEPALIVED_CRITICAL=1; + +HOST="127.0.0.1" +PORT="8126" +MODE_KEEPALIVED=0 + +print_usage() { + echo "Usage: check_statsd_health [-H host] [-P port] [-k]" + echo "Options:" + echo " -H host Specify the host to check. [default: 127.0.0.1]" + echo " -P port Specify the port to check. [default: 8126]" + echo " -k Use exit status for Keepalived MISC_CHECK. [default: false]" + + if [[ "${MODE_KEEPALIVED}" -eq "1" ]]; then + exit ${KEEPALIVED_CRITICAL} + else + exit ${UNKNOWN} + fi +} + +while getopts ":H:P:k" opt; do + case $opt in + H) HOST="${OPTARG}";; + P) PORT="${OPTARG}";; + k) MODE_KEEPALIVED=1;; + + :) + echo "Missing mandatory value for option '-${OPTARG}'" >&2 + print_usage + ;; + + \?) + echo "Invalid option '${OPTARG}'" >&2 + print_usage + ;; + + esac +done + +HEALTH="$(echo -e "health\n" | nc -q1 "${HOST}" "${PORT}")" +echo "Statsd '${HOST}:${PORT}' responded: '${HEALTH}'" + +if [[ "${HEALTH}" == "health: up" ]]; then + exit ${OK} +else + if [[ "${MODE_KEEPALIVED}" -eq "1" ]]; then + exit ${KEEPALIVED_CRITICAL} + else + exit ${CRITICAL} + fi +fi From cd90200af1f3e48bd58c1871e96fd8cfe6a858f0 Mon Sep 17 00:00:00 2001 From: Riccardo Coccioli Date: Tue, 19 Jan 2016 14:28:23 +0100 Subject: [PATCH 100/207] Fix forks notification Notify the other forks only when the fork count is greater than zero. --- proxy.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/proxy.js b/proxy.js index 5dddca02..54c0f196 100644 --- a/proxy.js +++ b/proxy.js @@ -172,10 +172,16 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { var cmdaction = parameters[0].toLowerCase(); if (cmdaction === 'up') { healthStatus = 'up'; - process.send({ healthStatus: healthStatus }); + if (forkCount > 0) { + // Notify the other forks + process.send({ healthStatus: healthStatus }); + } } else if (cmdaction === 'down') { healthStatus = 'down'; - process.send({ healthStatus: healthStatus }); + if (forkCount > 0) { + // Notify the other forks + process.send({ healthStatus: healthStatus }); + } } } stream.write("health: " + healthStatus + "\n"); From f417a9115d19374637cf61acf240c2058bc9fea3 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Wed, 23 Mar 2016 21:53:28 +0000 Subject: [PATCH 101/207] properly use ipv6 config when starting proxy, adds extra keys to exampleProxyConfig. --- exampleProxyConfig.js | 4 +++- proxy.js | 8 ++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/exampleProxyConfig.js b/exampleProxyConfig.js index ef7f26b1..040fb32a 100644 --- a/exampleProxyConfig.js +++ b/exampleProxyConfig.js @@ -3,6 +3,8 @@ Required Variables: port: StatsD Cluster Proxy listening port [default: 8125] + mgmt_port: StatsD Cluster Proxy telnet management port [default: 8126] + mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] nodes: list of StatsD instances host: address of an instance of StatsD port: port that this instance is listening on @@ -30,9 +32,9 @@ nodes: [ {host: '127.0.0.1', port: 8131, adminport: 8132} ], server: './servers/udp', - host: '0.0.0.0', port: 8125, +mgmt_port: 8126, forkCount: 0, checkInterval: 1000, cacheSize: 10000 diff --git a/proxy.js b/proxy.js index 54c0f196..6499cade 100644 --- a/proxy.js +++ b/proxy.js @@ -21,8 +21,8 @@ var l; // logger configlib.configFile(process.argv[2], function (conf, oldConfig) { config = conf; - var udp_version = config.udp_version - , nodes = config.nodes; + var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + var nodes = config.nodes; l = new logger.Logger(config.log || {}); var forkCount = config.forkCount; @@ -134,7 +134,6 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { } } }); - var client = dgram.createSocket(udp_version); // Listen for the send message, and process the metric key and msg packet.on('send', function(key, msg) { @@ -152,9 +151,6 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { } }); - // Bind the listening udp server to the configured port and host - server.bind(config.port, config.host || undefined); - mgmt_server.start( config, function(cmd, parameters, stream) { From 54e6478d9faa38442008611af313ac6425b4a0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BChlschlegel?= Date: Mon, 23 Feb 2015 09:46:07 +0100 Subject: [PATCH 102/207] create accurate timer for metric flush clamp timestamp to a precise interval #459 --- stats.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/stats.js b/stats.js index 157313e6..44a06c59 100644 --- a/stats.js +++ b/stats.js @@ -70,6 +70,8 @@ var conf; // Flush metrics to each backend. function flushMetrics() { + setTimeout(flushMetrics, getFlushTimeout(flushInterval)); + var time_stamp = Math.round(new Date().getTime() / 1000); if (old_timestamp > 0) { gauges[timestamp_lag_namespace] = (time_stamp - old_timestamp - (Number(conf.flushInterval)/1000)); @@ -139,7 +141,7 @@ function flushMetrics() { } } - // normally gauges are not reset. so if we don't delete them, continue to persist previous value + // normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { for (var gauge_key in metrics.gauges) { @@ -171,6 +173,10 @@ function sanitizeKeyName(key) { } } +function getFlushTimeout(interval) { + return interval - (new Date().getTime() - startup_time * 1000) % flushInterval +} + // Global for the logger var l; @@ -417,7 +423,7 @@ config.configFile(process.argv[2], function (config) { } // Setup the flush timer - var flushInt = setInterval(flushMetrics, flushInterval); + var flushInt = setTimeout(flushMetrics, getFlushTimeout(flushInterval)); if (keyFlushInterval > 0) { var keyFlushPercent = Number((config.keyFlush && config.keyFlush.percent) || 100); From 5b788bc3677e4792ae3c99ffdcf40a8c0b3eb820 Mon Sep 17 00:00:00 2001 From: pataquets Date: Thu, 14 Apr 2016 21:07:57 +0200 Subject: [PATCH 103/207] Initial version Dockerfile. --- Dockerfile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fca93c87 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM node:onbuild + +RUN \ + cp -v exampleConfig.js config.js && \ + sed -i 's/graphite.example.com/graphite/' config.js + +EXPOSE 8125/udp +EXPOSE 8126 + +ENTRYPOINT [ "node", "stats.js", "config.js" ] From 40eda61fa4fde59bb558bb2207570d7a0cc1afff Mon Sep 17 00:00:00 2001 From: pataquets Date: Fri, 29 Apr 2016 21:31:28 +0200 Subject: [PATCH 104/207] Initial Docker Compose manifest. --- docker-compose.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5d702861 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +statsd: + build: . + links: + - carbon:graphite + ports: + - 8125:8125/udp + - 8126:8126 + +graphite-web: + image: dockerana/graphite + links: + - carbon + ports: + - 8000:8000 + volumes_from: + - carbon + +carbon: + image: dockerana/carbon + ports: + - 2003:2003 + - 2004:2004 + - 7002:7002 + volumes: + - /opt/graphite From ee7479ac4dc3f0809cc0bc8e653ac43b85fbe6b4 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Wed, 4 May 2016 15:36:18 -0400 Subject: [PATCH 105/207] Adding node 5.x to travis builds --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b031a5ae..c1218400 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ node_js: - '0.10' - '0.12' - '4' + - '5' - 'stable' script: ./run_tests.sh notifications: From 833d650d72e8a97680310dc1ce05a2e00b453a38 Mon Sep 17 00:00:00 2001 From: pataquets Date: Thu, 5 May 2016 11:06:31 +0200 Subject: [PATCH 106/207] Pin Node version to 5.x on Docker image. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fca93c87..4785de24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:onbuild +FROM node:5-onbuild RUN \ cp -v exampleConfig.js config.js && \ From b45394985bdf205e7651f2a006ac2b565ab76674 Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Thu, 5 May 2016 16:04:52 +0200 Subject: [PATCH 107/207] Add test. --- test/server_tests.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/server_tests.js b/test/server_tests.js index b15e70d2..77bb7dac 100644 --- a/test/server_tests.js +++ b/test/server_tests.js @@ -1,5 +1,6 @@ var dgram = require('dgram'), - net = require('net'); + net = require('net'), + fs = require('fs'); var config = { address: '127.0.0.1', @@ -59,5 +60,24 @@ module.exports = { }); client.end(); }); + }, + unix_socket_data_received: function(test) { + test.expect(3); + var server = require('../servers/tcp'); + config.socket = './statsd_tmp.socket'; + var started = server.start(config, function (data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + fs.unlinkSync(config.socket); + config.socket = undefined; + test.done(); + }); + + test.ok(started); + + var client = net.connect(config.socket, function () { + client.write(msg); + client.end(); + }); } -} +}; From ea757755a414413ad64e8b74e6e1ace27f4a78ea Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Thu, 5 May 2016 17:13:23 +0200 Subject: [PATCH 108/207] Fix node 0.10 error. --- servers/tcp.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servers/tcp.js b/servers/tcp.js index 8427e794..940a3b7c 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -4,7 +4,7 @@ var fs = require('fs'); function rinfo(tcpstream, data) { this.address = tcpstream.remoteAddress; this.port = tcpstream.remotePort; - this.family = tcpstream.address().family; + this.family = tcpstream.address() ? tcpstream.address().family : 'IPv4'; this.size = data.length; } From 1406e740eb06a75ee7c06e956be95207af6ccbe5 Mon Sep 17 00:00:00 2001 From: Brian Hatfield Date: Fri, 6 May 2016 20:56:52 -0400 Subject: [PATCH 109/207] Account for negative clock skew in flushMetrics --- stats.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/stats.js b/stats.js index 44a06c59..96bd13f8 100644 --- a/stats.js +++ b/stats.js @@ -70,8 +70,6 @@ var conf; // Flush metrics to each backend. function flushMetrics() { - setTimeout(flushMetrics, getFlushTimeout(flushInterval)); - var time_stamp = Math.round(new Date().getTime() / 1000); if (old_timestamp > 0) { gauges[timestamp_lag_namespace] = (time_stamp - old_timestamp - (Number(conf.flushInterval)/1000)); @@ -141,7 +139,7 @@ function flushMetrics() { } } - // normally gauges are not reset. so if we don't delete them, continue to persist previous value + // Normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { for (var gauge_key in metrics.gauges) { @@ -154,6 +152,10 @@ function flushMetrics() { backendEvents.emit('flush', time_stamp, metrics); }); + // Performing this setTimeout at the end of this method rather than the beginning + // helps ensure we adapt to negative clock skew by letting the method's latency + // introduce a short delay that should more than compensate. + setTimeout(flushMetrics, getFlushTimeout(flushInterval)); } var stats = { From 757fa36f864b8d246a99668c17f53abc9940b5e9 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Wed, 11 May 2016 19:02:19 +0000 Subject: [PATCH 110/207] Updates changelogs. closes #568 --- Changelog.md | 16 ++++++++++++++++ debian/changelog | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/Changelog.md b/Changelog.md index 5c62ab01..acfeab66 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,21 @@ # Changelog +## v0.8.0 (05/05/2016) +- Modularized injest servers, with support for loading multiple servers +- Added configurable tcp injest server +- Added unix socket injest support +- Added tcp repeater functionality +- Added pickle protocol support to graphite backend +- Added configurable IPv6 and TCP support to proxy +- Added telnet admin interface to proxy +- Multiple variable scoping fixes +- Fixes to flush timer to reduce bucket drift +- Fixes to ruby and java example client code +- Dropped support for node v0.8.x +- Fixed dependency issues for modern node versions +- Updated npm hashring dependency to v3.2.0 +- Replaced npm node-syslog dependency with modern-syslog v1.1.2 + ## v0.7.2 (09/02/2014) - Fixes to detecting valid packets diff --git a/debian/changelog b/debian/changelog index 741eb57d..7a3fd501 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,26 @@ +statsd (0.8.0-1) unstable; urgency=low + + * Modularized injest servers, with support for loading multiple servers + * Added configurable tcp injest server + * Added tcp repeater functionality + * Added unix socket injest support + * Added pickle protocol support to graphite backend + * Added configurable IPv6 and TCP support to proxy + * Added telnet admin interface to proxy + * Multiple variable scoping fixes + * Fixes to flush timer to reduce bucket drift + * Fixes to ruby and java example client code + * Dropped support for node v0.8.x + * Fixed dependency issues for modern node versions + * Updated npm hashring dependency to v3.2.0 + * Replaced npm node-syslog dependency with modern-syslog v1.1.2 + * Add systemd services for statsd and statsd-proxy + * Add servers directory to package + * Removed duplicated libs from package + * Add npm dependency for postinst + + -- Patrick Koch Thu, 5 May 2016 01:00:00 +0000 + statsd (0.7.2-1) unstable; urgency=low * Fixes to detecting valid packets From fc4bce5dc870709118a82b82abe52a72c95bff45 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Wed, 11 May 2016 19:05:36 +0000 Subject: [PATCH 111/207] Updates package.json for v0.8.0 npm release --- package.json | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6023aa70..3868aa5f 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,28 @@ { "name": "statsd", - "description": "A simple, lightweight network daemon to collect metrics over UDP", - "author": "Etsy", + "version": "0.8.0", + "description": "Network daemon for the collection and aggregation of realtime application metrics", + "author": { + "name": "Etsy", + "url": "https://codeascraft.com" + }, "license" : "MIT", - "scripts": { - "test": "./run_tests.sh", - "start": "node stats.js config.js", - "install-windows-service": "node_modules\\.bin\\winser -i", - "uninstall-windows-service": "node_modules\\.bin\\winser -r" + "homepage": "https://github.com/etsy/statsd", + "bugs": "https://github.com/etsy/statsd/issues", + "keywords": { + "statsd", + "etsy", + "metric", + "aggregation", + "realtime" }, "repository": { "type": "git", "url": "https://github.com/etsy/statsd.git" }, - "version": "0.7.2", + "engines": { + "node" : ">=0.10" + }, "dependencies": { "generic-pool": "2.2.0" }, @@ -27,8 +36,13 @@ "hashring":"3.2.0", "winser": "=0.1.6" }, - "engines": { - "node" : ">=0.8" + "bin": { + "statsd": "./bin/statsd" }, - "bin": { "statsd": "./bin/statsd" } + "scripts": { + "test": "./run_tests.sh", + "start": "node stats.js config.js", + "install-windows-service": "node_modules\\.bin\\winser -i", + "uninstall-windows-service": "node_modules\\.bin\\winser -r" + } } From 5d729d95847db0189e2c5571014b54c63d661f31 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Wed, 11 May 2016 19:26:26 +0000 Subject: [PATCH 112/207] fixes package.json --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3868aa5f..ed6bb6d8 100644 --- a/package.json +++ b/package.json @@ -9,13 +9,13 @@ "license" : "MIT", "homepage": "https://github.com/etsy/statsd", "bugs": "https://github.com/etsy/statsd/issues", - "keywords": { + "keywords": [ "statsd", "etsy", "metric", "aggregation", "realtime" - }, + ], "repository": { "type": "git", "url": "https://github.com/etsy/statsd.git" From c4440188642d12df0f9c5c3b1bdba4db736fbba0 Mon Sep 17 00:00:00 2001 From: Lewis McMahon Date: Fri, 13 May 2016 19:03:51 +0100 Subject: [PATCH 113/207] Updated graphite link to read the docs http://graphite.wikidot.com/ says it is out of date on the left hand side and links to the read the docs --- docs/backend.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend.md b/docs/backend.md index cba05480..f86d5241 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -56,4 +56,4 @@ queues and third-party services. - [statsd aggregation backend](https://github.com/wanelo/gossip_girl) - [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) -[graphite]: http://graphite.wikidot.com +[graphite]: https://graphite.readthedocs.io/en/latest/ From 59ebaaf56a5f472b3d89bf58be0f6b22d61b7574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20H=C3=A9bert?= Date: Mon, 4 Jul 2016 15:34:14 +0200 Subject: [PATCH 114/207] Add plugin Warp10 to statsd --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index f86d5241..4dd641c8 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -54,6 +54,7 @@ queues and third-party services. - [statsd-backend](https://github.com/dynmeth/statsd-backend) - [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) - [statsd aggregation backend](https://github.com/wanelo/gossip_girl) +- [warp10-backend](https://github.com/cityzendata/statsd-warp10-backend) - [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) [graphite]: https://graphite.readthedocs.io/en/latest/ From 9c04385ae960765c915eaece0aa390f151d1eee1 Mon Sep 17 00:00:00 2001 From: Andrew Moss Date: Mon, 31 Oct 2016 16:16:13 +0000 Subject: [PATCH 115/207] fix usage of process.EventEmitter --- lib/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config.js b/lib/config.js index 5c1866b9..132f75c0 100644 --- a/lib/config.js +++ b/lib/config.js @@ -30,7 +30,7 @@ var Configurator = function (file) { }); }; -util.inherits(Configurator, process.EventEmitter); +util.inherits(Configurator, require('events')); exports.Configurator = Configurator; From 8d5363cb109cc6363661a1d5813e0b96787c4411 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 21 Nov 2016 11:48:29 -0500 Subject: [PATCH 116/207] Fix for failing test on node 0.10 After merging #593, tests against node 0.10 started failing with ``` util.js:556 ctor.prototype = Object.create(superCtor.prototype, { ^ TypeError: Object prototype may only be an Object or null ``` --- lib/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config.js b/lib/config.js index 132f75c0..ce7c86b9 100644 --- a/lib/config.js +++ b/lib/config.js @@ -30,7 +30,7 @@ var Configurator = function (file) { }); }; -util.inherits(Configurator, require('events')); +util.inherits(Configurator, require('events').EventEmitter); exports.Configurator = Configurator; From 46af23d23b18b8d5f24bd331e42c887dfc87652b Mon Sep 17 00:00:00 2001 From: "Markus \"Shorty\" Uckelmann" Date: Mon, 19 Dec 2016 13:17:43 +0100 Subject: [PATCH 117/207] removes -q switch This commit removesd the -q switch which isn't supported by all netcat variants. More information in #602. --- utils/check_statsd_health | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check_statsd_health b/utils/check_statsd_health index 962bc455..b6ec96ea 100644 --- a/utils/check_statsd_health +++ b/utils/check_statsd_health @@ -48,7 +48,7 @@ while getopts ":H:P:k" opt; do esac done -HEALTH="$(echo -e "health\n" | nc -q1 "${HOST}" "${PORT}")" +HEALTH="$(echo -e "health\n" | nc "${HOST}" "${PORT}")" echo "Statsd '${HOST}:${PORT}' responded: '${HEALTH}'" if [[ "${HEALTH}" == "health: up" ]]; then From e713daa517ed12a877a57d1ecb6199cffde238fd Mon Sep 17 00:00:00 2001 From: till Date: Sat, 14 Jan 2017 18:01:06 +0100 Subject: [PATCH 118/207] Update: ignore files - ensure config.js is not committed to the repo - ensure debian, examples, etc. are not put into container --- .dockerignore | 7 +++++++ .gitignore | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..836271e6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +example* +debian +config.js +packager +.git* +.pkgr.yml +.travis.yml diff --git a/.gitignore b/.gitignore index 7d618a48..b9abaafd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ node_modules - +/config.js # WebStorm / IntelliJ IDEA project files .idea *.iml From c9644428b425c958a7acf31870fde030ac1ccf27 Mon Sep 17 00:00:00 2001 From: Vincent Sisk Date: Mon, 9 Apr 2018 16:16:22 -0600 Subject: [PATCH 119/207] fix formatting on backend interface docs --- docs/backend_interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backend_interface.md b/docs/backend_interface.md index bbfc6f01..d37ecce5 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -29,7 +29,7 @@ the `events` object: and `metrics` is a hash representing the StatsD statistics: ``` -metrics: { + metrics: { counters: counters, gauges: gauges, timers: timers, @@ -38,7 +38,7 @@ metrics: { timer_data: timer_data, statsd_metrics: statsd_metrics, pctThreshold: pctThreshold -} + } ``` The counter_rates and timer_data are precalculated statistics to simplify From ead5fa288f4caf2b15c08868c6146e7007a7a844 Mon Sep 17 00:00:00 2001 From: Braden Steffaniak Date: Tue, 1 May 2018 21:46:10 -0600 Subject: [PATCH 120/207] Added StatsdClient Kotlin implementation --- examples/StatsdClient.kt | 206 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 examples/StatsdClient.kt diff --git a/examples/StatsdClient.kt b/examples/StatsdClient.kt new file mode 100644 index 00000000..733de62b --- /dev/null +++ b/examples/StatsdClient.kt @@ -0,0 +1,206 @@ +/** + * StatsdClient.kt + * + * + * Example usage: + * + * val client = StatsdClient("statsd.example.com", 8125) + * // increment by 1 + * client.increment("foo.bar.baz") + * // increment by 10 + * client.increment("foo.bar.baz", magnitude = 10) + * // sample rate + * client.increment("foo.bar.baz", sampleRate = 0.1) + * // magnitude and sample rate + * client.increment("foo.bar.baz", magnitude = 10, sampleRate = 0.1) + * // increment multiple keys by 1 + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar") + * // increment multiple keys by 10 + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar", magnitude = 10) + * // multiple keys with a sample rate and magnitude + * client.increment("foo.bar.baz", "foo.bar.boo", "foo.baz.bar", magnitude = 10, sampleRate = 0.1) + */ + +import org.apache.log4j.Logger +import java.io.IOException +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.StandardSocketOptions +import java.net.UnknownHostException +import java.nio.ByteBuffer +import java.nio.channels.DatagramChannel +import java.util.Locale +import java.util.Random +import java.util.Timer +import java.util.TimerTask + +class StatsdClient @Throws(IOException::class) +constructor(host: InetAddress, port: Int) : TimerTask() { + private var sendBuffer: ByteBuffer? = null + private var flushTimer: Timer? = null + private var multiMetrics = false + + private val address: InetSocketAddress = InetSocketAddress(host, port) + private val channel: DatagramChannel = DatagramChannel.open() + + @Throws(UnknownHostException::class, IOException::class) + constructor(host: String, port: Int) : this(InetAddress.getByName(host), port) + + init { + // Put this in non-blocking mode so send does not block forever. + channel.configureBlocking(false) + // Increase the size of the output buffer so that the size is larger than our buffer size. + channel.setOption(StandardSocketOptions.SO_SNDBUF, 4096) + setBufferSize(1500) + } + + @Synchronized + fun setBufferSize(packetBufferSize: Short) { + if (sendBuffer != null) { + flush() + } + sendBuffer = ByteBuffer.allocate(packetBufferSize.toInt()) + } + + @Synchronized + fun enableMultiMetrics(enable: Boolean) { + multiMetrics = enable + } + + @Synchronized + fun startFlushTimer(period: Long = 2000): Boolean { + return if (flushTimer == null) { + flushTimer = Timer() + + // We pass this object in as the TimerTask (which calls run()) + flushTimer!!.schedule(this, period, period) + true + } else { + false + } + } + + @Synchronized + fun stopFlushTimer() { + if (flushTimer != null) { + flushTimer!!.cancel() + flushTimer = null + } + } + + // used by Timer, we're a Runnable TimerTask + override fun run() { + flush() + } + + fun timing(key: String, value: Int, sampleRate: Double = 1.0): Boolean { + return send(sampleRate, String.format(Locale.ENGLISH, "%s:%d|ms", key, value)) + } + + fun decrement(vararg keys: String, magnitude: Int = -1, sampleRate: Double = 1.0): Boolean { + val stats = keys.map { String.format(Locale.ENGLISH, "%s:%s|c", it, magnitude) }.toTypedArray() + + return send(sampleRate, *stats) + } + + fun increment(vararg keys: String, magnitude: Int = 1, sampleRate: Double = 1.0): Boolean { + val stats = keys.map { String.format(Locale.ENGLISH, "%s:%s|c", it, magnitude) }.toTypedArray() + + return send(sampleRate, *stats) + } + + fun gauge(key: String, magnitude: Double, sampleRate: Double = 1.0): Boolean { + val stat = String.format(Locale.ENGLISH, "%s:%s|g", key, magnitude) + + return send(sampleRate, stat) + } + + private fun send(sampleRate: Double, vararg stats: String): Boolean { + return if (sampleRate < 1.0) { + stats.any { + if (RNG.nextDouble() <= sampleRate) { + val stat = String.format(Locale.ENGLISH, "%s|@%f", it, sampleRate) + + doSend(stat) + } else { + false + } + } + } else { + stats.any { doSend(it) } + } + } + + @Synchronized + private fun doSend(stat: String): Boolean { + try { + val data = stat.toByteArray(charset("utf-8")) + + // If we're going to go past the threshold of the buffer then flush. + // the +1 is for the potential '\n' in multi_metrics below + if (sendBuffer!!.remaining() < data.size + 1) { + flush() + } + + // multiple metrics are separated by '\n' + if (sendBuffer!!.position() > 0) { + sendBuffer!!.put('\n'.toByte()) + } + + sendBuffer!!.put(data) + + if (!multiMetrics) { + flush() + } + + return true + } catch (e: IOException) { + log.error(String.format("Could not send stat %s to host %s:%d", sendBuffer!!.toString(), address.hostName, address.port), e) + + return false + } + } + + @Synchronized + fun flush(): Boolean { + try { + val sizeOfBuffer = sendBuffer!!.position() + + if (sizeOfBuffer <= 0) { + return false + } // empty buffer + + // send and reset the buffer + sendBuffer!!.flip() + + val nbSentBytes = channel.send(sendBuffer, address) + + sendBuffer!!.limit(sendBuffer!!.capacity()) + sendBuffer!!.rewind() + + return if (sizeOfBuffer == nbSentBytes) { + true + } else { + log.error(String.format( + "Could not send entirely stat %s to host %s:%d. Only sent %d bytes out of %d bytes", + sendBuffer!!.toString(), + address.hostName, + address.port, + nbSentBytes, + sizeOfBuffer + )) + + false + } + } catch (e: IOException) { + /* This would be a good place to close the channel down and recreate it. */ + log.error(String.format("Could not send stat %s to host %s:%d", sendBuffer!!.toString(), address.hostName, address.port), e) + return false + } + } + + companion object { + private val RNG = Random() + private val log = Logger.getLogger(StatsdClient::class.java.name) + } +} From 7f864770f330cd79bd9661a2d33d020f9494514e Mon Sep 17 00:00:00 2001 From: lpmi-13 Date: Tue, 1 Jan 2019 22:46:16 +0000 Subject: [PATCH 121/207] fix simple typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 23ac959a..f263391a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ etc) * *values* Each stat will have a value. How it is interpreted depends on modifiers. In -general values should be integer. +general values should be integers. * *flush* After the flush interval timeout (defined by `config.flushInterval`, From 206689d9b89784e8e10ca719a4b6c451cdd29681 Mon Sep 17 00:00:00 2001 From: Daz Wilkin Date: Tue, 15 Jan 2019 12:05:15 -0800 Subject: [PATCH 122/207] Fixing Markdown formatting --- docs/backend_interface.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/backend_interface.md b/docs/backend_interface.md index bbfc6f01..6b4544cb 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -28,7 +28,7 @@ the `events` object: two parameters: `time_stamp` is the current time in epoch seconds and `metrics` is a hash representing the StatsD statistics: - ``` +``` metrics: { counters: counters, gauges: gauges, @@ -39,7 +39,7 @@ metrics: { statsd_metrics: statsd_metrics, pctThreshold: pctThreshold } - ``` +``` The counter_rates and timer_data are precalculated statistics to simplify the creation of backends, the statsd_metrics hash contains metrics generated @@ -70,5 +70,3 @@ metrics: { This is emitted for every incoming packet. The `packet` parameter contains the raw received message string and the `rinfo` parameter contains remote address information from the UDP socket. - - From 3818796c49368f1a1eb5b6f951e3e9624d505ac6 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 10:51:42 +0100 Subject: [PATCH 123/207] update README post transfer --- README.md | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 23ac959a..163ca00e 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,6 @@ listens for statistics, like counters and timers, sent over [UDP][udp] or [TCP][tcp] and sends aggregates to one or more pluggable backend services (e.g., [Graphite][graphite]). -We ([Etsy][etsy]) [blogged][blog post] about how it works and why we created it. - -Inspiration ------------ - -StatsD was inspired (heavily) by the project (of the same name) at Flickr. -Here's a post where Cal Henderson described it in depth: -[Counting and timing][counting-timing] -Cal re-released the code recently: -[Perl StatsD][Flicker-StatsD] - Key Concepts -------- @@ -95,6 +84,20 @@ Meta - IRC channel: `#statsd` on freenode - Mailing list: `statsd@librelist.com` +History +--------- +statsd was originally written at ([Etsy][etsy]) and released with a [blog post][blog post] +about how it works and why we created it. + +Inspiration +----------- + +StatsD was inspired (heavily) by the project (of the same name) at Flickr. +Here's a post where Cal Henderson described it in depth: +[Counting and timing][counting-timing] +Cal re-released the code recently: +[Perl StatsD][Flicker-StatsD] + [graphite]: http://graphite.readthedocs.org/ @@ -106,14 +109,14 @@ Meta [Flicker-StatsD]: https://github.com/iamcal/Flickr-StatsD [udp]: http://en.wikipedia.org/wiki/User_Datagram_Protocol [tcp]: http://en.wikipedia.org/wiki/Transmission_Control_Protocol -[docs_metric_types]: https://github.com/etsy/statsd/blob/master/docs/metric_types.md -[docs_graphite]: https://github.com/etsy/statsd/blob/master/docs/graphite.md -[docs_server]: https://github.com/etsy/statsd/blob/master/docs/server.md -[docs_backend]: https://github.com/etsy/statsd/blob/master/docs/backend.md -[docs_admin_interface]: https://github.com/etsy/statsd/blob/master/docs/admin_interface.md -[docs_server_interface]: https://github.com/etsy/statsd/blob/master/docs/server_interface.md -[docs_backend_interface]: https://github.com/etsy/statsd/blob/master/docs/backend_interface.md +[docs_metric_types]: https://github.com/statsd/statsd/blob/master/docs/metric_types.md +[docs_graphite]: https://github.com/statsd/statsd/blob/master/docs/graphite.md +[docs_server]: https://github.com/statsd/statsd/blob/master/docs/server.md +[docs_backend]: https://github.com/statsd/statsd/blob/master/docs/backend.md +[docs_admin_interface]: https://github.com/statsd/statsd/blob/master/docs/admin_interface.md +[docs_server_interface]: https://github.com/statsd/statsd/blob/master/docs/server_interface.md +[docs_backend_interface]: https://github.com/statsd/statsd/blob/master/docs/backend_interface.md [docs_namespacing]: https://github.com/etsy/statsd/blob/master/docs/namespacing.md [docs_cluster_proxy]: https://github.com/etsy/statsd/blob/master/docs/cluster_proxy.md -[travis-ci_status_img]: https://travis-ci.org/etsy/statsd.svg?branch=master -[travis-ci_statsd]: https://travis-ci.org/etsy/statsd +[travis-ci_status_img]: https://travis-ci.org/statsd/statsd.svg?branch=master +[travis-ci_statsd]: https://travis-ci.org/statsd/statsd From a8df9bc6ee07c0f5b70158a6bec06f791bd6c039 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 11:04:47 +0100 Subject: [PATCH 124/207] Create CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..35ce6077 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at +statsd-coc@googlegroups.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 5f58a9cc7442900c2e553ed1df3d6ce99e885226 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 11:06:21 +0100 Subject: [PATCH 125/207] Create DCO.txt --- DCO.txt | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 DCO.txt diff --git a/DCO.txt b/DCO.txt new file mode 100644 index 00000000..8201f992 --- /dev/null +++ b/DCO.txt @@ -0,0 +1,37 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. From ed841ba3fe21333837f394c83ade49cab18222ac Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 11:17:41 +0100 Subject: [PATCH 126/207] Create MAINTAINERS.md --- MAINTAINERS.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 MAINTAINERS.md diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 00000000..a249c784 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,26 @@ +statsd is maintained by the following people. + +All maintainers must agree to the [Developer Certificate of Origin][dco]. + +## Steps to becoming a maintainer +1. Open a PR where you add yourself to this file and agree to the DCO. Attach a little bit about your experience +with statsd and how much time you think you can roughly spend on the project +2. Current maintainers will review +3. If it gets merged, you're in! + +## Retiring from being a maintainer +1. Open a pull request moving yourself from current to past maintainers +2. Mention [@statsd/statsd-maintainers](https://github.com/orgs/statsd/teams/statsd-maintainers) on the PR +3. Have another maintainer approve/sign off +4. Merge it + +## Current maintainers + +- **Daniel Schauenberg**: As a maintainer of Phan, I agree to the [Developer Certificate of Origin][dco]. + +[dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt + +## Past maintainers +- [Ben Burry](https://github.com/benburry) +- [Dan Rowe](https://github.com/draco2003) +- [Erik Kastner](https://github.com/kastner) From 6da2ce8dcac317439a55e858fa03648596b8d5f6 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 11:24:55 +0100 Subject: [PATCH 127/207] remove meta section of README --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 163ca00e..6e5f071e 100644 --- a/README.md +++ b/README.md @@ -78,12 +78,6 @@ background (don't do this on a production machine!). Tests can be executed with `./run_tests.sh`. - -Meta ---------- -- IRC channel: `#statsd` on freenode -- Mailing list: `statsd@librelist.com` - History --------- statsd was originally written at ([Etsy][etsy]) and released with a [blog post][blog post] From 8db2f0de7d922888bc9688315e908bff74a75144 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 11:26:24 +0100 Subject: [PATCH 128/207] Update MAINTAINERS.md --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index a249c784..7413c91e 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -16,7 +16,7 @@ with statsd and how much time you think you can roughly spend on the project ## Current maintainers -- **Daniel Schauenberg**: As a maintainer of Phan, I agree to the [Developer Certificate of Origin][dco]. +- **Daniel Schauenberg**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. [dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt From 4b276202863557df1f4757fa365ce16d128a8a62 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Tue, 12 Feb 2019 17:10:16 +0100 Subject: [PATCH 129/207] Update MAINTAINERS.md --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 7413c91e..7bf47672 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -21,6 +21,7 @@ with statsd and how much time you think you can roughly spend on the project [dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt ## Past maintainers +- [Tera Koch](https://github.com/pathzzrd) - [Ben Burry](https://github.com/benburry) - [Dan Rowe](https://github.com/draco2003) - [Erik Kastner](https://github.com/kastner) From fe81cd2598a0effb74b944b0eace23621a628071 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 12 Feb 2019 18:00:37 +0000 Subject: [PATCH 130/207] correct package.json links to new github organisation --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ed6bb6d8..ffa24c94 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "name": "Etsy", "url": "https://codeascraft.com" }, - "license" : "MIT", - "homepage": "https://github.com/etsy/statsd", - "bugs": "https://github.com/etsy/statsd/issues", + "license": "MIT", + "homepage": "https://github.com/statsd/statsd", + "bugs": "https://github.com/statsd/statsd/issues", "keywords": [ "statsd", "etsy", @@ -18,7 +18,7 @@ ], "repository": { "type": "git", - "url": "https://github.com/etsy/statsd.git" + "url": "https://github.com/statsd/statsd.git" }, "engines": { "node" : ">=0.10" From fe58d813ecdc92e613734a6442423049618d7ecf Mon Sep 17 00:00:00 2001 From: Daz Wilkin Date: Thu, 14 Feb 2019 10:21:14 -0800 Subject: [PATCH 131/207] Added "opencensus-backend" --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 4dd641c8..e5b1e625 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -48,6 +48,7 @@ queues and third-party services. - [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) - [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) - [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) +- [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) - [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) - [socket.io-backend](https://github.com/Chatham/statsd-socket.io) - [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) From faea607f0382b283144a4ef7981ff215ffc3ac8e Mon Sep 17 00:00:00 2001 From: Vincent Sisk Date: Thu, 14 Feb 2019 13:51:29 -0700 Subject: [PATCH 132/207] remove extra newline from merge conflict fix --- docs/backend_interface.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/backend_interface.md b/docs/backend_interface.md index 42360e6d..c71806b7 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -30,7 +30,6 @@ the `events` object: ``` metrics: { - counters: counters, gauges: gauges, timers: timers, From ff308f672f598ed22d786e63206d785ed14bb0fc Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 12 Feb 2019 17:59:33 +0000 Subject: [PATCH 133/207] begin testing on node lts and up --- .travis.yml | 9 ++++----- package.json | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1218400..67472851 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ language: node_js node_js: - - '0.10' - - '0.12' - - '4' - - '5' - - 'stable' + - '6' + - '8' + - '10' + - '11' script: ./run_tests.sh notifications: email: false diff --git a/package.json b/package.json index ffa24c94..c369a9c1 100644 --- a/package.json +++ b/package.json @@ -3,25 +3,25 @@ "version": "0.8.0", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { - "name": "Etsy", - "url": "https://codeascraft.com" + "name": "Etsy", + "url": "https://codeascraft.com" }, "license": "MIT", "homepage": "https://github.com/statsd/statsd", "bugs": "https://github.com/statsd/statsd/issues", "keywords": [ - "statsd", - "etsy", - "metric", - "aggregation", - "realtime" + "statsd", + "etsy", + "metric", + "aggregation", + "realtime" ], "repository": { "type": "git", "url": "https://github.com/statsd/statsd.git" }, "engines": { - "node" : ">=0.10" + "node": ">=6" }, "dependencies": { "generic-pool": "2.2.0" @@ -32,12 +32,12 @@ "temp": "0.4.x" }, "optionalDependencies": { - "modern-syslog":"1.1.2", - "hashring":"3.2.0", + "modern-syslog": "1.1.2", + "hashring": "3.2.0", "winser": "=0.1.6" }, "bin": { - "statsd": "./bin/statsd" + "statsd": "./bin/statsd" }, "scripts": { "test": "./run_tests.sh", From 85cd02c8405c96fda2ab31329a3138449ac59b12 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Wed, 20 Feb 2019 09:25:44 -0500 Subject: [PATCH 134/207] Adding myself to MAINTAINERS. --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 7bf47672..e2b92f16 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -17,6 +17,7 @@ with statsd and how much time you think you can roughly spend on the project ## Current maintainers - **Daniel Schauenberg**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. +- **Mike Heffner**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. [dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt From 7bac183cf3ef807d3a2c209fbfa3561f30e6c6dc Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 21 Feb 2019 13:27:05 +0000 Subject: [PATCH 135/207] add mysql backend link to docs/backend.md --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 4dd641c8..fb87db17 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -47,6 +47,7 @@ queues and third-party services. - [librato-backend](https://github.com/librato/statsd-librato-backend) - [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) - [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) +- [mysql backend](https://github.com/fradinni/nodejs-statsd-mysql-backend) - [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) - [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) - [socket.io-backend](https://github.com/Chatham/statsd-socket.io) From 9d506f526569a9d9c97e3c114585ddcc877870e6 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 7 Mar 2019 07:45:37 +0000 Subject: [PATCH 136/207] Add myself (elliot blackburn) to maintainers.md (#666) --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index e2b92f16..3489461c 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -18,6 +18,7 @@ with statsd and how much time you think you can roughly spend on the project - **Daniel Schauenberg**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. - **Mike Heffner**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. +- **Elliot Blackburn**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. [dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt From 4a79c7c998a531d3596f1f5ea80965a238cd5073 Mon Sep 17 00:00:00 2001 From: Dmitry Menshikov Date: Wed, 13 Mar 2019 18:09:59 +0200 Subject: [PATCH 137/207] drop statsd instance from proxy ring in instance of healthcheck failures (#665) Change of healthcheck logic in proxy. In case of error failed instance will be dropped from the ring, not only on ECONNREFUSED error. --- proxy.js | 95 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/proxy.js b/proxy.js index 6499cade..d051d38f 100644 --- a/proxy.js +++ b/proxy.js @@ -12,7 +12,7 @@ var dgram = require('dgram') var packet = new events.EventEmitter(); var startup_time = Math.round(new Date().getTime() / 1000); -var node_status = []; +var node_status = {}; var workers = []; // Keep track of all forked childs var node_ring = {}; var servers_loaded; @@ -36,6 +36,8 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { }; var healthStatus = configlib.healthStatus || 'up'; + var healthCheckInterval = config.checkInterval || 10000; + var broadcastMsg = function(msg) { for (var i = 0; i < workers.length; i++) { workers[i].send(msg); @@ -214,7 +216,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { log("server is up", "INFO"); // Set the interval for healthchecks - setInterval(doHealthChecks, config.checkInterval || 10000); + setInterval(doHealthChecks, healthCheckInterval); } // Perform health check on all nodes @@ -224,52 +226,77 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { }); } + function markNodeAsHealthy(node_id) { + if (node_status[node_id] !== undefined) { + if (node_status[node_id] > 0) { + var new_server = {}; + new_server[node_id] = 100; + log('Adding node ' + node_id + ' to the ring.', 'WARNING'); + ring.add(new_server); + } + } + + node_status[node_id] = 0; + } + + function markNodeAsUnhealthy(node_id) { + if (node_status[node_id] === undefined) { + node_status[node_id] = 1; + } else { + node_status[node_id]++; + } + if (node_status[node_id] < 2) { + log('Removing node ' + node_id + ' from the ring.', 'WARNING'); + ring.remove(node_id); + } + } + // Perform health check on node function healthcheck(node) { + var ended = false; var node_id = node.host + ':' + node.port; - var client = net.connect({port: node.adminport, host: node.host}, - function() { - client.write('health\r\n'); + var client = net.connect( + {port: node.adminport, host: node.host}, + function onConnect() { + if (!ended) { + client.write('health\r\n'); + } + } + ); + + client.setTimeout(healthCheckInterval, function() { + client.end(); + markNodeAsUnhealthy(node_id); + client.removeAllListeners('data'); + ended = true; }); + client.on('data', function(data) { + if (ended) { + return; + } + var health_status = data.toString(); client.end(); + ended = true; + if (health_status.indexOf('up') < 0) { - if (node_status[node_id] === undefined) { - node_status[node_id] = 1; - } else { - node_status[node_id]++; - } - if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.', 'WARNING'); - ring.remove(node_id); - } + markNodeAsUnhealthy(node_id); } else { - if (node_status[node_id] !== undefined) { - if (node_status[node_id] > 0) { - var new_server = {}; - new_server[node_id] = 100; - log('Adding node ' + node_id + ' to the ring.', 'WARNING'); - ring.add(new_server); - } - } - node_status[node_id] = 0; + markNodeAsHealthy(node_id); } }); + client.on('error', function(e) { - if (e.code == 'ECONNREFUSED') { - if (node_status[node_id] === undefined) { - node_status[node_id] = 1; - } else { - node_status[node_id]++; - } - if (node_status[node_id] < 2) { - log('Removing node ' + node_id + ' from the ring.', 'WARNING'); - ring.remove(node_id); - } - } else { + if (ended) { + return; + } + + if (e.code !== 'ECONNREFUSED' && e.code !== 'EHOSTUNREACH' && e.code !== 'ECONNRESET') { log('Error during healthcheck on node ' + node_id + ' with ' + e.code, 'ERROR'); } + + markNodeAsUnhealthy(node_id); }); } From aa44a5e0c3f0d35c460c7da1532ec0ead14aebce Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 13 Mar 2019 16:13:17 +0000 Subject: [PATCH 138/207] increment version to 0.8.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c369a9c1..45a38781 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.0", + "version": "0.8.1", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From db4ec7b16b5d98b5b797cdb5892d8e2d25cbbc74 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Mon, 25 Mar 2019 14:31:23 +0000 Subject: [PATCH 139/207] run tests using python 3.7's pickle rather than 2.x cPickle (#669) --- .travis.yml | 2 +- package.json | 2 +- run_tests.sh => run_tests.js | 0 test/graphite_pickle_tests.js | 6 +++--- 4 files changed, 5 insertions(+), 5 deletions(-) rename run_tests.sh => run_tests.js (100%) diff --git a/.travis.yml b/.travis.yml index 67472851..d82cdf44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ node_js: - '8' - '10' - '11' -script: ./run_tests.sh +script: node run_tests.js notifications: email: false irc: diff --git a/package.json b/package.json index 45a38781..e0b8b90e 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "statsd": "./bin/statsd" }, "scripts": { - "test": "./run_tests.sh", + "test": "node run_tests.sh", "start": "node stats.js config.js", "install-windows-service": "node_modules\\.bin\\winser -i", "uninstall-windows-service": "node_modules\\.bin\\winser -r" diff --git a/run_tests.sh b/run_tests.js similarity index 100% rename from run_tests.sh rename to run_tests.js diff --git a/test/graphite_pickle_tests.js b/test/graphite_pickle_tests.js index 8a9fda2a..e68051a7 100644 --- a/test/graphite_pickle_tests.js +++ b/test/graphite_pickle_tests.js @@ -66,7 +66,7 @@ var collect_for = function(server,timeout,cb){ // wire protocol into JSON written to stdout. var script = "import sys\n" + - "import cPickle\n" + + "import pickle\n" + "import struct\n" + "import json\n" + "payload = open(sys.argv[1], 'rb').read()\n" + @@ -74,8 +74,8 @@ var script = "header_length = struct.calcsize(pack_format)\n" + "payload_length, = struct.unpack(pack_format, payload[:header_length])\n" + "batch_length = header_length + payload_length\n" + - "metrics = cPickle.loads(payload[header_length:batch_length])\n" + - "print json.dumps(metrics)\n"; + "metrics = pickle.loads(payload[header_length:batch_length])\n" + + "print(json.dumps(metrics))\n"; // Write our binary payload and unpickling script to disk // then process the unserialized results. From 5fd52130780460cd4f09d69cb9b1f87747a01327 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 11:16:11 +0100 Subject: [PATCH 140/207] Add gitter chat badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 639bac3a..c49c749e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] +StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/community](https://badges.gitter.im/statsd/community.svg)](https://gitter.im/statsd/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) ====== A network daemon that runs on the [Node.js][node] platform and From 6c91b4d93e3784b9222fab4044b4320cee9b3d90 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 11:39:04 +0100 Subject: [PATCH 141/207] update dockerfile base image to node lts --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4785de24..6c0b0e1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:5-onbuild +FROM node:10.15.3-alpine RUN \ cp -v exampleConfig.js config.js && \ From fc859069cef0dd205b2d4128ed768682eb6c08b7 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 11:42:09 +0100 Subject: [PATCH 142/207] correct gitter link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c49c749e..d9412e43 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/community](https://badges.gitter.im/statsd/community.svg)](https://gitter.im/statsd/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) ====== A network daemon that runs on the [Node.js][node] platform and From 7b4405b14d6edb36516d860b5ea68fd5cf03d3dc Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 21:14:46 +0100 Subject: [PATCH 143/207] update dockerfile to latest node-lts --- .dockerignore | 20 ++++++++++++++++---- Dockerfile | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 836271e6..9a75c204 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,19 @@ -example* -debian -config.js -packager +# Dotfiles .git* .pkgr.yml .travis.yml + +# Non-app files +*.md +*.txt +exampleProxyConfig.js + +# Non-app directories +debian +node_modules +packager +examples +docs + +# Local dev files +config.js diff --git a/Dockerfile b/Dockerfile index 6c0b0e1f..3fa2dab7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,31 @@ -FROM node:10.15.3-alpine +FROM node:10.15.3 +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Install python +# RUN apk add --no-cache --update g++ gcc libgcc libstdc++ linux-headers make python + +# Setup node envs +ARG NODE_ENV +ENV NODE_ENV $NODE_ENV + +# Install dependencies +COPY package.json /usr/src/app/ +RUN npm install && npm cache clean --force + +# Copy required src (see .dockerignore) +COPY . /usr/src/app + +# Set graphite hostname to "graphite" RUN \ + ls -la && \ cp -v exampleConfig.js config.js && \ sed -i 's/graphite.example.com/graphite/' config.js +# Expose required ports EXPOSE 8125/udp EXPOSE 8126 +# Start statsd ENTRYPOINT [ "node", "stats.js", "config.js" ] From 75de5ff061d586bfa05e7834b288e611a8e04a87 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 21:34:08 +0100 Subject: [PATCH 144/207] update travis npm token --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d82cdf44..25448414 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: node_js node_js: - - '6' - - '8' - - '10' - - '11' +- '6' +- '8' +- '10' +- '11' script: node run_tests.js notifications: email: false @@ -13,7 +13,7 @@ deploy: provider: npm email: d@unwiredcouch.com api_key: - secure: IE9nz50eZsRL1Dbcxj2eY0apO1Io2swGF3ezZCzny20WgqQXiiVs24rHUi1GywDELGDc7+Vp0zJfmXigE+zTMvx0N3fTASiuDzd3C7fULa4JUSH2DoHNOXx7WSkr4EmujDsB7y1mEBDHOdBlLWBRApExt67TlYzvZiT8/Sffq3k= + secure: deHm/dx0AJoSpkgbi+480BbC8Qxsv5vf4TPfMqM8uGYicj6qMJBjG7AZtCUIcJ0mSEoc9Zf79jOJCeWC53CWziQX+D+AXDC/8RGlqtx1h1zgpM6m1NonceqUt9wN/tVG1eNa/22WrerMA3F2rxo5QtY3rMxGMlJjCYOwyFR++eM= on: tags: true repo: etsy/statsd From a8adbdb35428201402acbb2c253412cf57170731 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 21:40:35 +0100 Subject: [PATCH 145/207] release 0.8.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e0b8b90e..e833e0fe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.1", + "version": "0.8.2", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 839a8106f82baf586e40d5a6f49c166f93ec1b75 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 2 Apr 2019 21:57:21 +0100 Subject: [PATCH 146/207] correct travis deploy step --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25448414..ed779ce4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,9 @@ notifications: - irc.freenode.org#statsd deploy: provider: npm - email: d@unwiredcouch.com + email: elliot@lybrary.io api_key: secure: deHm/dx0AJoSpkgbi+480BbC8Qxsv5vf4TPfMqM8uGYicj6qMJBjG7AZtCUIcJ0mSEoc9Zf79jOJCeWC53CWziQX+D+AXDC/8RGlqtx1h1zgpM6m1NonceqUt9wN/tVG1eNa/22WrerMA3F2rxo5QtY3rMxGMlJjCYOwyFR++eM= on: tags: true - repo: etsy/statsd + repo: statsd/statsd From ef8e4d76d76c3a0cf771e3724cf79ea22f3c50d9 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Mon, 29 Apr 2019 16:37:45 +0100 Subject: [PATCH 147/207] correct npm test script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e833e0fe..643b9f77 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "statsd": "./bin/statsd" }, "scripts": { - "test": "node run_tests.sh", + "test": "node run_tests.js", "start": "node stats.js config.js", "install-windows-service": "node_modules\\.bin\\winser -i", "uninstall-windows-service": "node_modules\\.bin\\winser -r" From 0a21156de137360a938af3983fdf10d5b9f787da Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 29 May 2019 14:22:15 +0100 Subject: [PATCH 148/207] Convert codebase from var -> let / const (#673) --- lib/config.js | 14 ++-- lib/helpers.js | 10 +-- lib/logger.js | 4 +- lib/mgmt_console.js | 8 +-- lib/mgmt_server.js | 8 +-- lib/process_metrics.js | 74 ++++++++++----------- lib/process_mgmt.js | 4 +- lib/set.js | 6 +- run_tests.js | 2 +- servers/tcp.js | 12 ++-- servers/udp.js | 6 +- stats.js | 147 ++++++++++++++++++++--------------------- 12 files changed, 147 insertions(+), 148 deletions(-) diff --git a/lib/config.js b/lib/config.js index ce7c86b9..11ff71c3 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,13 +1,13 @@ /*jshint node:true, laxcomma:true */ -var fs = require('fs') - , util = require('util'); +const fs = require('fs') +const util = require('util'); -var Configurator = function (file) { +let Configurator = function (file) { - var self = this; - var config = {}; - var oldConfig = {}; + let self = this; + let config = {}; + let oldConfig = {}; this.updateConfig = function () { util.log('[' + process.pid + '] reading config file: ' + file); @@ -35,7 +35,7 @@ util.inherits(Configurator, require('events').EventEmitter); exports.Configurator = Configurator; exports.configFile = function(file, callbackFunc) { - var config = new Configurator(file); + let config = new Configurator(file); config.on('configChanged', function() { callbackFunc(config.config, config.oldConfig); }); diff --git a/lib/helpers.js b/lib/helpers.js index be6f5530..d570816b 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -12,9 +12,9 @@ function isNumber(str) { } function isValidSampleRate(str) { - var validSampleRate = false; + let validSampleRate = false; if(str.length > 1 && str[0] === '@') { - var numberStr = str.substring(1); + const numberStr = str.substring(1); validSampleRate = isNumber(numberStr) && numberStr[0] != '-'; } return validSampleRate; @@ -55,7 +55,7 @@ exports.is_valid_packet = is_valid_packet; exports.writeConfig = function(config, stream) { stream.write("\n"); - for (var prop in config) { + for (const prop in config) { if (!config.hasOwnProperty(prop)) { continue; } @@ -63,8 +63,8 @@ exports.writeConfig = function(config, stream) { stream.write(prop + ": " + config[prop] + "\n"); continue; } - var subconfig = config[prop]; - for (var subprop in subconfig) { + const subconfig = config[prop]; + for (const subprop in subconfig) { if (!subconfig.hasOwnProperty(subprop)) { continue; } diff --git a/lib/logger.js b/lib/logger.js index 56e27ee8..8867f825 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -var Logger = function (config) { +const Logger = function (config) { this.config = config; this.backend = this.config.backend || 'stdout'; this.level = this.config.level || "LOG_INFO"; @@ -24,7 +24,7 @@ Logger.prototype = { } this.util.log(type + ": " + msg); } else { - var level; + let level; if (!type) { level = this.level; } else { diff --git a/lib/mgmt_console.js b/lib/mgmt_console.js index e9431c5f..bd99e274 100644 --- a/lib/mgmt_console.js +++ b/lib/mgmt_console.js @@ -14,7 +14,7 @@ exports.delete_stats = function(stats_type, cmdline, stream) { //for each metric requested on the command line - for (var index in cmdline) { + for (const index in cmdline) { //get a list of deletable metrics that match the request deletable = existing_stats(stats_type, cmdline[index]); @@ -25,7 +25,7 @@ exports.delete_stats = function(stats_type, cmdline, stream) { } //delete all requested metrics - for (var del_idx in deletable) { + for (const del_idx in deletable) { delete stats_type[deletable[del_idx]]; stream.write("deleted: " + deletable[del_idx] + "\n"); } @@ -53,9 +53,9 @@ function existing_stats(stats_type, bucket){ //special case: match a whole 'folder' (and subfolders) of stats if (bucket.slice(-2) == ".*") { - var folder = bucket.slice(0,-1); + const folder = bucket.slice(0,-1); - for (var name in stats_type) { + for (const name in stats_type) { //check if stat is in bucket, ie~ name starts with folder if (name.substring(0, folder.length) == folder) { matches.push(name); diff --git a/lib/mgmt_server.js b/lib/mgmt_server.js index c89e32bc..172826d4 100644 --- a/lib/mgmt_server.js +++ b/lib/mgmt_server.js @@ -1,14 +1,14 @@ /*jshint node:true, laxcomma:true */ -var net = require('net'); +const net = require('net'); exports.start = function(config, on_data_callback, on_error_callback) { - var server = net.createServer(function(stream) { + const server = net.createServer(function(stream) { stream.setEncoding('ascii'); stream.on('data', function(data) { - var cmdline = data.trim().split(" "); - var cmd = cmdline.shift(); + const cmdline = data.trim().split(" "); + const cmd = cmdline.shift(); on_data_callback(cmd, cmdline, stream); }); diff --git a/lib/process_metrics.js b/lib/process_metrics.js index 8228f989..489eb39a 100644 --- a/lib/process_metrics.js +++ b/lib/process_metrics.js @@ -1,53 +1,53 @@ /*jshint node:true, laxcomma:true */ -var process_metrics = function (metrics, flushInterval, ts, flushCallback) { - var starttime = Date.now(); - var key; - var counter_rates = {}; - var timer_data = {}; - var statsd_metrics = {}; - var counters = metrics.counters; - var timers = metrics.timers; - var timer_counters = metrics.timer_counters; - var pctThreshold = metrics.pctThreshold; - var histogram = metrics.histogram; +const process_metrics = function (metrics, flushInterval, ts, flushCallback) { + const starttime = Date.now(); + let key; + let counter_rates = {}; + let timer_data = {}; + let statsd_metrics = {}; + const counters = metrics.counters; + const timers = metrics.timers; + const timer_counters = metrics.timer_counters; + const pctThreshold = metrics.pctThreshold; + const histogram = metrics.histogram; for (key in counters) { - var value = counters[key]; + const value = counters[key]; // calculate "per second" rate counter_rates[key] = value / (flushInterval / 1000); } for (key in timers) { - var current_timer_data = {}; + const current_timer_data = {}; if (timers[key].length > 0) { timer_data[key] = {}; - var values = timers[key].sort(function (a,b) { return a-b; }); - var count = values.length; - var min = values[0]; - var max = values[count - 1]; + const values = timers[key].sort(function (a,b) { return a-b; }); + const count = values.length; + const min = values[0]; + const max = values[count - 1]; - var cumulativeValues = [min]; - var cumulSumSquaresValues = [min * min]; - for (var i = 1; i < count; i++) { + const cumulativeValues = [min]; + const cumulSumSquaresValues = [min * min]; + for (let i = 1; i < count; i++) { cumulativeValues.push(values[i] + cumulativeValues[i-1]); cumulSumSquaresValues.push((values[i] * values[i]) + cumulSumSquaresValues[i - 1]); } - var sum = min; - var sumSquares = min * min; - var mean = min; - var thresholdBoundary = max; + let sum = min; + let sumSquares = min * min; + let mean = min; + let thresholdBoundary = max; - var key2; + let key2; for (key2 in pctThreshold) { - var pct = pctThreshold[key2]; - var numInThreshold = count; + const pct = pctThreshold[key2]; + let numInThreshold = count; if (count > 1) { numInThreshold = Math.round(Math.abs(pct) / 100 * count); @@ -68,7 +68,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { mean = sum / numInThreshold; } - var clean_pct = '' + pct; + let clean_pct = '' + pct; clean_pct = clean_pct.replace('.', '_').replace('-', 'top'); current_timer_data["count_" + clean_pct] = numInThreshold; current_timer_data["mean_" + clean_pct] = mean; @@ -82,15 +82,15 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { sumSquares = cumulSumSquaresValues[count-1]; mean = sum / count; - var sumOfDiffs = 0; - for (var i = 0; i < count; i++) { + let sumOfDiffs = 0; + for (let i = 0; i < count; i++) { sumOfDiffs += (values[i] - mean) * (values[i] - mean); } - var mid = Math.floor(count/2); - var median = (count % 2) ? values[mid] : (values[mid-1] + values[mid])/2; + const mid = Math.floor(count/2); + const median = (count % 2) ? values[mid] : (values[mid-1] + values[mid])/2; - var stddev = Math.sqrt(sumOfDiffs / count); + const stddev = Math.sqrt(sumOfDiffs / count); current_timer_data["std"] = stddev; current_timer_data["upper"] = max; current_timer_data["lower"] = min; @@ -104,7 +104,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { // note: values bigger than the upper limit of the last bin are ignored, by design conf = histogram || []; bins = []; - for (var i = 0; i < conf.length; i++) { + for (let i = 0; i < conf.length; i++) { if (key.indexOf(conf[i].metric) > -1) { bins = conf[i].bins; break; @@ -116,9 +116,9 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { // the outer loop iterates bins, the inner loop iterates timer values; // within each run of the inner loop we should only consider the timer value range that's within the scope of the current bin // so we leverage the fact that the values are already sorted to end up with only full 1 iteration of the entire values range - var i = 0; - for (var bin_i = 0; bin_i < bins.length; bin_i++) { - var freq = 0; + let i = 0; + for (let bin_i = 0; bin_i < bins.length; bin_i++) { + let freq = 0; for (; i < count && (bins[bin_i] == 'inf' || values[i] < bins[bin_i]); i++) { freq += 1; } diff --git a/lib/process_mgmt.js b/lib/process_mgmt.js index b512e656..a0dcf7ba 100644 --- a/lib/process_mgmt.js +++ b/lib/process_mgmt.js @@ -1,6 +1,6 @@ -var util = require('util'); +const util = require('util'); -var conf; +let conf; exports.init = function(config) { conf = config; diff --git a/lib/set.js b/lib/set.js index b8ea3da0..58211ab6 100644 --- a/lib/set.js +++ b/lib/set.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -var Set = function() { +const Set = function() { this.store = {}; }; @@ -21,8 +21,8 @@ Set.prototype = { this.store = {}; }, values: function() { - var values = []; - for (var value in this.store) { + let values = []; + for (const value in this.store) { values.push(value); } return values; diff --git a/run_tests.js b/run_tests.js index 333fa7d7..6883f3de 100755 --- a/run_tests.js +++ b/run_tests.js @@ -1,6 +1,6 @@ #!/usr/bin/env node try { - var reporter = require('nodeunit').reporters.default; + const reporter = require('nodeunit').reporters.default; } catch(e) { console.log("Cannot find nodeunit module."); diff --git a/servers/tcp.js b/servers/tcp.js index 940a3b7c..49875446 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -1,5 +1,5 @@ -var net = require('net'); -var fs = require('fs'); +const net = require('net'); +const fs = require('fs'); function rinfo(tcpstream, data) { this.address = tcpstream.remoteAddress; @@ -9,15 +9,15 @@ function rinfo(tcpstream, data) { } exports.start = function(config, callback) { - var server = net.createServer(function(stream) { + const server = net.createServer(function(stream) { stream.setEncoding('ascii'); - var buffer = ''; + let buffer = ''; stream.on('data', function(data) { buffer += data; - var offset = buffer.lastIndexOf("\n"); + const offset = buffer.lastIndexOf("\n"); if (offset > -1) { - var packet = buffer.slice(0, offset + 1); + const packet = buffer.slice(0, offset + 1); buffer = buffer.slice(offset + 1); callback(packet, new rinfo(stream, packet)); } diff --git a/servers/udp.js b/servers/udp.js index 345cc567..bca72608 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,8 +1,8 @@ -var dgram = require('dgram'); +const dgram = require('dgram'); exports.start = function(config, callback) { - var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; - var server = dgram.createSocket(udp_version, callback); + const udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + const server = dgram.createSocket(udp_version, callback); server.bind(config.port || 8125, config.address || undefined); this.server = server; diff --git a/stats.js b/stats.js index 96bd13f8..58975a30 100644 --- a/stats.js +++ b/stats.js @@ -1,45 +1,44 @@ /*jshint node:true, laxcomma:true */ -var util = require('util') - , config = require('./lib/config') - , helpers = require('./lib/helpers') - , fs = require('fs') - , events = require('events') - , logger = require('./lib/logger') - , set = require('./lib/set') - , pm = require('./lib/process_metrics') - , process_mgmt = require('./lib/process_mgmt') - , mgmt_server = require('./lib/mgmt_server') - , mgmt = require('./lib/mgmt_console'); - +const util = require('util'); +const config = require('./lib/config'); +const helpers = require('./lib/helpers'); +const fs = require('fs'); +const events = require('events'); +const logger = require('./lib/logger'); +const set = require('./lib/set'); +const pm = require('./lib/process_metrics'); +const process_mgmt = require('./lib/process_mgmt'); +const mgmt_server = require('./lib/mgmt_server'); +const mgmt = require('./lib/mgmt_console'); // initialize data structures with defaults for statsd stats -var keyCounter = {}; -var counters = {}; -var timers = {}; -var timer_counters = {}; -var gauges = {}; -var sets = {}; -var counter_rates = {}; -var timer_data = {}; -var pctThreshold = null; -var flushInterval, keyFlushInt, serversLoaded, mgmtServer; -var startup_time = Math.round(new Date().getTime() / 1000); -var backendEvents = new events.EventEmitter(); -var healthStatus = config.healthStatus || 'up'; -var old_timestamp = 0; -var timestamp_lag_namespace; -var keyNameSanitize = true; +let keyCounter = {}; +let counters = {}; +let timers = {}; +let timer_counters = {}; +let gauges = {}; +let sets = {}; +let counter_rates = {}; +let timer_data = {}; +let pctThreshold = null; +let flushInterval, keyFlushInt, serversLoaded, mgmtServer; +let startup_time = Math.round(new Date().getTime() / 1000); +let backendEvents = new events.EventEmitter(); +let healthStatus = config.healthStatus || 'up'; +let old_timestamp = 0; +let timestamp_lag_namespace; +let keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { - var backendmod = require(name); + const backendmod = require(name); if (config.debug) { l.log("Loading backend: " + name, 'DEBUG'); } - var ret = backendmod.init(startup_time, config, backendEvents, l); + const ret = backendmod.init(startup_time, config, backendEvents, l); if (!ret) { l.log("Failed to load backend: " + name, "ERROR"); process.exit(1); @@ -52,13 +51,13 @@ function loadBackend(config, name) { // rinfo: contains remote address information and message length // (attributes are .address, .port, .family, .size - you're welcome) function startServer(config, name, callback) { - var servermod = require(name); + const servermod = require(name); if (config.debug) { l.log("Loading server: " + name, 'DEBUG'); } - var ret = servermod.start(config, callback); + const ret = servermod.start(config, callback); if (!ret) { l.log("Failed to load server: " + name, "ERROR"); process.exit(1); @@ -66,17 +65,17 @@ function startServer(config, name, callback) { } // global for conf -var conf; +let conf; // Flush metrics to each backend. function flushMetrics() { - var time_stamp = Math.round(new Date().getTime() / 1000); + const time_stamp = Math.round(new Date().getTime() / 1000); if (old_timestamp > 0) { gauges[timestamp_lag_namespace] = (time_stamp - old_timestamp - (Number(conf.flushInterval)/1000)); } old_timestamp = time_stamp; - var metrics_hash = { + const metrics_hash = { counters: counters, gauges: gauges, timers: timers, @@ -103,7 +102,7 @@ function flushMetrics() { // Clear the counters conf.deleteCounters = conf.deleteCounters || false; - for (var counter_key in metrics.counters) { + for (const counter_key in metrics.counters) { if (conf.deleteCounters) { if ((counter_key.indexOf("packets_received") != -1) || (counter_key.indexOf("metrics_received") != -1) || @@ -119,7 +118,7 @@ function flushMetrics() { // Clear the timers conf.deleteTimers = conf.deleteTimers || false; - for (var timer_key in metrics.timers) { + for (const timer_key in metrics.timers) { if (conf.deleteTimers) { delete(metrics.timers[timer_key]); delete(metrics.timer_counters[timer_key]); @@ -131,7 +130,7 @@ function flushMetrics() { // Clear the sets conf.deleteSets = conf.deleteSets || false; - for (var set_key in metrics.sets) { + for (const set_key in metrics.sets) { if (conf.deleteSets) { delete(metrics.sets[set_key]); } else { @@ -142,7 +141,7 @@ function flushMetrics() { // Normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { - for (var gauge_key in metrics.gauges) { + for (const gauge_key in metrics.gauges) { delete(metrics.gauges[gauge_key]); } } @@ -158,7 +157,7 @@ function flushMetrics() { setTimeout(flushMetrics, getFlushTimeout(flushInterval)); } -var stats = { +const stats = { messages: { last_msg_seen: startup_time, bad_lines_seen: 0 @@ -180,7 +179,7 @@ function getFlushTimeout(interval) { } // Global for the logger -var l; +let l; config.configFile(process.argv[2], function (config) { conf = config; @@ -190,7 +189,7 @@ config.configFile(process.argv[2], function (config) { l = new logger.Logger(config.log || {}); // setup config for stats prefix - var prefixStats = config.prefixStats; + let prefixStats = config.prefixStats; prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; //setup the names for the stats stored in counters{} bad_lines_seen = prefixStats + ".bad_lines_seen"; @@ -209,20 +208,20 @@ config.configFile(process.argv[2], function (config) { if (!serversLoaded) { // key counting - var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); + const keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); - var handlePacket = function (msg, rinfo) { + const handlePacket = function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; - var metrics; - var packet_data = msg.toString(); + let metrics; + const packet_data = msg.toString(); if (packet_data.indexOf("\n") > -1) { metrics = packet_data.split("\n"); } else { metrics = [ packet_data ] ; } - for (var midx in metrics) { + for (const midx in metrics) { if (metrics[midx].length === 0) { continue; } @@ -231,8 +230,8 @@ config.configFile(process.argv[2], function (config) { if (config.dumpMessages) { l.log(metrics[midx].toString()); } - var bits = metrics[midx].toString().split(':'); - var key = sanitizeKeyName(bits.shift()); + const bits = metrics[midx].toString().split(':'); + const key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { @@ -245,9 +244,9 @@ config.configFile(process.argv[2], function (config) { bits.push("1"); } - for (var i = 0; i < bits.length; i++) { - var sampleRate = 1; - var fields = bits[i].split("|"); + for (let i = 0; i < bits.length; i++) { + let sampleRate = 1; + const fields = bits[i].split("|"); if (!helpers.is_valid_packet(fields)) { l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"'); counters[bad_lines_seen]++; @@ -258,7 +257,7 @@ config.configFile(process.argv[2], function (config) { sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]); } - var metric_type = fields[1].trim(); + const metric_type = fields[1].trim(); if (metric_type === "ms") { if (! timers[key]) { timers[key] = []; @@ -290,10 +289,10 @@ config.configFile(process.argv[2], function (config) { }; // If config.servers isn't specified, use the top-level config for backwards-compatibility - var server_config = config.servers || [config]; - for (var i = 0; i < server_config.length; i++) { + const server_config = config.servers || [config]; + for (let i = 0; i < server_config.length; i++) { // The default server is UDP - var server = server_config[i].server || './servers/udp'; + const server = server_config[i].server || './servers/udp'; startServer(server_config[i], server, handlePacket); } @@ -311,7 +310,7 @@ config.configFile(process.argv[2], function (config) { case "health": if (parameters.length > 0) { - var cmdaction = parameters[0].toLowerCase(); + const cmdaction = parameters[0].toLowerCase(); if (cmdaction === 'up') { healthStatus = 'up'; } else if (cmdaction === 'down') { @@ -322,13 +321,13 @@ config.configFile(process.argv[2], function (config) { break; case "stats": - var now = Math.round(new Date().getTime() / 1000); - var uptime = now - startup_time; + const now = Math.round(new Date().getTime() / 1000); + const uptime = now - startup_time; stream.write("uptime: " + uptime + "\n"); - var stat_writer = function(group, metric, val) { - var delta; + const stat_writer = function(group, metric, val) { + let delta; if (metric.match("^last_")) { delta = now - val; @@ -341,13 +340,13 @@ config.configFile(process.argv[2], function (config) { }; // Loop through the base stats - for (var group in stats) { - for (var metric in stats[group]) { + for (const group in stats) { + for (const metric in stats[group]) { stat_writer(group, metric, stats[group][metric]); } } - backendEvents.once('status', function(writeCb) { + backendEvents.once('status', function() { stream.write("END\n\n"); }); @@ -416,7 +415,7 @@ config.configFile(process.argv[2], function (config) { config.flushInterval = flushInterval; if (config.backends) { - for (var j = 0; j < config.backends.length; j++) { + for (const j = 0; j < config.backends.length; j++) { loadBackend(config, config.backends[j]); } } else { @@ -425,31 +424,31 @@ config.configFile(process.argv[2], function (config) { } // Setup the flush timer - var flushInt = setTimeout(flushMetrics, getFlushTimeout(flushInterval)); + const flushInt = setTimeout(flushMetrics, getFlushTimeout(flushInterval)); if (keyFlushInterval > 0) { - var keyFlushPercent = Number((config.keyFlush && config.keyFlush.percent) || 100); - var keyFlushLog = config.keyFlush && config.keyFlush.log; + const keyFlushPercent = Number((config.keyFlush && config.keyFlush.percent) || 100); + const keyFlushLog = config.keyFlush && config.keyFlush.log; keyFlushInt = setInterval(function () { - var sortedKeys = []; + const sortedKeys = []; - for (var key in keyCounter) { + for (const key in keyCounter) { sortedKeys.push([key, keyCounter[key]]); } sortedKeys.sort(function(a, b) { return b[1] - a[1]; }); - var logMessage = ""; - var timeString = (new Date()) + ""; + let logMessage = ""; + const timeString = (new Date()) + ""; // only show the top "keyFlushPercent" keys - for (var i = 0, e = sortedKeys.length * (keyFlushPercent / 100); i < e; i++) { + for (let i = 0, e = sortedKeys.length * (keyFlushPercent / 100); i < e; i++) { logMessage += timeString + " count=" + sortedKeys[i][1] + " key=" + sortedKeys[i][0] + "\n"; } if (keyFlushLog) { - var logFile = fs.createWriteStream(keyFlushLog, {flags: 'a+'}); + const logFile = fs.createWriteStream(keyFlushLog, {flags: 'a+'}); logFile.write(logMessage); logFile.end(); } else { From 3306cc641858b74932cc80de9ff3fc69bf54d6f1 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 29 May 2019 14:41:47 +0100 Subject: [PATCH 149/207] Correct reporter decleration in test runner --- run_tests.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_tests.js b/run_tests.js index 6883f3de..75be28cd 100755 --- a/run_tests.js +++ b/run_tests.js @@ -1,6 +1,7 @@ #!/usr/bin/env node +let reporter; try { - const reporter = require('nodeunit').reporters.default; + reporter = require('nodeunit').reporters.default; } catch(e) { console.log("Cannot find nodeunit module."); From baec7759782ac34b943c86bab8fce620b4ee2681 Mon Sep 17 00:00:00 2001 From: Francis Gulotta Date: Mon, 13 May 2019 18:06:58 -0400 Subject: [PATCH 150/207] test and declare support for Current and LTS node This PR tests statsd on all Current and LTS Node.js versions and declares their support in the readme. --- .travis.yml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed779ce4..539886bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: node_js node_js: -- '6' - '8' - '10' - '11' +- '12' script: node run_tests.js notifications: email: false diff --git a/README.md b/README.md index d9412e43..36408028 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ general values should be integers. Installation and Configuration ------------------------------ - * Install node.js + * Install Node.js (All [`Current` and `LTS` Node.js versions](https://nodejs.org/en/about/releases/) are supported.) * Clone the project * Create a config file from `exampleConfig.js` and put it somewhere - * Start the Daemon: + * Start the Daemon: `node stats.js /path/to/config` Usage From f64faef4b41fe89d459b52457427a8bf798b7271 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 30 May 2019 16:14:19 +0100 Subject: [PATCH 151/207] correct backend flush loop --- stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats.js b/stats.js index 58975a30..b3cb31af 100644 --- a/stats.js +++ b/stats.js @@ -415,7 +415,7 @@ config.configFile(process.argv[2], function (config) { config.flushInterval = flushInterval; if (config.backends) { - for (const j = 0; j < config.backends.length; j++) { + for (let j = 0; j < config.backends.length; j++) { loadBackend(config, config.backends[j]); } } else { From 30b779b5189bb0002465081277475f3bfff96a6e Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Fri, 21 Jun 2019 15:00:55 +0100 Subject: [PATCH 152/207] update nodeunit and add a package-lock.json --- package-lock.json | 2313 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 2 files changed, 2314 insertions(+), 1 deletion(-) create mode 100644 package-lock.json diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..363189e0 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2313 @@ +{ + "name": "statsd", + "version": "0.8.2", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@babel/code-frame": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", + "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "dev": true, + "requires": { + "@babel/highlight": "^7.0.0" + } + }, + "@babel/generator": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.4.4.tgz", + "integrity": "sha512-53UOLK6TVNqKxf7RUh8NE851EHRxOOeVXKbK2bivdb+iziMyk03Sr4eaE9OELCbyZAAafAKPDwF2TPUES5QbxQ==", + "dev": true, + "requires": { + "@babel/types": "^7.4.4", + "jsesc": "^2.5.1", + "lodash": "^4.17.11", + "source-map": "^0.5.0", + "trim-right": "^1.0.1" + } + }, + "@babel/helper-function-name": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.1.0.tgz", + "integrity": "sha512-A95XEoCpb3TO+KZzJ4S/5uW5fNe26DjBGqf1o9ucyLyCmi1dXq/B3c8iaWTfBk3VvetUxl16e8tIrd5teOCfGw==", + "dev": true, + "requires": { + "@babel/helper-get-function-arity": "^7.0.0", + "@babel/template": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "@babel/helper-get-function-arity": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0.tgz", + "integrity": "sha512-r2DbJeg4svYvt3HOS74U4eWKsUAMRH01Z1ds1zx8KNTPtpTL5JAsdFv8BNyOpVqdFhHkkRDIg5B4AsxmkjAlmQ==", + "dev": true, + "requires": { + "@babel/types": "^7.0.0" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.4.4.tgz", + "integrity": "sha512-Ro/XkzLf3JFITkW6b+hNxzZ1n5OQ80NvIUdmHspih1XAhtN3vPTuUFT4eQnela+2MaZ5ulH+iyP513KJrxbN7Q==", + "dev": true, + "requires": { + "@babel/types": "^7.4.4" + } + }, + "@babel/highlight": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", + "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", + "dev": true, + "requires": { + "chalk": "^2.0.0", + "esutils": "^2.0.2", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz", + "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", + "dev": true + }, + "@babel/template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.4.4.tgz", + "integrity": "sha512-CiGzLN9KgAvgZsnivND7rkA+AeJ9JB0ciPOD4U59GKbQP2iQl+olF1l76kJOupqidozfZ32ghwBEJDhnk9MEcw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.4.4", + "@babel/types": "^7.4.4" + } + }, + "@babel/traverse": { + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.4.5.tgz", + "integrity": "sha512-Vc+qjynwkjRmIFGxy0KYoPj4FdVDxLej89kMHFsWScq999uX+pwcX4v9mWRjW0KcAYTPAuVQl2LKP1wEVLsp+A==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/generator": "^7.4.4", + "@babel/helper-function-name": "^7.1.0", + "@babel/helper-split-export-declaration": "^7.4.4", + "@babel/parser": "^7.4.5", + "@babel/types": "^7.4.4", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.11" + } + }, + "@babel/types": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.4.4.tgz", + "integrity": "sha512-dOllgYdnEFOebhkKCjzSVFqw/PmmB8pH6RGOWkY4GsboQNd47b1fBThBSwlHAq9alF9vc1M3+6oqR47R50L0tQ==", + "dev": true, + "requires": { + "esutils": "^2.0.2", + "lodash": "^4.17.11", + "to-fast-properties": "^2.0.0" + } + }, + "ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "append-transform": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz", + "integrity": "sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==", + "dev": true, + "requires": { + "default-require-extensions": "^2.0.0" + } + }, + "archy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", + "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=", + "dev": true + }, + "arg": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.0.tgz", + "integrity": "sha512-ZWc51jO3qegGkVh8Hwpv636EkbesNV5ZNQPCtRa+0qytRYPEs9IYT9qITY9buezqUH5uqyzlWLcufrzU2rffdg==", + "dev": true + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "bind-obj-methods": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/bind-obj-methods/-/bind-obj-methods-2.0.0.tgz", + "integrity": "sha512-3/qRXczDi2Cdbz6jE+W3IflJOutRVica8frpBn14de1mBOkzDo+6tY33kNhvkw54Kn3PzRRD2VnGbGPcTAk4sw==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "caching-transform": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz", + "integrity": "sha512-Mtgcv3lh3U0zRii/6qVgQODdPA4G3zhG+jtbCWj39RXuUFTMzH0vcdMtaJS1jPowd+It2Pqr6y3NJMQqOqCE2w==", + "dev": true, + "requires": { + "hasha": "^3.0.0", + "make-dir": "^2.0.0", + "package-hash": "^3.0.0", + "write-file-atomic": "^2.4.2" + } + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "capture-stack-trace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", + "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==", + "dev": true + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "clean-yaml-object": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/clean-yaml-object/-/clean-yaml-object-0.1.0.tgz", + "integrity": "sha1-Y/sRDcLOGoTcIfbZM0h20BCui2g=", + "dev": true + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "dev": true + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-1.3.1.tgz", + "integrity": "sha1-AkQ+AtuW9LMrZ0IlRRq7bpUQAA4=", + "optional": true, + "requires": { + "keypress": "0.1.x" + } + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "connection-parse": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", + "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=", + "optional": true + }, + "convert-source-map": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.6.0.tgz", + "integrity": "sha512-eFu7XigvxdZ1ETfbgPBohgyQ/Z++C0eEhTor0qRwBw9unw+L0/6V8wkSuGgzdThkiS5lSpdptOQPD8Ak40a+7A==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.1" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "coveralls": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.4.tgz", + "integrity": "sha512-eyqUWA/7RT0JagiL0tThVhjbIjoiEUyWCjtUJoOPcWoeofP5WK/jb2OJYoBFrR6DvplR+AxOyuBqk4JHkk5ykA==", + "dev": true, + "requires": { + "growl": "~> 1.10.0", + "js-yaml": "^3.11.0", + "lcov-parse": "^0.0.10", + "log-driver": "^1.2.7", + "minimist": "^1.2.0", + "request": "^2.86.0" + } + }, + "cp-file": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/cp-file/-/cp-file-6.2.0.tgz", + "integrity": "sha512-fmvV4caBnofhPe8kOcitBwSn2f39QLjnAnGq3gO9dfd75mUytzKNZB1hde6QHunW2Rt+OwuBOMc3i1tNElbszA==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "make-dir": "^2.0.0", + "nested-error-stacks": "^2.0.0", + "pify": "^4.0.1", + "safe-buffer": "^5.0.1" + } + }, + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "default-require-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", + "integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=", + "dev": true, + "requires": { + "strip-bom": "^3.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, + "diff": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz", + "integrity": "sha1-fyjS657nsVqX79ic5j3P2qPMur8=", + "dev": true + }, + "domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "ejs": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.6.2.tgz", + "integrity": "sha512-PcW2a0tyTuPHz3tWyYqtK6r1fZ3gp+3Sop8Ph+ZYN81Ob5rwmbHEzaqs10N3BEsaGTkh/ooniXK+WwszGlc2+Q==", + "dev": true + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "events-to-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/events-to-array/-/events-to-array-1.1.2.tgz", + "integrity": "sha1-LUH1Y+H+QA7Uli/hpNXGp1Od9/Y=", + "dev": true + }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dev": true, + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + } + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "foreground-child": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-1.5.6.tgz", + "integrity": "sha1-T9ca0t/elnibmApcCilZN8svXOk=", + "dev": true, + "requires": { + "cross-spawn": "^4", + "signal-exit": "^3.0.0" + } + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "fs-exists-cached": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz", + "integrity": "sha1-zyVVTKBQ3EmuZla0HeQiWJidy84=", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "function-loop": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/function-loop/-/function-loop-1.0.2.tgz", + "integrity": "sha512-Iw4MzMfS3udk/rqxTiDDCllhGwlOrsr50zViTOO/W6lS/9y6B1J0BD2VZzrnWUYBJsl3aeqjgR5v7bWWhZSYbA==", + "dev": true + }, + "generic-pool": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.2.0.tgz", + "integrity": "sha1-i0ZcGnWI6p3SuxM72gu2a/74pj4=" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "dev": true + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, + "handlebars": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", + "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", + "dev": true, + "requires": { + "neo-async": "^2.6.0", + "optimist": "^0.6.1", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "hasha": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-3.0.0.tgz", + "integrity": "sha1-UqMvq4Vp1BymmmH/GiFPjrfIvTk=", + "dev": true, + "requires": { + "is-stream": "^1.0.1" + } + }, + "hashring": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", + "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", + "optional": true, + "requires": { + "connection-parse": "0.0.x", + "simple-lru-cache": "0.0.x" + } + }, + "hosted-git-info": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", + "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", + "dev": true + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "invert-kv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", + "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==", + "dev": true + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true, + "optional": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, + "istanbul-lib-coverage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", + "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==", + "dev": true + }, + "istanbul-lib-hook": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz", + "integrity": "sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA==", + "dev": true, + "requires": { + "append-transform": "^1.0.0" + } + }, + "istanbul-lib-instrument": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz", + "integrity": "sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA==", + "dev": true, + "requires": { + "@babel/generator": "^7.4.0", + "@babel/parser": "^7.4.3", + "@babel/template": "^7.4.0", + "@babel/traverse": "^7.4.3", + "@babel/types": "^7.4.0", + "istanbul-lib-coverage": "^2.0.5", + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", + "dev": true + } + } + }, + "istanbul-lib-report": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz", + "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==", + "dev": true, + "requires": { + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "supports-color": "^6.1.0" + }, + "dependencies": { + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "istanbul-lib-source-maps": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.6.tgz", + "integrity": "sha512-R47KzMtDJH6X4/YW9XTx+jrLnZnscW4VpNN+1PViSYTejLVPWv7oov+Duf8YQSPyVRUvueQqz1TcsC6mooZTXw==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "rimraf": "^2.6.3", + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "istanbul-reports": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.6.tgz", + "integrity": "sha512-SKi4rnMyLBKe0Jy2uUdx28h8oG7ph2PPuQPvIAh31d+Ci+lSiEu4C+h3oBPuJ9+mPKhOyW0M8gY4U5NM1WLeXA==", + "dev": true, + "requires": { + "handlebars": "^4.1.2" + } + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "keypress": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/keypress/-/keypress-0.1.0.tgz", + "integrity": "sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo=", + "optional": true + }, + "lcid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", + "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==", + "dev": true, + "requires": { + "invert-kv": "^2.0.0" + } + }, + "lcov-parse": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz", + "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=", + "dev": true + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "dev": true + }, + "lodash.flattendeep": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", + "dev": true + }, + "log-driver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", + "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==", + "dev": true + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "make-error": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", + "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==", + "dev": true + }, + "map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "requires": { + "p-defer": "^1.0.0" + } + }, + "mem": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz", + "integrity": "sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==", + "dev": true, + "requires": { + "map-age-cleaner": "^0.1.1", + "mimic-fn": "^2.0.0", + "p-is-promise": "^2.0.0" + } + }, + "merge-source-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/merge-source-map/-/merge-source-map-1.1.0.tgz", + "integrity": "sha512-Qkcp7P2ygktpMPh2mCQZaf3jhN6D3Z/qVZHSdWvQ+2Ef5HgRAPBO57A77+ENm0CPx2+1Ce/MYKi3ymqdfuqibw==", + "dev": true, + "requires": { + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==", + "dev": true + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "dev": true, + "requires": { + "mime-db": "1.40.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "dev": true + } + } + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + } + } + }, + "modern-syslog": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/modern-syslog/-/modern-syslog-1.1.2.tgz", + "integrity": "sha1-8fpYiZ8/RS14jxVzQBISpO+JjeU=", + "optional": true, + "requires": { + "nan": "^2.0.5" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "nan": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", + "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==", + "optional": true + }, + "neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + }, + "nested-error-stacks": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz", + "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", + "dev": true + }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "nodeunit": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/nodeunit/-/nodeunit-0.11.3.tgz", + "integrity": "sha512-gDNxrDWpx07BxYNO/jn1UrGI1vNhDQZrIFphbHMcTCDc5mrrqQBWfQMXPHJ5WSgbFwD1D6bv4HOsqtTrPG03AA==", + "dev": true, + "requires": { + "ejs": "^2.5.2", + "tap": "^12.0.1" + } + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "requires": { + "path-key": "^2.0.0" + } + }, + "nyc": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/nyc/-/nyc-14.1.1.tgz", + "integrity": "sha512-OI0vm6ZGUnoGZv/tLdZ2esSVzDwUC88SNs+6JoSOMVxA+gKMB8Tk7jBwgemLx4O40lhhvZCVw1C+OYLOBOPXWw==", + "dev": true, + "requires": { + "archy": "^1.0.0", + "caching-transform": "^3.0.2", + "convert-source-map": "^1.6.0", + "cp-file": "^6.2.0", + "find-cache-dir": "^2.1.0", + "find-up": "^3.0.0", + "foreground-child": "^1.5.6", + "glob": "^7.1.3", + "istanbul-lib-coverage": "^2.0.5", + "istanbul-lib-hook": "^2.0.7", + "istanbul-lib-instrument": "^3.3.0", + "istanbul-lib-report": "^2.0.8", + "istanbul-lib-source-maps": "^3.0.6", + "istanbul-reports": "^2.2.4", + "js-yaml": "^3.13.1", + "make-dir": "^2.1.0", + "merge-source-map": "^1.1.0", + "resolve-from": "^4.0.0", + "rimraf": "^2.6.3", + "signal-exit": "^3.0.2", + "spawn-wrap": "^1.4.2", + "test-exclude": "^5.2.3", + "uuid": "^3.3.2", + "yargs": "^13.2.2", + "yargs-parser": "^13.0.0" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "opener": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", + "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==", + "dev": true + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + }, + "dependencies": { + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", + "dev": true + } + } + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "dev": true + }, + "os-locale": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", + "dev": true, + "requires": { + "execa": "^1.0.0", + "lcid": "^2.0.0", + "mem": "^4.0.0" + } + }, + "own-or": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/own-or/-/own-or-1.0.0.tgz", + "integrity": "sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=", + "dev": true + }, + "own-or-env": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/own-or-env/-/own-or-env-1.0.1.tgz", + "integrity": "sha512-y8qULRbRAlL6x2+M0vIe7jJbJx/kmUTzYonRAa2ayesR2qWLswninkVyeJe4x3IEXhdgoNodzjQRKAoEs6Fmrw==", + "dev": true, + "requires": { + "own-or": "^1.0.0" + } + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true + }, + "p-is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", + "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", + "dev": true + }, + "p-limit": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", + "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "package-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-3.0.0.tgz", + "integrity": "sha512-lOtmukMDVvtkL84rJHI7dpTYq+0rli8N2wlnqUcBuDWCfVhRUfOmnR9SsoHFMLpACvEV60dX7rd0rFaYDZI+FA==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.15", + "hasha": "^3.0.0", + "lodash.flattendeep": "^4.4.0", + "release-zalgo": "^1.0.0" + } + }, + "parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "requires": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "optional": true + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "psl": { + "version": "1.1.33", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.33.tgz", + "integrity": "sha512-LTDP2uSrsc7XCb5lO7A8BI1qYxRe/8EqlRvMeEl6rsnYAqDOl8xHR+8lSAIVfrNaSAlTPTNOCgNjWcoUL3AZsw==", + "dev": true + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + }, + "read-pkg-up": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz", + "integrity": "sha512-6etQSH7nJGsK0RbG/2TeDzZFa8shjQ1um+SwQQ5cwKy0dhSXdOncEhb1CPpvQG4h7FyOV6EB6YlV0yJvZQNAkA==", + "dev": true, + "requires": { + "find-up": "^3.0.0", + "read-pkg": "^3.0.0" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "release-zalgo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", + "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=", + "dev": true, + "requires": { + "es6-error": "^4.0.1" + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "dev": true, + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "resolve": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", + "integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + }, + "sequence": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/sequence/-/sequence-2.2.1.tgz", + "integrity": "sha1-f1YXiV1ENRwKBH52RGdpBJChawM=", + "optional": true + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "simple-lru-cache": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", + "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=", + "optional": true + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true + }, + "source-map-support": { + "version": "0.5.12", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.12.tgz", + "integrity": "sha512-4h2Pbvyy15EE02G+JOZpUCmqWJuqrs+sEkzewTm++BPi7Hvn/HwcqLAcNxYAyI0x13CpPPn+kMjl+hplXMHITQ==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "spawn-wrap": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-1.4.2.tgz", + "integrity": "sha512-vMwR3OmmDhnxCVxM8M+xO/FtIp6Ju/mNaDfCMMW7FDcLRTPFWUswec4LXJHTJE2hwTI9O0YBfygu4DalFl7Ylg==", + "dev": true, + "requires": { + "foreground-child": "^1.5.6", + "mkdirp": "^0.5.0", + "os-homedir": "^1.0.1", + "rimraf": "^2.6.2", + "signal-exit": "^3.0.2", + "which": "^1.3.0" + } + }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.4.tgz", + "integrity": "sha512-7j8LYJLeY/Yb6ACbQ7F76qy5jHkp0U6jgBfJsk97bwWlVUnUWsAgpyaCvo17h0/RQGnQ036tVDomiwoI4pDkQA==", + "dev": true + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "stack-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz", + "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "optional": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "tap": { + "version": "12.7.0", + "resolved": "https://registry.npmjs.org/tap/-/tap-12.7.0.tgz", + "integrity": "sha512-SjglJmRv0pqrQQ7d5ZBEY8ZOqv3nYDBXEX51oyycOH7piuhn82JKT/yDNewwmOsodTD/RZL9MccA96EjDgK+Eg==", + "dev": true, + "requires": { + "bind-obj-methods": "^2.0.0", + "browser-process-hrtime": "^1.0.0", + "capture-stack-trace": "^1.0.0", + "clean-yaml-object": "^0.1.0", + "color-support": "^1.1.0", + "coveralls": "^3.0.2", + "domain-browser": "^1.2.0", + "esm": "^3.2.5", + "foreground-child": "^1.3.3", + "fs-exists-cached": "^1.0.0", + "function-loop": "^1.0.1", + "glob": "^7.1.3", + "isexe": "^2.0.0", + "js-yaml": "^3.13.1", + "minipass": "^2.3.5", + "mkdirp": "^0.5.1", + "nyc": "^14.0.0", + "opener": "^1.5.1", + "os-homedir": "^1.0.2", + "own-or": "^1.0.0", + "own-or-env": "^1.0.1", + "rimraf": "^2.6.3", + "signal-exit": "^3.0.0", + "source-map-support": "^0.5.10", + "stack-utils": "^1.0.2", + "tap-mocha-reporter": "^3.0.9", + "tap-parser": "^7.0.0", + "tmatch": "^4.0.0", + "trivial-deferred": "^1.0.1", + "ts-node": "^8.0.2", + "tsame": "^2.0.1", + "typescript": "^3.3.3", + "write-file-atomic": "^2.4.2", + "yapool": "^1.0.0" + } + }, + "tap-mocha-reporter": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/tap-mocha-reporter/-/tap-mocha-reporter-3.0.9.tgz", + "integrity": "sha512-VO07vhC9EG27EZdOe7bWBj1ldbK+DL9TnRadOgdQmiQOVZjFpUEQuuqO7+rNSO2kfmkq5hWeluYXDWNG/ytXTQ==", + "dev": true, + "requires": { + "color-support": "^1.1.0", + "debug": "^2.1.3", + "diff": "^1.3.2", + "escape-string-regexp": "^1.0.3", + "glob": "^7.0.5", + "js-yaml": "^3.3.1", + "readable-stream": "^2.1.5", + "tap-parser": "^5.1.0", + "unicode-length": "^1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "tap-parser": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/tap-parser/-/tap-parser-5.4.0.tgz", + "integrity": "sha512-BIsIaGqv7uTQgTW1KLTMNPSEQf4zDDPgYOBRdgOfuB+JFOLRBfEu6cLa/KvMvmqggu1FKXDfitjLwsq4827RvA==", + "dev": true, + "requires": { + "events-to-array": "^1.0.1", + "js-yaml": "^3.2.7", + "readable-stream": "^2" + } + } + } + }, + "tap-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tap-parser/-/tap-parser-7.0.0.tgz", + "integrity": "sha512-05G8/LrzqOOFvZhhAk32wsGiPZ1lfUrl+iV7+OkKgfofZxiceZWMHkKmow71YsyVQ8IvGBP2EjcIjE5gL4l5lA==", + "dev": true, + "requires": { + "events-to-array": "^1.0.1", + "js-yaml": "^3.2.7", + "minipass": "^2.2.0" + } + }, + "temp": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.4.0.tgz", + "integrity": "sha1-ZxrWPVe+D+nXKUZks/xABjZnimA=", + "dev": true + }, + "test-exclude": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", + "integrity": "sha512-M+oxtseCFO3EDtAaGH7iiej3CBkzXqFMbzqYAACdzKui4eZA+pq3tZEwChvOdNfa7xxy8BfbmgJSIr43cC/+2g==", + "dev": true, + "requires": { + "glob": "^7.1.3", + "minimatch": "^3.0.4", + "read-pkg-up": "^4.0.0", + "require-main-filename": "^2.0.0" + } + }, + "tmatch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tmatch/-/tmatch-4.0.0.tgz", + "integrity": "sha512-Ynn2Gsp+oCvYScQXeV+cCs7citRDilq0qDXA6tuvFwDgiYyyaq7D5vKUlAPezzZR5NDobc/QMeN6e5guOYmvxg==", + "dev": true + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "dev": true, + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, + "trim-right": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", + "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", + "dev": true + }, + "trivial-deferred": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trivial-deferred/-/trivial-deferred-1.0.1.tgz", + "integrity": "sha1-N21NKdlR1jaKb3oK6FwvTV4GWPM=", + "dev": true + }, + "ts-node": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.3.0.tgz", + "integrity": "sha512-dyNS/RqyVTDcmNM4NIBAeDMpsAdaQ+ojdf0GOLqE6nwJOgzEkdRNzJywhDfwnuvB10oa6NLVG1rUJQCpRN7qoQ==", + "dev": true, + "requires": { + "arg": "^4.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.6", + "yn": "^3.0.0" + }, + "dependencies": { + "diff": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", + "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==", + "dev": true + } + } + }, + "tsame": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tsame/-/tsame-2.0.1.tgz", + "integrity": "sha512-jxyxgKVKa4Bh5dPcO42TJL22lIvfd9LOVJwdovKOnJa4TLLrHxquK+DlGm4rkGmrcur+GRx+x4oW00O2pY/fFw==", + "dev": true + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, + "typescript": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.2.tgz", + "integrity": "sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==", + "dev": true + }, + "uglify-js": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", + "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "dev": true, + "optional": true, + "requires": { + "commander": "~2.20.0", + "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "dev": true, + "optional": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "optional": true + } + } + }, + "underscore": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz", + "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ=", + "dev": true + }, + "unicode-length": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/unicode-length/-/unicode-length-1.0.3.tgz", + "integrity": "sha1-Wtp6f+1RhBpBijKM8UlHisg1irs=", + "dev": true, + "requires": { + "punycode": "^1.3.2", + "strip-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true, + "optional": true + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "winser": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/winser/-/winser-0.1.6.tgz", + "integrity": "sha1-CGY9wyh4oSu84WLYQNpQl7SEZsk=", + "optional": true, + "requires": { + "commander": "1.3.1", + "sequence": "2.2.1" + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + }, + "yapool": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yapool/-/yapool-1.0.0.tgz", + "integrity": "sha1-9pPymjFbUNmp2iZGp6ZkXJaYW2o=", + "dev": true + }, + "yargs": { + "version": "13.2.4", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.4.tgz", + "integrity": "sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "os-locale": "^3.1.0", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.0" + } + }, + "yargs-parser": { + "version": "13.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", + "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "yn": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.0.tgz", + "integrity": "sha512-kKfnnYkbTfrAdd0xICNFw7Atm8nKpLcLv9AZGEt+kczL/WQVai4e2V6ZN8U/O+iI6WrNuJjNNOyu4zfhl9D3Hg==", + "dev": true + } + } +} diff --git a/package.json b/package.json index 643b9f77..3066f1f0 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "generic-pool": "2.2.0" }, "devDependencies": { - "nodeunit": "0.9.x", + "nodeunit": "^0.11.3", "underscore": "1.4.x", "temp": "0.4.x" }, From f86f834524ebf60ab715a5cd415afddb69b45e49 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 11 Jul 2019 12:51:30 +0100 Subject: [PATCH 153/207] update package.json version to 0.8.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3066f1f0..d8638aaf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.2", + "version": "0.8.3", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From a2b926552bb88d3b7fe5b48e459c0bf2984d3a60 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 11 Jul 2019 13:12:04 +0100 Subject: [PATCH 154/207] update modern-syslog to 1.2.0 for node 12 compatibility --- package-lock.json | 10 +++++----- package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 363189e0..9b0a25bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.2", + "version": "0.8.3", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1225,12 +1225,12 @@ } }, "modern-syslog": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/modern-syslog/-/modern-syslog-1.1.2.tgz", - "integrity": "sha1-8fpYiZ8/RS14jxVzQBISpO+JjeU=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/modern-syslog/-/modern-syslog-1.2.0.tgz", + "integrity": "sha512-dmFE23qpyZJf8MOdzuNKliW4j1PCqxaRtSzyNnv6QDUWjf1z8T4ZoQ7Qf0t6It2ewNv9/XJZSJoUgwpq3D0X7A==", "optional": true, "requires": { - "nan": "^2.0.5" + "nan": "^2.13.2" } }, "ms": { diff --git a/package.json b/package.json index d8638aaf..a752e34e 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "temp": "0.4.x" }, "optionalDependencies": { - "modern-syslog": "1.1.2", + "modern-syslog": "1.2.0", "hashring": "3.2.0", "winser": "=0.1.6" }, From dd99864886415d2191daf2f2e50e0d6a5c124e34 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 11 Jul 2019 13:14:57 +0100 Subject: [PATCH 155/207] update package.json version 0.8.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9b0a25bd..3bdd07fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.3", + "version": "0.8.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a752e34e..a26b99cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.3", + "version": "0.8.4", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 338f1e944fe07836b6bc840e8e684676fc9015c4 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 11 Jul 2019 14:09:12 +0100 Subject: [PATCH 156/207] Add docker image info to readme --- README.md | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 36408028..8d15cf14 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,11 @@ -StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) -====== +# StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) A network daemon that runs on the [Node.js][node] platform and listens for statistics, like counters and timers, sent over [UDP][udp] or [TCP][tcp] and sends aggregates to one or more pluggable backend services (e.g., [Graphite][graphite]). -Key Concepts --------- +## Key Concepts * *buckets* Each stat is in its own "bucket". They are not predefined anywhere. Buckets @@ -23,17 +21,21 @@ general values should be integers. default 10 seconds), stats are aggregated and sent to an upstream backend service. -Installation and Configuration ------------------------------- +## Installation and Configuration +### Docker +Statsd supports docker in two ways: +* The official docker image on on [docker hub](https://hub.docker.com/r/statsd/statsd) +* Building the image from the bundled [Dockerfile](./Dockerfile) + +### Manual installation * Install Node.js (All [`Current` and `LTS` Node.js versions](https://nodejs.org/en/about/releases/) are supported.) * Clone the project * Create a config file from `exampleConfig.js` and put it somewhere * Start the Daemon: `node stats.js /path/to/config` -Usage -------- +## Usage The basic line protocol expects metrics to be sent in the format: :| @@ -43,8 +45,7 @@ StatsD running with the default UDP server on localhost would be: echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 -More Specific Topics --------- +## More Specific Topics * [Metric Types][docs_metric_types] * [Graphite Integration][docs_graphite] * [Supported Servers][docs_server] @@ -55,9 +56,7 @@ More Specific Topics * [Metric Namespacing][docs_namespacing] * [Statsd Cluster Proxy][docs_cluster_proxy] -Debugging ---------- - +## Debugging There are additional config variables available for debugging: * `debug` - log exceptions and print out more diagnostic info @@ -66,9 +65,7 @@ There are additional config variables available for debugging: For more information, check the `exampleConfig.js`. -Tests ------ - +## Tests A test framework has been added using node-unit and some custom code to start and manipulate statsd. Please add tests under test/ for any new features or bug fixes encountered. Testing a live server can be tricky, attempts were made to @@ -78,14 +75,11 @@ background (don't do this on a production machine!). Tests can be executed with `./run_tests.sh`. -History ---------- +## History statsd was originally written at ([Etsy][etsy]) and released with a [blog post][blog post] about how it works and why we created it. -Inspiration ------------ - +## Inspiration StatsD was inspired (heavily) by the project (of the same name) at Flickr. Here's a post where Cal Henderson described it in depth: [Counting and timing][counting-timing] From cd3821484bad06e46f66b4bc77994737d02aa1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Here=C3=B1=C3=BA?= Date: Sat, 22 Jun 2019 12:49:32 -0300 Subject: [PATCH 157/207] Minor formatting proposals --- docs/admin_interface.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/admin_interface.md b/docs/admin_interface.md index 8c2b84d0..845ebe36 100644 --- a/docs/admin_interface.md +++ b/docs/admin_interface.md @@ -1,10 +1,10 @@ TCP Stats Interface =================== -A really simple TCP management interface is available by default on port 8126 +A really simple TCP management interface is available by default on port `8126` or overriden in the configuration file. Inspired by the memcache stats approach this can be used to monitor a live statsd server. You can interact with the -management server by telnetting to port 8126, the following commands are +management server by telnetting to port `8126`, the following commands are available based on the running server. Common commands @@ -55,7 +55,7 @@ Those statistics will also be sent to graphite under the namespaces `stats.statsd.graphiteStats.last_exception` and `stats.statsd.graphiteStats.last_flush`. -A simple nagios check can be found in the utils/ directory that can be used to +A simple nagios check can be found in the `utils/` directory that can be used to check metric thresholds, for example the number of seconds since the last successful flush to graphite. From 062fd4b372d37d852e9767f82a99d43ce7eafc09 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 23 Jul 2019 15:53:09 +0100 Subject: [PATCH 158/207] Migrate docs from github wiki, and standardise markdown notation --- docs/additional_tools.md | 5 ++ docs/admin_interface.md | 12 ++-- docs/backend.md | 60 +++++++++--------- docs/backend_interface.md | 3 +- docs/client_implementations.md | 108 +++++++++++++++++++++++++++++++++ docs/cluster_proxy.md | 8 +-- docs/graphite.md | 3 +- docs/graphite_pickle.md | 9 +-- docs/metric_types.md | 26 +++----- docs/namespacing.md | 4 +- docs/protocol.md | 3 + docs/server.md | 3 +- docs/server_implementations.md | 18 ++++++ docs/server_interface.md | 3 +- 14 files changed, 190 insertions(+), 75 deletions(-) create mode 100644 docs/additional_tools.md create mode 100644 docs/client_implementations.md create mode 100644 docs/protocol.md create mode 100644 docs/server_implementations.md diff --git a/docs/additional_tools.md b/docs/additional_tools.md new file mode 100644 index 00000000..99aa04a3 --- /dev/null +++ b/docs/additional_tools.md @@ -0,0 +1,5 @@ +# Additional Tools + +The following are tools you might find useful when using, contributing, or testing statsd. + +* [statsd-tg](http://octo.it/statsd-tg) – StatsD traffic generator; generates dummy traffic for load testing (C). diff --git a/docs/admin_interface.md b/docs/admin_interface.md index 845ebe36..0c1c9b2e 100644 --- a/docs/admin_interface.md +++ b/docs/admin_interface.md @@ -1,5 +1,4 @@ -TCP Stats Interface -=================== +# TCP Stats Interface A really simple TCP management interface is available by default on port `8126` or overriden in the configuration file. Inspired by the memcache stats approach @@ -7,15 +6,13 @@ this can be used to monitor a live statsd server. You can interact with the management server by telnetting to port `8126`, the following commands are available based on the running server. -Common commands ---------------- +## Common commands * health [up|down] - a way to get/set the health status of statsd. Alone will get you the current health status. Passing a second command will set the status to the new value. Accepted values are _up_ and _down_. * config - a dump of the current configuration * quit - close the connection from the server side -Statsd specific commands ------------------------- +## Statsd specific commands * stats - some stats about the running server * counters - a dump of all the current counters @@ -64,8 +61,7 @@ The health output: * using health up or health down, you can change the current health status. * the healthStatus configuration option allows you to set the default health status at start. -Statsd Proxy specific commands ------------------------------- +## Statsd Proxy specific commands * status - the status of the current server diff --git a/docs/backend.md b/docs/backend.md index e9917b41..7839e1db 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -1,5 +1,4 @@ -Supported Backends ------------------- +# Supported Backends StatsD supports pluggable backend modules that can publish statistics from the local StatsD daemon to a backend service or data @@ -30,33 +29,34 @@ giving the relative path (e.g. `./backends/graphite`). A robust set of are also available as plugins to allow easy reporting into databases, queues and third-party services. -## Available Third-party backends -- [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) -- [atsd-backend](https://github.com/axibase/atsd-statsd-backend) -- [aws-cloudwatch-backend](https://github.com/camitz/aws-cloudwatch-statsd-backend) -- [node-bell](https://github.com/eleme/node-bell) -- [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) -- [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) -- [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) -- [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) -- [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) -- [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) -- [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) -- [jut-backend](https://github.com/jut-io/statsd-jut-backend) -- [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) -- [librato-backend](https://github.com/librato/statsd-librato-backend) -- [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) -- [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) -- [mysql backend](https://github.com/fradinni/nodejs-statsd-mysql-backend) -- [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) -- [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) -- [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) -- [socket.io-backend](https://github.com/Chatham/statsd-socket.io) -- [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) -- [statsd-backend](https://github.com/dynmeth/statsd-backend) -- [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) -- [statsd aggregation backend](https://github.com/wanelo/gossip_girl) -- [warp10-backend](https://github.com/cityzendata/statsd-warp10-backend) -- [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) +## Available third-party backends + +* [amqp-backend](https://github.com/mrtazz/statsd-amqp-backend) +* [atsd-backend](https://github.com/axibase/atsd-statsd-backend) +* [aws-cloudwatch-backend](https://github.com/camitz/aws-cloudwatch-statsd-backend) +* [node-bell](https://github.com/eleme/node-bell) +* [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) +* [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) +* [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) +* [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) +* [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) +* [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) +* [instrumental backend](https://github.com/collectiveidea/statsd-instrumental-backend) +* [jut-backend](https://github.com/jut-io/statsd-jut-backend) +* [leftronic backend](https://github.com/sreuter/statsd-leftronic-backend) +* [librato-backend](https://github.com/librato/statsd-librato-backend) +* [mongo-backend](https://github.com/dynmeth/mongo-statsd-backend) +* [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) +* [mysql backend](https://github.com/fradinni/nodejs-statsd-mysql-backend) +* [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) +* [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) +* [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) +* [socket.io-backend](https://github.com/Chatham/statsd-socket.io) +* [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) +* [statsd-backend](https://github.com/dynmeth/statsd-backend) +* [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) +* [statsd aggregation backend](https://github.com/wanelo/gossip_girl) +* [warp10-backend](https://github.com/cityzendata/statsd-warp10-backend) +* [zabbix-backend](https://github.com/parkerd/statsd-zabbix-backend) [graphite]: https://graphite.readthedocs.io/en/latest/ diff --git a/docs/backend_interface.md b/docs/backend_interface.md index c71806b7..c8a3feac 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -1,5 +1,4 @@ -Backend Interface ------------------ +# Backend Interface Backend modules are Node.js [modules][nodemods] that listen for a number of events emitted from StatsD. Each backend module should diff --git a/docs/client_implementations.md b/docs/client_implementations.md new file mode 100644 index 00000000..e8b66746 --- /dev/null +++ b/docs/client_implementations.md @@ -0,0 +1,108 @@ +# StatsD Clients + +A number of clients have been made for pushing metrics into statsd and open sourced by the wider community. + +**Node** +* [lynx](https://github.com/dscape/lynx) — Node.js client used by Mozilla, Nodejitsu, etc. +* [Node-Statsd](https://github.com/sivy/node-statsd) — Node.js client +* [node-statsd-client](https://github.com/msiebuhr/node-statsd-client) — Node.js client +* [node-statsd-instrument](https://github.com/syrio/node-statsd-instrument) — Node.js client +* [statistik](https://github.com/godmodelabs/statistik) - Node.js client with timers & CLI +* [statsy](https://github.com/segmentio/statsy) - clean idiomatic statsd client + +**Java** +* [java-statsd-client](https://github.com/youdevise/java-statsd-client) — Lightweight (zero deps) Java client +* [Statsd over SLF4J](https://github.com/nzjess/statsd-over-slf4j) — Java client with SLF4J logging tie-in +* [play-statsd](https://github.com/vznet/play-statsd) — Play Framework 2.0 client for Java and Scala +* [statsd-netty](https://github.com/flozano/statsd-netty) — Netty-based Java 8 client + +**Python** +* [Py-Statsd](https://github.com/sivy/py-statsd) — Server and Client +* [Python-Statsd](https://github.com/WoLpH/python-statsd) — Python client +* [pystatsd](https://github.com/jsocol/pystatsd) — Python client +* [Django-Statsd](https://github.com/WoLpH/django-statsd) — Django client + +**Ruby** +* [statsd-instrument](https://github.com/Shopify/statsd-instrument) — Ruby client +* [statsd](https://github.com/reinh/statsd/) — Ruby client (needs new maintainer) +* [Statsd-Client](https://github.com/dawanda/statsd-client) — Ruby client (not maintained) + +**Perl** +* [Net::Statsd](https://github.com/cosimo/perl5-net-statsd) — Perl client, also available on [CPAN](https://metacpan.org/module/Net::Statsd) +* [Net::StatsD::Client](https://github.com/sivy/statsd-client) — Perl client, not available on CPAN +* [Etsy::StatsD](https://github.com/sanbeg/Etsy-Statsd) - Perl client, also available on [CPAN] (https://metacpan.org/module/Etsy::StatsD) + +**PHP** +* [Metrics](https://github.com/beberlei/metrics#metrics) +* [PHP client](https://gist.github.com/1065177/5f7debc212724111f9f500733c626416f9f54ee6) +* [php-statsd](https://github.com/seejohnrun/php-statsd) and Spark +* [php-statsd-client](https://github.com/godmodelabs/php-statsd-client) - supports SplClassLoader +* [statsd-php-client](https://github.com/iFixit/statsd-php-client) - Minimalist performant client +* [phpLeague-statsd-client](https://github.com/thephpleague/statsd) - Php League StatsD client +* [statsd-php-client](https://github.com/liuggio/statsd-php-client) - optimized client with monolog and symfony2 integrations available +* [statsd-php](https://github.com/domnikl/statsd-php) - PSR-4 compatible client + +**Clojure** +* [Clojure client](https://github.com/pyr/clj-statsd) + +**Io** +* [io-statsd](https://github.com/seejohnrun/io-statsd) — StatsD Client for Io + +**C** +* [C client](https://github.com/romanbsd/statsd-c-client) — A trivial C client + +**C++** +* [statsd-client-cpp](https://github.com/talebook/statsd-client-cpp) — StatsD Client in CPP +* [cpp-statsd-client](https://github.com/vthiery/cpp-statsd-client) — A header-only StatsD client implemented in C++ + +**.NET** +* [NStatsD.Client](https://github.com/robbihun/NStatsD.Client) — .NET 4.0 client +* [C# client](https://github.com/goncalopereira/statsd-csharp-client) — C# client +* [graphite-client](https://github.com/peschuster/graphite-client) — .NET client library for StatsD and Graphite +* [StatsC](https://bitbucket.org/pavlos256/statsc) — An asynchronous client with built-in support for batching +* [JustEat.StatsD](https://github.com/justeat/JustEat.StatsD) — A .NET library for publishing metrics to statsd. Targets both .NET full framework and .NET Standard 2.0. + +**Go** +* [GoE](https://godoc.org/github.com/pascaldekloe/goe/metrics) — Minimal & Performant +* [go-statsd-client](https://github.com/cactus/go-statsd-client) — Simple Go client +* [g2s](https://github.com/peterbourgon/g2s) +* [StatsD](https://github.com/quipo/statsd) +* [statsd](https://github.com/alexcesaro/statsd) — A simple and very fast StatsD client + +**Apache** +* [mod_statsd](https://github.com/jib/mod_statsd) - StatsD client to send stats straight from [Apache](https://modules.apache.org/) + +**Varnish** +* [libvmod-statsd](https://github.com/jib/libvmod-statsd) - StatsD client to send stats straight from [Varnish](http://varnish-cache.org) + +**PowerShell** +* [powershell-statsd](https://github.com/joehack3r/powershell-statsd) - PowerShell client + +**Browser** +* [StatsC](https://github.com/godmodelabs/statsc) - Push stats to StatsD from the browser! +* [StatsD HTTP Proxy](https://github.com/sokil/statsd-http-proxy) - HTTP proxy to StatsD with REST interface for using in browsers +* [StatsD HTTP Client](https://github.com/Molyakos/statsd-http-client) - StatsD client over http for using in browsers + +**Objective-C** +* [MCStatsd](https://github.com/Marketcircle/MCStatsd) - Cocoa client + +**ActionScript** +* [flash-statsd](https://github.com/simongregory/flash-statsd) - Flash client + +**WordPress** +* [wordpress-statsd](https://github.com/uglyrobot/wordpress-statsd) - WordPress Plugin + +**Drupal** +* [StatsD](https://www.drupal.org/project/statsd) - Drupal module + +**Haskell** +* [statsd-client](https://github.com/keithduncan/statsd-client) + +**R** +* [rstatsd](https://github.com/stumpyfr/rstatsd) + +**Lua** +* [lua-statsd](https://github.com/stvp/lua-statsd-client) + +**Nim** +* [statsd_client](https://github.com/FedericoCeratto/nim-statsd-client) diff --git a/docs/cluster_proxy.md b/docs/cluster_proxy.md index ca2d39a2..cef4514f 100644 --- a/docs/cluster_proxy.md +++ b/docs/cluster_proxy.md @@ -1,9 +1,7 @@ -Statsd Cluster Proxy -============== +# Statsd Cluster Proxy Statsd Cluster Proxy is a udp proxy that sits infront of multiple statsd instances. - Create a proxyConfig.js file: `cp exampleProxyConfig.js proxyConfig.js` @@ -20,8 +18,8 @@ It handles a simple health check that dynamically recalculates the hashring if a Config Options are documented in the [exampleProxyConfig.js][exampleProxyConfig.js] -Notes --------------- +## Notes + In your statsd configuration make sure to have the following configuration set: `deleteIdleStats: true` We plan to remove this restriction in the near future: [#pull/348][pull_348] diff --git a/docs/graphite.md b/docs/graphite.md index 7a7bfc55..f74dc7ca 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -1,5 +1,4 @@ -Configuring Graphite for StatsD -------------------------------- +# Configuring Graphite for StatsD Many users have been confused to see their hit counts averaged, gone missing when the data is intermittent, or never stored when statsd is sending at a different diff --git a/docs/graphite_pickle.md b/docs/graphite_pickle.md index 2151f59f..3ddcfe4e 100644 --- a/docs/graphite_pickle.md +++ b/docs/graphite_pickle.md @@ -1,5 +1,4 @@ -Pickling for Graphite -===================== +# Pickling for Graphite The graphite statsd backend can optionally be configured to use pickle for its over-the-wire protocol. @@ -15,8 +14,7 @@ more CPU processing by the graphite endpoint. The message format expected by the graphite pickle endpoint consists of a header and payload. -The Payload ------------ +## The Payload The message payload is a list of tuples. Each tuple contains the measurement for a single metric name. The measurement is encoded as a second, @@ -60,8 +58,7 @@ The trailing `L` for long fields is unnecessary, but we are adding the character to match Python pickle output. It's a side-effect of `repr(long(1234))`. -The Header ----------- +## The Header The message header is a 32-bit integer sent over the wire as four-bytes. This integer must describe the length of the pickled diff --git a/docs/metric_types.md b/docs/metric_types.md index 53cdda23..ffa93c1a 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -1,9 +1,6 @@ -StatsD Metric Types -================== +# StatsD Metric Types - -Counting --------- +## Counting gorets:1|c @@ -13,14 +10,13 @@ If the count at flush is 0 then you can opt to send no metric at all for this counter, by setting `config.deleteCounters` (applies only to graphite backend). Statsd will send both the rate as well as the count at each flush. -### Sampling +## Sampling gorets:1|c|@0.1 Tells StatsD that this counter is being sent sampled every 1/10th of the time. -Timing ------- +## Timing glork:320|ms|@0.1 @@ -79,8 +75,8 @@ Note: histograms, as you can make each bin arbitrarily wide, i.e. class intervals of different sizes. -Gauges ------- +## Gauges + StatsD now also supports gauges, arbitrary values, which can be recorded. gaugor:333|g @@ -101,8 +97,8 @@ Note: This implies you can't explicitly set a gauge to a negative number without first setting it to zero. -Sets ----- +## Sets + StatsD supports counting unique occurences of events between flushes, using a Set to store all occuring events. @@ -111,8 +107,8 @@ using a Set to store all occuring events. If the count at flush is 0 then you can opt to send no metric at all for this set, by setting `config.deleteSets`. -Multi-Metric Packets --------------------- +## Multi-Metric Packets + StatsD supports receiving multiple metrics in a single packet by separating them with a newline. @@ -130,5 +126,3 @@ scenarios: of all the hops in your route. *(These payload numbers take into account the maximum IP + UDP header sizes)* - - diff --git a/docs/namespacing.md b/docs/namespacing.md index cba54762..fb5956b9 100644 --- a/docs/namespacing.md +++ b/docs/namespacing.md @@ -1,5 +1,5 @@ -Metric namespacing -------------------- +# Metric namespacing + The metric namespacing in the Graphite backend is configurable with regard to the prefixes. Per default all stats are put under `stats` in Graphite, which makes it easier to consolidate them all under one schema. However it is diff --git a/docs/protocol.md b/docs/protocol.md new file mode 100644 index 00000000..87aad92e --- /dev/null +++ b/docs/protocol.md @@ -0,0 +1,3 @@ +# The StatsD Protocol + +Coming soon! Meanwhile, see https://github.com/b/statsd_spec diff --git a/docs/server.md b/docs/server.md index 7b85b05a..8516f112 100644 --- a/docs/server.md +++ b/docs/server.md @@ -1,5 +1,4 @@ -Supported Servers ------------------- +# Supported Servers StatsD supports pluggable server modules that listen for incoming metrics. diff --git a/docs/server_implementations.md b/docs/server_implementations.md new file mode 100644 index 00000000..a8a479e7 --- /dev/null +++ b/docs/server_implementations.md @@ -0,0 +1,18 @@ +# Server Implementations + +The following is a list of projects that re-implement statsd, if the the main project isn't for you, perhaps one of these is. + +* [brubeck](https://github.com/github/brubeck) - Server in C +* [clj-statsd-svr](https://github.com/netmelody/clj-statsd-svr) — Clojure server +* [gographite](https://github.com/amir/gographite) — Server in Go +* [gostatsd](https://github.com/atlassian/gostatsd) — Server in Go +* [netdata](https://github.com/firehol/netdata) - Embedded statsd server in the netdata server, in C, with visualization +* [Net::Statsd::Server](https://github.com/cosimo/perl5-net-statsd-server) — Perl server, also available on [CPAN](https://metacpan.org/module/Net::Statsd::Server) +* [Py-Statsd](https://github.com/sivy/py-statsd) — Server and Client +* [Ruby-Statsdserver](https://github.com/fetep/ruby-statsdserver) — Ruby server +* [statsd-c](https://github.com/jbuchbinder/statsd-c) — Server in C +* [statsdaemon (bitly)](https://github.com/bitly/statsdaemon) — Server in Go +* [statsdaemon (vimeo)](https://github.com/vimeo/statsdaemon) — Server in Go +* [statsdcc](https://github.com/wayfair/statsdcc) - Server in C++ +* [statsdpy](https://github.com/pandemicsyn/statsdpy) — Python/eventlet Server +* [statsite](https://github.com/armon/statsite.git) — Server in C diff --git a/docs/server_interface.md b/docs/server_interface.md index b0b1e416..e208e759 100644 --- a/docs/server_interface.md +++ b/docs/server_interface.md @@ -1,5 +1,4 @@ -Server Interface ------------------ +# Server Interface Server modules are Node.js [modules][nodemods] that receive metrics for StatsD. Each server module should export the following initialization function: From 9c985196f08156d9e794831a8e013711d40a2198 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 23 Jul 2019 15:55:07 +0100 Subject: [PATCH 159/207] Add third party server interfaces to docs --- docs/server_interface.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/server_interface.md b/docs/server_interface.md index e208e759..454bde32 100644 --- a/docs/server_interface.md +++ b/docs/server_interface.md @@ -1,6 +1,8 @@ # Server Interface Server modules are Node.js [modules][nodemods] that receive metrics for StatsD. +Server interfaces can be distributed and installed via systems such as NPM. + Each server module should export the following initialization function: * `start(config, callback)`: This method is invoked from StatsD to initialize @@ -15,3 +17,7 @@ Each server module should export the following initialization function: The server module should return `true` from start() to indicate success. A return of `false` indicates a failure to load the module (missing configuration?) and will cause StatsD to exit. + +# Available third-party interfaces + +* [http-interface](https://github.com/msiebuhr/statsd-http-interface) Accepts data over HTTP. From f1791d240ba698691c164e48ca2fc207f3f14837 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 23 Jul 2019 15:56:35 +0100 Subject: [PATCH 160/207] Add the statsd history to the docs --- docs/history.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/history.md diff --git a/docs/history.md b/docs/history.md new file mode 100644 index 00000000..c96b0fb0 --- /dev/null +++ b/docs/history.md @@ -0,0 +1,7 @@ +# What is StatsD? + +StatsD is a front-end proxy for the Graphite/Carbon metrics server, +originally written by Etsy's Erik Kastner. It is based on ideas from +Flickr and this post by Cal Henderson: Counting and Timing. The +server was written in Node, though there have been implementations +in other languages since then. From 33d4bfceb24b123914ff165d5c76755925262afd Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 23 Jul 2019 16:04:16 +0100 Subject: [PATCH 161/207] Update lodash (sub dependency) for security fix --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3bdd07fe..186e2904 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1073,9 +1073,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lodash.flattendeep": { From 3a3ced99de00855dfc0a45386f466ad5a8d8a549 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 23 Jul 2019 16:04:42 +0100 Subject: [PATCH 162/207] Release 0.8.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a26b99cd..ac7d8e3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.4", + "version": "0.8.5", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 3aa72f2842fd6936c2652ed4ffdf60a7ae3c9375 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 24 Jul 2019 16:02:02 +0100 Subject: [PATCH 163/207] Clarify how the gauge metric works --- docs/metric_types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metric_types.md b/docs/metric_types.md index ffa93c1a..6e95554f 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -77,7 +77,7 @@ i.e. class intervals of different sizes. ## Gauges -StatsD now also supports gauges, arbitrary values, which can be recorded. +StatsD also supports gauges. A gauge will take on the arbitrary value assigned to it, and will maintain it's value until it is next set. gaugor:333|g From 11cfd133855b9a953a2f22bb755c7fa07527e5df Mon Sep 17 00:00:00 2001 From: Shobhit Chittora Date: Fri, 13 Sep 2019 13:17:27 +0530 Subject: [PATCH 164/207] Fixes a small typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d15cf14..0ac11fd9 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ general values should be integers. ### Docker Statsd supports docker in two ways: -* The official docker image on on [docker hub](https://hub.docker.com/r/statsd/statsd) +* The official docker image on [docker hub](https://hub.docker.com/r/statsd/statsd) * Building the image from the bundled [Dockerfile](./Dockerfile) ### Manual installation From d47bf4dee0c07e44bb4fff608e223dfc071ba462 Mon Sep 17 00:00:00 2001 From: wyllys Date: Fri, 3 Jan 2020 11:23:47 -0500 Subject: [PATCH 165/207] Update Changelog.md and debian/changelog to reflect latest tagged releases. --- Changelog.md | 58 ++++++++++++++++++++++++++++++++++++ debian/changelog | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/Changelog.md b/Changelog.md index acfeab66..6a0e1431 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,63 @@ # Changelog +## v0.8.5 (07/23/2019) + +- Update lodash (sub dependency) for security fix +- Add the statsd history to the docs +- Add third party server interfaces to docs +- Migrate docs from github wiki, and standardise markdown notation +- Minor formatting proposals +- Add docker image info to readme + +## v0.8.4 (07/11/2019) + +- update modern-syslog to 1.2.0 for node 12 compatibility +- update package.json version to 0.8.3 + +## v0.8.3 (07/11/2019) + +- correct backend flush loop +- test and declare support for Current and LTS node +- Correct reporter decleration in test runner +- Convert codebase from var -> let / const (#673) +- correct npm test script +- correct travis deploy step + +## v0.8.2 (04/02/2019) + +- update travis npm token +- update dockerfile to latest node-lts +- correct gitter link +- update dockerfile base image to node lts +- Add gitter chat badge +- run tests using python 3.7's pickle rather than 2.x cPickle (#669) + +## v0.8.1 (03/13/2019) + +- drop statsd instance from proxy ring in instance of healthcheck failures (#665) +- Add myself (elliot blackburn) to maintainers.md (#666) +- add mysql backend link to docs/backend.md +- Adding myself to MAINTAINERS. +- begin testing on node lts and up +- Added "opencensus-backend" +- correct package.json links to new github organisation +- Update MAINTAINERS.md +- remove meta section of README +- Create MAINTAINERS.md +- Create DCO.txt +- Create CODE_OF_CONDUCT.md +- update README post transfer +- Fixing Markdown formatting +- fix simple typo +- Added StatsdClient Kotlin implementation +- fix formatting on backend interface docs +- Update: ignore files +- removes -q switch +- Fix for failing test on node 0.10 +- fix usage of process.EventEmitter +- Add plugin Warp10 to statsd +- Updated graphite link to read the docs + ## v0.8.0 (05/05/2016) - Modularized injest servers, with support for loading multiple servers - Added configurable tcp injest server diff --git a/debian/changelog b/debian/changelog index 7a3fd501..55cc954a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,80 @@ +statsd (0.8.5-1) unstable; urgency=low + + * Update lodash (sub dependency) for security fix + * Add the statsd history to the docs + * Add third party server interfaces to docs + * Migrate docs from github wiki, and standardise markdown notation + * Minor formatting proposals + * Add docker image info to readme + + -- Elliot Blackburn Tue, 23 Jul 2019 00:00:00 +0000 + +statsd (0.8.4-1) unstable; urgency=low + + * update modern-syslog to 1.2.0 for node 12 compatibility + * update package.json version to 0.8.3 + + -- Elliot Blackburn Thu, 11 Jul 2019 00:00:00 +0000 + +statsd (0.8.3-1) unstable; urgency=low + + * correct backend flush loop + * test and declare support for Current and LTS node + * Correct reporter decleration in test runner + * Convert codebase from var -> let / const (#673) + * correct npm test script + * correct travis deploy step + + -- Elliot Blackburn Thu, 11 Jul 2019 00:00:00 +0000 + +statsd (0.8.2-1) unstable; urgency=low + + * update travis npm token + * update dockerfile to latest node-lts + * correct gitter link + * update dockerfile base image to node lts + * Add gitter chat badge + * run tests using python 3.7's pickle rather than 2.x cPickle (#669) + + -- Elliot Blackburn Tue, 02 Apr 2019 00:00:00 +0000 + +statsd (0.8.1-1) unstable; urgency=low + + * drop statsd instance from proxy ring in instance of healthcheck failures (#665) + * Add myself (elliot blackburn) to maintainers.md (#666) + * add mysql backend link to docs/backend + -- Elliot Blackburn Wed, 13 Mar 2019 00:00:00 +0000 + +statsd (0.8.1-1) unstable; urgency=low + + * drop statsd instance from proxy ring in instance of healthcheck failures (#665) + * Add myself (elliot blackburn) to maintainers.md (#666) + * add mysql backend link to docs/backend.md + * Adding myself to MAINTAINERS. + * begin testing on node lts and up + * remove extra newline from merge conflict fix + * Added "opencensus-backend" + * correct package.json links to new github organisation + * Update MAINTAINERS.md + * Update MAINTAINERS.md + * remove meta section of README + * Create MAINTAINERS.md + * Create DCO.txt + * Create CODE_OF_CONDUCT.md + * update README post transfer + * Fixing Markdown formatting + * fix simple typo + * Added StatsdClient Kotlin implementation + * fix formatting on backend interface docs + * Update: ignore files + * removes -q switch + * Fix for failing test on node 0.10 + * fix usage of process.EventEmitter + * Add plugin Warp10 to statsd + * Updated graphite link to read the docs + + -- Elliot Blackburn Wed, 13 Mar 2019 00:00:00 +0000 + statsd (0.8.0-1) unstable; urgency=low * Modularized injest servers, with support for loading multiple servers From a7fa3aff963e21a98570392d9c496eb47fbe74ec Mon Sep 17 00:00:00 2001 From: Tristan Starck Date: Tue, 8 Jan 2019 21:35:20 -0800 Subject: [PATCH 166/207] add filter option for metrics Currently there are over a dozen metrics that get sent out for every timer stat that comes in. Being able to filter on the specific aggregated metrics you want for a timer at the config level will allow for drastic reduction in the data sent out from statsd and stored. The config that I've added is calculated_timer_metrics which by default will send all metrics, however once any other value(s) are added, it will only send those specified. This also allows for the not sending percentile metrics as well. Currently running these changes in production has reduced our Carbon/whisper load by ~20%. Related to issue 235 --- exampleConfig.js | 8 +- lib/process_metrics.js | 20 ++- stats.js | 2 +- test/graphite_tests_filters.js | 180 +++++++++++++++++++++ test/process_metrics_tests.js | 287 ++++++++++++++++++++++++++++----- 5 files changed, 456 insertions(+), 41 deletions(-) create mode 100644 test/graphite_tests_filters.js diff --git a/exampleConfig.js b/exampleConfig.js index 169dd80c..d61298d1 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -14,7 +14,7 @@ Optional Variables: graphiteProtocol: either 'text' or 'pickle' [default: 'text'] backends: an array of backends to load. Each backend must exist by name in the directory backends/. If not specified, - the default graphite backend will be loaded. + the default graphite backend will be loaded. * example for console and graphite: [ "./backends/console", "./backends/graphite" ] @@ -60,7 +60,7 @@ Optional Variables: log: location of log file for frequent keys [default: STDOUT] deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers as opposed to sending 0. For gauges, this unsets the gauge (instead of sending - the previous value). Can be individually overriden. [default: false] + the previous value). Can be individually overridden. [default: false] deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] @@ -71,6 +71,10 @@ Optional Variables: If disabled, it is up to the backends to sanitize keynames as appropriate per their storage requirements. + calculatedTimerMetrics: List of timer metrics that will be sent. Default will send all metrics. + To filter on percents and top percents: append '_percent' to the metric name. + Example: calculatedTimerMetrics: ['count', 'median', 'upper_percent', 'histogram'] + console: prettyprint: whether to prettyprint the console backend output [true or false, default: true] diff --git a/lib/process_metrics.js b/lib/process_metrics.js index 489eb39a..52c160d6 100644 --- a/lib/process_metrics.js +++ b/lib/process_metrics.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -const process_metrics = function (metrics, flushInterval, ts, flushCallback) { +const process_metrics = function (metrics, calculatedTimerMetrics, flushInterval, ts, flushCallback) { const starttime = Date.now(); let key; let counter_rates = {}; @@ -132,7 +132,7 @@ const process_metrics = function (metrics, flushInterval, ts, flushCallback) { } - timer_data[key] = current_timer_data; + timer_data[key] = filter_timer_metrics(current_timer_data, calculatedTimerMetrics); } statsd_metrics["processing_time"] = (Date.now() - starttime); @@ -144,4 +144,20 @@ const process_metrics = function (metrics, flushInterval, ts, flushCallback) { flushCallback(metrics); }; +var filter_timer_metrics = function (currentTimerMetrics, calculatedTimerMetrics = []) { + if (!Array.isArray(calculatedTimerMetrics) || calculatedTimerMetrics.length == 0) { + return currentTimerMetrics; + } else { + return Object.keys(currentTimerMetrics) + .filter((key) => { + // Generalizes filtering percent metrics by cleaning key from _ to _percent + let cleaned_key = key.replace(/_(top)?\d+$/, "_percent") + return calculatedTimerMetrics.includes(cleaned_key); + }) + .reduce((obj, key) => { + obj[key] = currentTimerMetrics[key]; + return obj; + }, {}); + } +} exports.process_metrics = process_metrics; diff --git a/stats.js b/stats.js index b3cb31af..9ecf2111 100644 --- a/stats.js +++ b/stats.js @@ -147,7 +147,7 @@ function flushMetrics() { } }); - pm.process_metrics(metrics_hash, flushInterval, time_stamp, function emitFlush(metrics) { + pm.process_metrics(metrics_hash, conf.calculatedTimerMetrics, flushInterval, time_stamp, function emitFlush(metrics) { backendEvents.emit('flush', time_stamp, metrics); }); diff --git a/test/graphite_tests_filters.js b/test/graphite_tests_filters.js new file mode 100644 index 00000000..8de66d7c --- /dev/null +++ b/test/graphite_tests_filters.js @@ -0,0 +1,180 @@ +var fs = require('fs'), + net = require('net'), + temp = require('temp'), + spawn = require('child_process').spawn, + util = require('util'), + urlparse = require('url').parse, + _ = require('underscore'), + dgram = require('dgram'), + qsparse = require('querystring').parse, + http = require('http'); + + +var writeconfig = function(text, worker, cb, obj){ + temp.open({suffix: '-statsdconf.js'}, function(err, info) { + if (err) throw err; + fs.writeSync(info.fd, text); + fs.close(info.fd, function(err) { + if (err) throw err; + worker(info.path, cb, obj); + }); + }); +} + +var statsd_send = function(data,sock,host,port,cb){ + send_data = new Buffer(data); + sock.send(send_data,0,send_data.length,port,host,function(err,bytes){ + if (err) { + throw err; + } + cb(); + }); +} + +// keep collecting data until a specified timeout period has elapsed +// this will let us capture all data chunks so we don't miss one +var collect_for = function(server,timeout,cb){ + var received = []; + var in_flight = 0; + var timed_out = false; + var collector = function(req,res){ + in_flight += 1; + var body = ''; + req.on('data',function(data){ body += data; }); + req.on('end',function(){ + received = received.concat(body.split("\n")); + in_flight -= 1; + if((in_flight < 1) && timed_out){ + server.removeListener('request',collector); + cb(received); + } + }); + } + + setTimeout(function (){ + timed_out = true; + if((in_flight < 1)) { + server.removeListener('connection',collector); + cb(received); + } + },timeout); + + server.on('connection',collector); +} +module.exports = { + setUp: function (callback) { + this.testport = 31337; + this.myflush = 200; + var configfile = "{graphService: \"graphite\"\n\ + , batch: 200 \n\ + , flushInterval: " + this.myflush + " \n\ + , percentThreshold: 90\n\ + , calculatedTimerMetrics: ['count_ps', 'count', 'count_percent', 'mean_percent', 'histogram']\n\ + , histogram: [ { metric: \"a_test_value\", bins: [1000] } ]\n\ + , port: 8125\n\ + , dumpMessages: false \n\ + , debug: false\n\ + , graphite: { legacyNamespace: false }\n\ + , graphitePort: " + this.testport + "\n\ + , graphiteHost: \"127.0.0.1\"}"; + + this.acceptor = net.createServer(); + this.acceptor.listen(this.testport); + this.sock = dgram.createSocket('udp4'); + + this.server_up = true; + this.ok_to_die = false; + this.exit_callback_callback = process.exit; + + writeconfig(configfile,function(path, cb, obj){ + obj.path = path; + obj.server = spawn('node',['stats.js', path]); + obj.exit_callback = function (code) { + obj.server_up = false; + if(!obj.ok_to_die){ + console.log('node server unexpectedly quit with code: ' + code); + process.exit(1); + } + else { + obj.exit_callback_callback(); + } + }; + obj.server.on('exit', obj.exit_callback); + obj.server.stderr.on('data', function (data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + /* + obj.server.stdout.on('data', function (data) { + console.log('stdout: ' + data.toString().replace(/\n$/,'')); + }); + */ + obj.server.stdout.on('data', function (data) { + // wait until server is up before we finish setUp + if (data.toString().match(/server is up/)) { + cb(); + } + }); + + },callback,this); + }, + tearDown: function (callback) { + this.sock.close(); + this.acceptor.close(); + this.ok_to_die = true; + if(this.server_up){ + this.exit_callback_callback = callback; + this.server.kill(); + } else { + callback(); + } + }, + + timers_are_valid: function (test) { + test.expect(11); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection', function(c){ + statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + var numstat_test = function(post){ + var mykey = 'stats.statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(hashes,numstat_test), 'stats.statsd.numStats should be 5'); + + var testtimervalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.mean_90'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + var testtimerhistogramvalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.histogram.bin_1000'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,testtimerhistogramvalue_test), 'stats.timers.a_test_value.histogram.bin_1000 should be 1'); + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean_90 should be ' + testvalue); + + var count_test = function(post, metric){ + var mykey = 'stats.timers.a_test_value.' + metric; + return _.first(_.filter(_.pluck(post, mykey), function (e) { return e; })); + }; + test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5'); + test.equals(count_test(hashes, 'count'), 1, 'count should be 1'); + test.equals(count_test(hashes, 'count_90'), 1, 'count_90 should be 1'); + test.equals(count_test(hashes, 'sum'), null, 'sum should be null'); + test.equals(count_test(hashes, 'sum_squares'), null, 'sum_squares should be null'); + test.equals(count_test(hashes, 'sum_90'), null, 'sum_90 should be null'); + test.equals(count_test(hashes, 'sum_squares_90'), null, 'sum_squares_90 should be null'); + test.done(); + }); + }); + }); + }, +} diff --git a/test/process_metrics_tests.js b/test/process_metrics_tests.js index 6a5b1823..390f863c 100644 --- a/test/process_metrics_tests.js +++ b/test/process_metrics_tests.js @@ -11,6 +11,7 @@ module.exports = { var timer_counters = {}; var sets = {}; var pctThreshold = null; + var calculatedTimerMetrics = []; this.metrics = { counters: counters, @@ -25,14 +26,14 @@ module.exports = { counters_has_stats_count: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 1000, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 1000, this.time_stamp, function(){}); test.equal(2, this.metrics.counters['a']); test.done(); }, counters_has_correct_rate: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); test.equal(20, this.metrics.counter_rates['a']); test.done(); }, @@ -40,7 +41,7 @@ module.exports = { test.expect(1); this.metrics.timers['a'] = []; this.metrics.timer_counters['a'] = 0; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); //potentially a cleaner way to check this test.equal(undefined, this.metrics.counter_rates['a']); test.done(); @@ -49,7 +50,7 @@ module.exports = { test.expect(9); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(0, timer_data.std); test.equal(100, timer_data.upper); @@ -62,11 +63,49 @@ module.exports = { test.equal(100, timer_data.median); test.done(); }, - timers_multiple_times: function(test) { + timer_single_time_with_one_filter: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + let filter = ['upper', 'lower', 'count', 'count_ps', 'sum', 'sum_squares', 'mean', 'median'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(8, Object.keys(timer_data).length) + test.equal(null, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(1, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(100, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(100, timer_data.mean); + test.equal(100, timer_data.median); + test.done(); + }, + timer_single_time_multiple_filter: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + let filter = ['upper', 'lower', 'count_ps', 'sum_squares'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(4, Object.keys(timer_data).length) + test.equal(null, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(null, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(null, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(null, timer_data.mean); + test.equal(null, timer_data.median); + test.done(); + }, + timers_multiple_times: function(test) { test.expect(9); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(81.64965809277261, timer_data.std); test.equal(300, timer_data.upper); @@ -75,17 +114,36 @@ module.exports = { test.equal(30, timer_data.count_ps); test.equal(600, timer_data.sum); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares); + timer_data.sum_squares); + test.equal(200, timer_data.mean); + test.equal(200, timer_data.median); + test.done(); + }, + timers_multiple_times_with_calculated_timer_metrics: function(test) { + test.expect(9); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + let calculatedTimerMetrics = ['std', 'count', 'sum_squares', 'mean', 'median'] + pm.process_metrics(this.metrics, calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(81.64965809277261, timer_data.std); + test.equal(null, timer_data.upper); + test.equal(null, timer_data.lower); + test.equal(3, timer_data.count); + test.equal(null, timer_data.count_ps); + test.equal(null, timer_data.sum); + test.equal(100 * 100 + 200 * 200 + 300 * 300, + timer_data.sum_squares); test.equal(200, timer_data.mean); test.equal(200, timer_data.median); test.done(); }, - timers_single_time_single_percentile: function(test) { + timers_single_time_single_percentile: function(test) { test.expect(4); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); @@ -93,45 +151,94 @@ module.exports = { test.equal(100 * 100, timer_data.sum_squares_90); test.done(); }, - timers_single_time_multiple_percentiles: function(test) { - test.expect(9); + timers_single_time_single_percentile_with_calculated_timer_metrics: function(test) { + test.expect(4); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + this.metrics.pctThreshold = [90]; + pm.process_metrics(this.metrics, ['upper_percent', 'sum_squares_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.mean_90); + test.equal(100, timer_data.upper_90); + test.equal(null, timer_data.sum_90); + test.equal(100 * 100, timer_data.sum_squares_90); + test.done(); + }, + timers_single_time_multiple_percentiles: function(test) { + test.expect(10); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(1, timer_data.count_90); test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); test.equal(100, timer_data.sum_90); test.equal(100 * 100, timer_data.sum_squares_90); + test.equal(1, timer_data.count_80); test.equal(100, timer_data.mean_80); test.equal(100, timer_data.upper_80); test.equal(100, timer_data.sum_80); test.equal(100 * 100, timer_data.sum_squares_80); test.done(); }, - timers_multiple_times_single_percentiles: function(test) { + timers_single_time_multiple_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(10); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + this.metrics.pctThreshold = [90, 80]; + let calculatedTimerMetrics = ['mean_percent', 'sum_percent'] + pm.process_metrics(this.metrics, calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.count_90); + test.equal(100, timer_data.mean_90); + test.equal(null, timer_data.upper_90); + test.equal(100, timer_data.sum_90); + test.equal(null, timer_data.sum_squares_90); + test.equal(null, timer_data.count_80); + test.equal(100, timer_data.mean_80); + test.equal(null, timer_data.upper_80); + test.equal(100, timer_data.sum_80); + test.equal(null, timer_data.sum_squares_80); + test.done(); + }, + timers_multiple_times_single_percentiles: function(test) { test.expect(5); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(3, timer_data.count_90); test.equal(200, timer_data.mean_90); test.equal(300, timer_data.upper_90); test.equal(600, timer_data.sum_90); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares_90); + timer_data.sum_squares_90); + test.done(); + }, + timers_multiple_times_single_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(5); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + this.metrics.pctThreshold = [90]; + let filter = ['count_percent', 'mean_percent', 'upper_percent'] + pm.process_metrics(this.metrics, filter, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(3, timer_data.count_90); + test.equal(200, timer_data.mean_90); + test.equal(300, timer_data.upper_90); + test.equal(null, timer_data.sum_90); + test.equal(null, timer_data.sum_squares_90); test.done(); }, - timers_multiple_times_multiple_percentiles: function(test) { + timers_multiple_times_multiple_percentiles: function(test) { test.expect(11); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(3, timer_data.count); test.equal(3, timer_data.count_90); @@ -139,22 +246,45 @@ module.exports = { test.equal(300, timer_data.upper_90); test.equal(600, timer_data.sum_90); test.equal(100 * 100 + 200 * 200 + 300 * 300, - timer_data.sum_squares_90); + timer_data.sum_squares_90); test.equal(2, timer_data.count_80); test.equal(150, timer_data.mean_80); test.equal(200, timer_data.upper_80); test.equal(300, timer_data.sum_80); test.equal(100 * 100 + 200 * 200, - timer_data.sum_squares_80); + timer_data.sum_squares_80); + test.done(); + }, + timers_multiple_times_multiple_percentiles_with_calculated_timer_metrics: function(test) { + test.expect(11); + this.metrics.timers['a'] = [100, 200, 300]; + this.metrics.timer_counters['a'] = 3; + this.metrics.pctThreshold = [90, 80]; + pm.process_metrics(this.metrics, ['count_percent', 'sum_percent', 'sum_squares_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.count); + test.equal(3, timer_data.count_90); + test.equal(null, timer_data.mean_90); + test.equal(null, timer_data.upper_90); + test.equal(600, timer_data.sum_90); + test.equal(100 * 100 + 200 * 200 + 300 * 300, + timer_data.sum_squares_90); + + test.equal(2, timer_data.count_80); + test.equal(null, timer_data.mean_80); + test.equal(null, timer_data.upper_80); + test.equal(300, timer_data.sum_80); + test.equal(100 * 100 + 200 * 200, + timer_data.sum_squares_80); test.done(); }, - timers_sampled_times: function(test) { + timers_sampled_times: function(test) { test.expect(8); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 50; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(50, timer_data.count); test.equal(500, timer_data.count_ps); @@ -166,7 +296,45 @@ module.exports = { test.equal(300, timer_data.sum_80); test.done(); }, // check if the correct settings are being applied. as well as actual counts - timers_histogram: function (test) { + timers_histogram: function (test) { + test.expect(13); + this.metrics.timers['a'] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + this.metrics.timers['abc'] = [0.1234, 2.89, 4, 6, 8]; + this.metrics.timers['foo'] = [0, 2, 4, 6, 8]; + this.metrics.timers['barbazfoobar'] = [0, 2, 4, 6, 8]; + this.metrics.timers['bar.bazfoobar.abc'] = [0, 2, 4, 6, 8]; + this.metrics.timers['xyz'] = [0, 2, 4, 6, 8]; + this.metrics.histogram = [ { metric: 'foo', bins: [] }, + { metric: 'abcd', bins: [ 1, 5, 'inf'] }, + { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, + { metric: 'a', bins: [ 1, 2] } ]; + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data; + // nothing matches the 'abcd' calculatedTimerMetrics, so nothing has bin_5 + test.equal(undefined, timer_data['a']['histogram']['bin_5']); + test.equal(undefined, timer_data['abc']['histogram']['bin_5']); + + // check that 'a' got the right calculatedTimerMetrics and numbers + test.equal(0, timer_data['a']['histogram']['bin_1']); + test.equal(1, timer_data['a']['histogram']['bin_2']); + test.equal(undefined, timer_data['a']['histogram']['bin_inf']); + + // only 'abc' should have a bin_inf; also check all its counts, + // and make sure it has no other bins + test.equal(1, timer_data['abc']['histogram']['bin_1']); + test.equal(0, timer_data['abc']['histogram']['bin_2_21']); + test.equal(4, timer_data['abc']['histogram']['bin_inf']); + test.equal(3, _.size(timer_data['abc']['histogram'])); + + // these all have histograms disabled ('foo' explicitly, rest implicitly) + test.equal(undefined, timer_data['foo']['histogram']); + test.equal(undefined, timer_data['barbazfoobar']['histogram']); + test.equal(undefined, timer_data['bar.bazfoobar.abc']['histogram']); + test.equal(undefined, timer_data['xyz']['histogram']); + + test.done(); + }, + timers_histogram_with_calculated_timer_metrics: function (test) { test.expect(13); this.metrics.timers['a'] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; this.metrics.timers['abc'] = [0.1234, 2.89, 4, 6, 8]; @@ -175,16 +343,16 @@ module.exports = { this.metrics.timers['bar.bazfoobar.abc'] = [0, 2, 4, 6, 8]; this.metrics.timers['xyz'] = [0, 2, 4, 6, 8]; this.metrics.histogram = [ { metric: 'foo', bins: [] }, - { metric: 'abcd', bins: [ 1, 5, 'inf'] }, - { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, - { metric: 'a', bins: [ 1, 2] } ]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + { metric: 'abcd', bins: [ 1, 5, 'inf'] }, + { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, + { metric: 'a', bins: [ 1, 2] } ]; + pm.process_metrics(this.metrics, ['histogram'], 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data; - // nothing matches the 'abcd' config, so nothing has bin_5 + // nothing matches the 'abcd' calculatedTimerMetrics, so nothing has bin_5 test.equal(undefined, timer_data['a']['histogram']['bin_5']); test.equal(undefined, timer_data['abc']['histogram']['bin_5']); - // check that 'a' got the right config and numbers + // check that 'a' got the right calculatedTimerMetrics and numbers test.equal(0, timer_data['a']['histogram']['bin_1']); test.equal(1, timer_data['a']['histogram']['bin_2']); test.equal(undefined, timer_data['a']['histogram']['bin_inf']); @@ -204,41 +372,88 @@ module.exports = { test.done(); }, - timers_single_time_single_top_percentile: function(test) { + timers_single_time_single_top_percentile: function(test) { test.expect(3); this.metrics.timers['a'] = [100]; this.metrics.pctThreshold = [-10]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_top10); test.equal(100, timer_data.lower_top10); test.equal(100, timer_data.sum_top10); test.done(); }, - timers_multiple_times_single_top_percentile: function(test) { + timers_single_time_single_top_percentile_with_calculated_timer_metrics: function(test) { + test.expect(3); + this.metrics.timers['a'] = [100]; + this.metrics.pctThreshold = [-10]; + pm.process_metrics(this.metrics, ['lower_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(null, timer_data.mean_top10); + test.equal(100, timer_data.lower_top10); + test.equal(null, timer_data.sum_top10); + test.done(); + }, + timers_multiple_times_single_top_percentile: function(test) { test.expect(3); this.metrics.timers['a'] = [10, 10, 10, 10, 10, 10, 10, 10, 100, 200]; this.metrics.pctThreshold = [-20]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(150, timer_data.mean_top20); test.equal(100, timer_data.lower_top20); test.equal(300, timer_data.sum_top20); test.done(); }, - statsd_metrics_exist: function(test) { + timers_multiple_times_single_top_percentile_with_calculated_timer_metrics: function(test) { + test.expect(3); + this.metrics.timers['a'] = [10, 10, 10, 10, 10, 10, 10, 10, 100, 200]; + this.metrics.pctThreshold = [-20]; + pm.process_metrics(this.metrics, ['mean_percent', 'sum_percent'], 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(150, timer_data.mean_top20); + test.equal(null, timer_data.lower_top20); + test.equal(300, timer_data.sum_top20); + test.done(); + }, + statsd_metrics_exist: function(test) { test.expect(1); - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); statsd_metrics = this.metrics.statsd_metrics; test.notEqual(undefined, statsd_metrics["processing_time"]); test.done(); }, - timers_multiple_times_even: function(test) { + timers_multiple_times_even: function(test) { + test.expect(1); + this.metrics.timers['a'] = [300, 200, 400, 100]; + pm.process_metrics(this.metrics, this.calculatedTimerMetrics, 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(250, timer_data.median); + test.done(); + }, + timers_multiple_times_even_with_calculated_timer_metrics: function(test) { test.expect(1); this.metrics.timers['a'] = [300, 200, 400, 100]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, ['median'], 100, this.time_stamp, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(250, timer_data.median); test.done(); + }, + timers_with_invalid_filter: function(test) { + test.expect(9); + this.metrics.timers['a'] = [100]; + this.metrics.timer_counters['a'] = 1; + pm.process_metrics(this.metrics, 'not a valid filter', 100, this.time_stamp, function(){}); + timer_data = this.metrics.timer_data['a']; + test.equal(0, timer_data.std); + test.equal(100, timer_data.upper); + test.equal(100, timer_data.lower); + test.equal(1, timer_data.count); + test.equal(10, timer_data.count_ps); + test.equal(100, timer_data.sum); + test.equal(100 * 100, timer_data.sum_squares); + test.equal(100, timer_data.mean); + test.equal(100, timer_data.median); + test.done(); } } From 814f253336e1d12ce070b210417a0373ab63de44 Mon Sep 17 00:00:00 2001 From: Claudio Benfatto Date: Wed, 19 Feb 2020 01:15:31 +0100 Subject: [PATCH 167/207] Add an optional max TTL setting for gauges (#599) Add an optional max TTL setting for gauges. This covers for the edge case where the gauge value is sent less frequently than the flush, causing the gauge to be deleted. An overview of the changes: * Move away idleStats related configuration variables from the flushMetrics method * Write new tests for the gaugesMaxTTL option * Write the logic for handling the deletion in the presence of gaugesMaxTTL The aim of this PR is to fix the behaviour involved in gauges deletion. The typical use case scenario is when gauges deletion is set to true via the deleteGauges flag but new values are actually sent with a frequency lower than the flushInterval. In this case, the gauge's default behaviour and assumption of sending the previous value if no new one has been received since is broken by the gauge deletion process. I've chosen the path of least impact to the existing code, but I'm aware that at least 2 optimisations are possible: * We actually don't need to populate an additional data structure (gaugesTTL) when the gaugesMaxTTL param is set to 1, as this would exactly coincide with the previous behaviour * We could save additional memory if we kept the gaugesMaxTTL within the metrics.gauges associative array, but this would require code changes in other parts of the program Related to issue: #584 --- debian/changelog | 8 +++ exampleConfig.js | 1 + lib/helpers.js | 5 ++ stats.js | 45 +++++++++++----- test/graphite_delete_counters_tests.js | 71 +++++++++++++++++++++++++- 5 files changed, 117 insertions(+), 13 deletions(-) diff --git a/debian/changelog b/debian/changelog index 55cc954a..05c2fdaa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +statsd (0.8.5-2) unstable; urgency=low + + * Move away idleStats related configuration variables from the flushMetrics method + * Write new tests for the gaugesMaxTTL option + * Write the logic for handling the deletion in the presence of gaugesMaxTTL + + -- Claudio Benfatto Thu, 17 Feb 2020 00:00:00 +0000 + statsd (0.8.5-1) unstable; urgency=low * Update lodash (sub dependency) for security fix diff --git a/exampleConfig.js b/exampleConfig.js index d61298d1..271b8e28 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -62,6 +62,7 @@ Optional Variables: as opposed to sending 0. For gauges, this unsets the gauge (instead of sending the previous value). Can be individually overridden. [default: false] deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] + gaugesMaxTTL: number of flush cycles to wait before the gauge is marked as inactive, to use in combination with deleteGauges [default: 1] deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] diff --git a/lib/helpers.js b/lib/helpers.js index d570816b..cf64b575 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -11,6 +11,10 @@ function isNumber(str) { return Boolean(str && !isNaN(str)); } +function isInteger(x) { + return (typeof x === 'number') && (x % 1 === 0); +} + function isValidSampleRate(str) { let validSampleRate = false; if(str.length > 1 && str[0] === '@') { @@ -52,6 +56,7 @@ function is_valid_packet(fields) { } exports.is_valid_packet = is_valid_packet; +exports.isInteger= isInteger; exports.writeConfig = function(config, stream) { stream.write("\n"); diff --git a/stats.js b/stats.js index 9ecf2111..8268994f 100644 --- a/stats.js +++ b/stats.js @@ -18,6 +18,7 @@ let counters = {}; let timers = {}; let timer_counters = {}; let gauges = {}; +let gaugesTTL = {}; let sets = {}; let counter_rates = {}; let timer_data = {}; @@ -89,16 +90,6 @@ function flushMetrics() { // After all listeners, reset the stats backendEvents.once('flush', function clear_metrics(ts, metrics) { - // TODO: a lot of this should be moved up into an init/constructor so we don't have to do it every - // single flushInterval.... - // allows us to flag all of these on with a single config but still override them individually - conf.deleteIdleStats = conf.deleteIdleStats !== undefined ? conf.deleteIdleStats : false; - if (conf.deleteIdleStats) { - conf.deleteCounters = conf.deleteCounters !== undefined ? conf.deleteCounters : true; - conf.deleteTimers = conf.deleteTimers !== undefined ? conf.deleteTimers : true; - conf.deleteSets = conf.deleteSets !== undefined ? conf.deleteSets : true; - conf.deleteGauges = conf.deleteGauges !== undefined ? conf.deleteGauges : true; - } // Clear the counters conf.deleteCounters = conf.deleteCounters || false; @@ -141,8 +132,14 @@ function flushMetrics() { // Normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { - for (const gauge_key in metrics.gauges) { - delete(metrics.gauges[gauge_key]); + for (const gauge_key in gaugesTTL) { + gaugesTTL[gauge_key]--; + + // if the gauge has been idle for more than the allowed TTL cycles delete it + if (gaugesTTL[gauge_key] < 1) { + delete(metrics.gauges[gauge_key]); + delete(gaugesTTL[gauge_key]); + } } } }); @@ -188,6 +185,26 @@ config.configFile(process.argv[2], function (config) { l = new logger.Logger(config.log || {}); + // force conf.gaugesMaxTTL to 1 if it not a positive integer > 1 + if (helpers.isInteger(conf.gaugesMaxTTL) && conf.gaugesMaxTTL > 1) { + conf.gaugesMaxTTL = conf.gaugesMaxTTL; + } else { + conf.gaugesMaxTTL = 1; + } + + // allows us to flag all of these on with a single config but still override them individually + conf.deleteIdleStats = conf.deleteIdleStats !== undefined ? conf.deleteIdleStats : false; + if (conf.deleteIdleStats) { + conf.deleteCounters = conf.deleteCounters !== undefined ? conf.deleteCounters : true; + conf.deleteTimers = conf.deleteTimers !== undefined ? conf.deleteTimers : true; + conf.deleteSets = conf.deleteSets !== undefined ? conf.deleteSets : true; + conf.deleteGauges = conf.deleteGauges !== undefined ? conf.deleteGauges : true; + } + + // if gauges are not being deleted, clear gaugesTTL counters to save memory + if (! conf.deleteGauges) { + gaugesTTL = {} + } // setup config for stats prefix let prefixStats = config.prefixStats; prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; @@ -266,6 +283,10 @@ config.configFile(process.argv[2], function (config) { timers[key].push(Number(fields[0] || 0)); timer_counters[key] += (1 / sampleRate); } else if (metric_type === "g") { + // if deleteGauges is true reset the max TTL to its initial value + if (conf.deleteGauges) { + gaugesTTL[key] = conf.gaugesMaxTTL; + } if (gauges[key] && fields[0].match(/^[-+]/)) { gauges[key] += Number(fields[0] || 0); } else { diff --git a/test/graphite_delete_counters_tests.js b/test/graphite_delete_counters_tests.js index 05093f41..87a34fbe 100644 --- a/test/graphite_delete_counters_tests.js +++ b/test/graphite_delete_counters_tests.js @@ -87,6 +87,7 @@ module.exports = { , dumpMessages: false \n\ , debug: false\n\ , deleteIdleStats: true\n\ + , gaugesMaxTTL: 2\n\ , graphitePort: " + this.testport + "\n\ , graphiteHost: \"127.0.0.1\"}"; @@ -266,5 +267,73 @@ module.exports = { }); }); }); - } + }, + + gauges_are_valid: function (test) { + test.expect(7); + + var testvalue = "+1"; + var me = this; + this.acceptor.once('connection',function(g){ + statsd_send('a_test_value:' + testvalue + '|g',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*3,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + + // create an associative array which has the flush cycle number as a key + // and a list of hashes of the metrics sent during that flush cycle as value + flushes = {}; + i_number = 0; + hashes.forEach(function (item) { + key = Object.keys(item)[0] + if (key == '') { + i_number++; + return; + } + if (! (i_number in flushes)) { + flushes[i_number] = []; + } + + flushes[i_number].push(item); + }); + + var testavgvalue_test = function(post){ + var mykey = 'stats.gauges.a_test_value'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + + test.ok(_.any(flushes[0],testavgvalue_test), 'stats.gauges.a_test_value after first flush should be ' + 1); + test.ok(_.any(flushes[1],testavgvalue_test), 'stats.gauges.a_test_value after second flush should be ' + 1); + + var testavgvalue_test_after_delete = function(post){ + var mykey = 'stats.gauges.a_test_value'; + return !(_.include(_.keys(post),mykey)); + }; + + test.ok(_.every(flushes[2],testavgvalue_test_after_delete), 'stats.gauges.a_test_value after third flush should not be present'); + + var numstat_test = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 5); + }; + test.ok(_.any(flushes[0],numstat_test), 'statsd.numStats after first flush should be 5'); + test.ok(_.any(flushes[1],numstat_test), 'statsd.numStats after second flush should be 5'); + + var numstat_test_after_delete = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 4); + }; + test.ok(_.any(flushes[2],numstat_test_after_delete), 'statsd.numStats after third flush should be 4'); + + test.done() + }); + + }); + }); +} } From 5a0c8b09a970d33c169b06b0e940698c2bd60642 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 19 Feb 2020 00:44:36 +0000 Subject: [PATCH 168/207] release 0.8.6 --- Changelog.md | 5 +++++ debian/changelog | 9 ++++----- debian/control | 4 ++-- package-lock.json | 2 +- package.json | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Changelog.md b/Changelog.md index 6a0e1431..3e3d58ad 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,10 @@ # Changelog +## v0.8.6 (02/19/2020) + +- Add an optional max TTL setting for gauges +- add filter option for metrics + ## v0.8.5 (07/23/2019) - Update lodash (sub dependency) for security fix diff --git a/debian/changelog b/debian/changelog index 05c2fdaa..cf54a135 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,9 @@ -statsd (0.8.5-2) unstable; urgency=low +statsd (0.8.6-1) unstable; urgency=low - * Move away idleStats related configuration variables from the flushMetrics method - * Write new tests for the gaugesMaxTTL option - * Write the logic for handling the deletion in the presence of gaugesMaxTTL + * Add an optional max TTL setting for gauges + * add filter option for metrics - -- Claudio Benfatto Thu, 17 Feb 2020 00:00:00 +0000 + -- Elliot Blackburn Tue, 19 Feb 2020 00:43:00 +0000 statsd (0.8.5-1) unstable; urgency=low diff --git a/debian/control b/debian/control index 03e84f1b..c1434984 100644 --- a/debian/control +++ b/debian/control @@ -1,13 +1,13 @@ Source: statsd Section: devel Priority: optional -Maintainer: Stephen Koenig +Maintainer: Elliot Blackburn Standards-Version: 3.9.3 Build-Depends: debhelper (>= 8.0.0), dh-systemd (>= 1.5) Package: statsd Architecture: all -Depends: nodejs (>= 0.10), adduser, npm +Depends: nodejs (>= 8.0), adduser, npm Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. diff --git a/package-lock.json b/package-lock.json index 186e2904..241ea4fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.4", + "version": "0.8.6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ac7d8e3c..0fcbc658 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.5", + "version": "0.8.6", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", @@ -21,7 +21,7 @@ "url": "https://github.com/statsd/statsd.git" }, "engines": { - "node": ">=6" + "node": ">=8" }, "dependencies": { "generic-pool": "2.2.0" From 9c4a66a7bd33b85fa74f576ac2c14a6bbf5c6a14 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 25 Feb 2020 18:52:14 +0000 Subject: [PATCH 169/207] account for double flush timeouts in docker environments Issue #685 highlighted a problem which still to this day only seems to appear in docker environments. However, given the numbers extracted from a container they can be recreated when pushed through the getFlushTimeout function. The issue resulted in a timeout value being calculated which was around 30ms rather than 10s (in the default settings scenario). @aonischenko contributed a nifty workaround which fixes the timeout in these scenarios without too much extra code to get your head around. The result was run in a docker container for 8 hours, before this patch the issue was very prevelant and easy to spot and now it is solved with no clock slippage. --- stats.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stats.js b/stats.js index 8268994f..ceb2ae1b 100644 --- a/stats.js +++ b/stats.js @@ -172,7 +172,12 @@ function sanitizeKeyName(key) { } function getFlushTimeout(interval) { - return interval - (new Date().getTime() - startup_time * 1000) % flushInterval + const now = new Date().getTime() + const deltaTime = now - startup_time * 1000; + const timeoutAttempt = Math.round(deltaTime / interval) + 1; + const fixedTimeout = (startup_time * 1000 + timeoutAttempt * interval) - now; + + return fixedTimeout; } // Global for the logger From 0c241dbbb3d19e13f61a4626df190fc9c0e1c27a Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 25 Feb 2020 19:00:54 +0000 Subject: [PATCH 170/207] update debian changelog --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index cf54a135..b89d7ae4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ statsd (0.8.6-1) unstable; urgency=low * Add an optional max TTL setting for gauges * add filter option for metrics + * fix double flush scenario in docker containers -- Elliot Blackburn Tue, 19 Feb 2020 00:43:00 +0000 From 258c531cb6eee6dbb1ad68d0f382ea37677de514 Mon Sep 17 00:00:00 2001 From: haguenau Date: Mon, 9 Mar 2020 12:40:48 -0400 Subject: [PATCH 171/207] Documentation spelling and gramatical fixes (#693) * Fix gramatical errors * Standardise on "StatsD" in docs over "statsd" and "Statsd" --- Changelog.md | 8 ++++---- MAINTAINERS.md | 10 +++++----- README.md | 19 +++++++++++-------- docs/additional_tools.md | 2 +- docs/admin_interface.md | 14 +++++++------- docs/backend_interface.md | 2 +- docs/client_implementations.md | 2 +- docs/cluster_proxy.md | 12 ++++++------ docs/graphite.md | 26 +++++++++++++------------- docs/graphite_pickle.md | 4 ++-- docs/metric_types.md | 12 ++++++------ docs/server_implementations.md | 4 ++-- 12 files changed, 59 insertions(+), 56 deletions(-) diff --git a/Changelog.md b/Changelog.md index 3e3d58ad..91cd7b4d 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,7 +8,7 @@ ## v0.8.5 (07/23/2019) - Update lodash (sub dependency) for security fix -- Add the statsd history to the docs +- Add the StatsD history to the docs - Add third party server interfaces to docs - Migrate docs from github wiki, and standardise markdown notation - Minor formatting proposals @@ -39,7 +39,7 @@ ## v0.8.1 (03/13/2019) -- drop statsd instance from proxy ring in instance of healthcheck failures (#665) +- drop StatsD instance from proxy ring in instance of healthcheck failures (#665) - Add myself (elliot blackburn) to maintainers.md (#666) - add mysql backend link to docs/backend.md - Adding myself to MAINTAINERS. @@ -60,7 +60,7 @@ - removes -q switch - Fix for failing test on node 0.10 - fix usage of process.EventEmitter -- Add plugin Warp10 to statsd +- Add plugin Warp10 to StatsD - Updated graphite link to read the docs ## v0.8.0 (05/05/2016) @@ -115,7 +115,7 @@ - added standard Deviation to timers stats (.std) - added last_flush_time and last_flush_length metrics to graphite backend - added ipv6 support -- added Statsd repeater backend +- added StatsD repeater backend - added helper script to decide which timers to sample down - added Windows service support - added Scala example diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 3489461c..78c91746 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,10 +1,10 @@ -statsd is maintained by the following people. +StatsD is maintained by the following people. All maintainers must agree to the [Developer Certificate of Origin][dco]. ## Steps to becoming a maintainer 1. Open a PR where you add yourself to this file and agree to the DCO. Attach a little bit about your experience -with statsd and how much time you think you can roughly spend on the project +with StatsD and how much time you think you can roughly spend on the project 2. Current maintainers will review 3. If it gets merged, you're in! @@ -16,9 +16,9 @@ with statsd and how much time you think you can roughly spend on the project ## Current maintainers -- **Daniel Schauenberg**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. -- **Mike Heffner**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. -- **Elliot Blackburn**: As a maintainer of Statsd, I agree to the [Developer Certificate of Origin][dco]. +- **Daniel Schauenberg**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. +- **Mike Heffner**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. +- **Elliot Blackburn**: As a maintainer of StatsD, I agree to the [Developer Certificate of Origin][dco]. [dco]: https://github.com/statsd/statsd/blob/5f58a9cc7442900c2e553ed1df3d6ce99e885226/DCO.txt diff --git a/README.md b/README.md index 0ac11fd9..fd7847c0 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,18 @@ listens for statistics, like counters and timers, sent over [UDP][udp] or ## Key Concepts * *buckets* + Each stat is in its own "bucket". They are not predefined anywhere. Buckets can be named anything that will translate to Graphite (periods make folders, etc) * *values* + Each stat will have a value. How it is interpreted depends on modifiers. In general values should be integers. * *flush* + After the flush interval timeout (defined by `config.flushInterval`, default 10 seconds), stats are aggregated and sent to an upstream backend service. @@ -24,7 +27,7 @@ general values should be integers. ## Installation and Configuration ### Docker -Statsd supports docker in two ways: +StatsD supports docker in two ways: * The official docker image on [docker hub](https://hub.docker.com/r/statsd/statsd) * Building the image from the bundled [Dockerfile](./Dockerfile) @@ -54,7 +57,7 @@ StatsD running with the default UDP server on localhost would be: * [Server Interface][docs_server_interface] * [Backend Interface][docs_backend_interface] * [Metric Namespacing][docs_namespacing] -* [Statsd Cluster Proxy][docs_cluster_proxy] +* [StatsD Cluster Proxy][docs_cluster_proxy] ## Debugging There are additional config variables available for debugging: @@ -67,7 +70,7 @@ For more information, check the `exampleConfig.js`. ## Tests A test framework has been added using node-unit and some custom code to start -and manipulate statsd. Please add tests under test/ for any new features or bug +and manipulate StatsD. Please add tests under test/ for any new features or bug fixes encountered. Testing a live server can be tricky, attempts were made to eliminate race conditions but it may be possible to encounter a stuck state. If doing dev work, a `killall statsd` will kill any stray test servers in the @@ -76,13 +79,13 @@ background (don't do this on a production machine!). Tests can be executed with `./run_tests.sh`. ## History -statsd was originally written at ([Etsy][etsy]) and released with a [blog post][blog post] -about how it works and why we created it. +StatsD was originally written at [Etsy][etsy] and released with a +[blog post][blog post] about how it works and why we created it. ## Inspiration -StatsD was inspired (heavily) by the project (of the same name) at Flickr. +StatsD was inspired (heavily) by the project of the same name at Flickr. Here's a post where Cal Henderson described it in depth: -[Counting and timing][counting-timing] +[Counting and timing][counting-timing]. Cal re-released the code recently: [Perl StatsD][Flicker-StatsD] @@ -90,7 +93,7 @@ Cal re-released the code recently: [graphite]: http://graphite.readthedocs.org/ [etsy]: http://www.etsy.com -[blog post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ +[blog post]: https://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ [node]: http://nodejs.org [nodemods]: http://nodejs.org/api/modules.html [counting-timing]: http://code.flickr.com/blog/2008/10/27/counting-timing/ diff --git a/docs/additional_tools.md b/docs/additional_tools.md index 99aa04a3..251c36aa 100644 --- a/docs/additional_tools.md +++ b/docs/additional_tools.md @@ -1,5 +1,5 @@ # Additional Tools -The following are tools you might find useful when using, contributing, or testing statsd. +The following are tools you might find useful when using, contributing, or testing StatsD. * [statsd-tg](http://octo.it/statsd-tg) – StatsD traffic generator; generates dummy traffic for load testing (C). diff --git a/docs/admin_interface.md b/docs/admin_interface.md index 0c1c9b2e..beeb9b40 100644 --- a/docs/admin_interface.md +++ b/docs/admin_interface.md @@ -2,17 +2,17 @@ A really simple TCP management interface is available by default on port `8126` or overriden in the configuration file. Inspired by the memcache stats approach -this can be used to monitor a live statsd server. You can interact with the +this can be used to monitor a live StatsD server. You can interact with the management server by telnetting to port `8126`, the following commands are available based on the running server. ## Common commands -* health [up|down] - a way to get/set the health status of statsd. Alone will get you the current health status. Passing a second command will set the status to the new value. Accepted values are _up_ and _down_. +* health [up|down] - a way to get/set the health status of StatsD. Alone will get you the current health status. Passing a second command will set the status to the new value. Accepted values are _up_ and _down_. * config - a dump of the current configuration * quit - close the connection from the server side -## Statsd specific commands +## StatsD specific commands * stats - some stats about the running server * counters - a dump of all the current counters @@ -24,8 +24,8 @@ available based on the running server. The stats output currently will give you: -* uptime: the number of seconds elapsed since statsd started -* messages.last_msg_seen: the number of elapsed seconds since statsd received a message +* uptime: the number of seconds elapsed since StatsD started +* messages.last_msg_seen: the number of elapsed seconds since StatsD received a message * messages.bad_lines_seen: the number of bad lines seen since startup You can use the del commands to delete an individual metric like this : @@ -61,11 +61,11 @@ The health output: * using health up or health down, you can change the current health status. * the healthStatus configuration option allows you to set the default health status at start. -## Statsd Proxy specific commands +## StatsD Proxy specific commands * status - the status of the current server The __status__ output currently will give you: -* uptime: the number of seconds elapsed since statsd proxy started +* uptime: the number of seconds elapsed since StatsD proxy started * nodes: a space separated list of host:port for each active node in the ring diff --git a/docs/backend_interface.md b/docs/backend_interface.md index c8a3feac..2cd2a4fb 100644 --- a/docs/backend_interface.md +++ b/docs/backend_interface.md @@ -42,7 +42,7 @@ the `events` object: The counter_rates and timer_data are precalculated statistics to simplify the creation of backends, the statsd_metrics hash contains metrics generated - by statsd itself. Each backend module is passed the same set of + by StatsD itself. Each backend module is passed the same set of statistics, so a backend module should treat the metrics as immutable structures. StatsD will reset timers and counters after each listener has handled the event. diff --git a/docs/client_implementations.md b/docs/client_implementations.md index e8b66746..42e70612 100644 --- a/docs/client_implementations.md +++ b/docs/client_implementations.md @@ -1,6 +1,6 @@ # StatsD Clients -A number of clients have been made for pushing metrics into statsd and open sourced by the wider community. +A number of clients have been made for pushing metrics into StatsD and open sourced by the wider community. **Node** * [lynx](https://github.com/dscape/lynx) — Node.js client used by Mozilla, Nodejitsu, etc. diff --git a/docs/cluster_proxy.md b/docs/cluster_proxy.md index cef4514f..4d12403a 100644 --- a/docs/cluster_proxy.md +++ b/docs/cluster_proxy.md @@ -1,26 +1,26 @@ -# Statsd Cluster Proxy +# StatsD Cluster Proxy -Statsd Cluster Proxy is a udp proxy that sits infront of multiple statsd instances. +StatsD Cluster Proxy is a udp proxy that sits infront of multiple StatsD instances. Create a proxyConfig.js file: `cp exampleProxyConfig.js proxyConfig.js` Once you have modified your config file run: - + `node proxy.js proxyConfig.js` -It uses a consistent hashring to send the unique metric names to the same statsd instances so that +It uses a consistent hashring to send the unique metric names to the same StatsD instances so that the aggregation works properly. -It handles a simple health check that dynamically recalculates the hashring if a statsd instance goes offline. +It handles a simple health check that dynamically recalculates the hashring if a StatsD instance goes offline. Config Options are documented in the [exampleProxyConfig.js][exampleProxyConfig.js] ## Notes -In your statsd configuration make sure to have the following configuration set: `deleteIdleStats: true` +In your StatsD configuration make sure to have the following configuration set: `deleteIdleStats: true` We plan to remove this restriction in the near future: [#pull/348][pull_348] diff --git a/docs/graphite.md b/docs/graphite.md index f74dc7ca..9f4758a5 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -1,22 +1,22 @@ # Configuring Graphite for StatsD Many users have been confused to see their hit counts averaged, gone missing when -the data is intermittent, or never stored when statsd is sending at a different +the data is intermittent, or never stored when StatsD is sending at a different interval than graphite expects. Careful setup of Graphite as suggested below should help to alleviate all these issues. When configuring Graphite, two main factors you need to consider are: -1. What is the highest resolution of data points kept by Graphite, and at which points in time is data downsampled to lower resolutions. This decision is by nature directly related to your functional requirements: how far back should you keep data? what is the data resolution you actually need? However, the retention rules you set must also be in sync with statsd. +1. What is the highest resolution of data points kept by Graphite, and at which points in time is data downsampled to lower resolutions. This decision is by nature directly related to your functional requirements: how far back should you keep data? what is the data resolution you actually need? However, the retention rules you set must also be in sync with StatsD. -2. How should data be aggregated when downsampled, in order to correctly preserve its meaning? Graphite of course knows nothing of the 'meaning' of your data, so let's explore the correct setup for the various metrics sent by statsd. +2. How should data be aggregated when downsampled, in order to correctly preserve its meaning? Graphite of course knows nothing of the 'meaning' of your data, so let's explore the correct setup for the various metrics sent by StatsD. ### Storage Schemas -To define retention and downsampling which match your needs, edit Graphite's conf/storage-schemas.conf file. Here is a simple example file that would handle all metrics sent by statsd: +To define retention and downsampling which match your needs, edit Graphite's conf/storage-schemas.conf file. Here is a simple example file that would handle all metrics sent by StatsD: [stats] pattern = ^stats.* retentions = 10s:6h,1min:6d,10min:1800d -This translates to: for all metrics starting with 'stats' (i.e. all metrics sent by statsd), capture: +This translates to: for all metrics starting with 'stats' (i.e. all metrics sent by StatsD), capture: * 6 hours of 10 second data (what we consider "near-realtime") * 6 days of 1 minute data @@ -24,12 +24,12 @@ This translates to: for all metrics starting with 'stats' (i.e. all metrics sent These settings have been a good tradeoff so far between size-of-file (database files are fixed size) and data we care about. Each "stats" database file is about 3.2 megs with these retentions. -Retentions are read from the file in order and the first pattern that matches is used. +Retentions are read from the file in order and the first pattern that matches is used. Graphite stores each metric in its own database file, and the retentions take effect when a metric file is first created. This means that changing this config file would not affect any files already created. To view or alter the settings on existing files, use whisper-info.py and whisper-resize.py included with the Whisper package. -#### Correlation with statsd's flush interval: +#### Correlation with StatsD's flush interval: -In the case of the above example, what would happen if you flush from statsd any faster than every 10 seconds? in that case, multiple values for the same metric may reach Graphite at any given 10-second timespan, and only the last value would take hold and be persisted - so your data would immediately be partially lost. +In the case of the above example, what would happen if you flush from StatsD any faster than every 10 seconds? in that case, multiple values for the same metric may reach Graphite at any given 10-second timespan, and only the last value would take hold and be persisted - so your data would immediately be partially lost. To fix that, simply ensure your flush interval is at least as long as the highest-resolution retention. However, a long interval may cause other unfortunate mishaps, so keep reading - it pays to understand what's really going on. @@ -37,15 +37,15 @@ To fix that, simply ensure your flush interval is at least as long as the highes ### Storage Aggregation -The next step is ensuring your data isn't corrupted or discarded when downsampled. Continuing with the example above, take for instance the downsampling of .mean values calculated for all statsd timers: +The next step is ensuring your data isn't corrupted or discarded when downsampled. Continuing with the example above, take for instance the downsampling of .mean values calculated for all StatsD timers: -Graphite should downsample up to 6 samples representing 10-second mean values into a single value signfying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurences per flush interval, so you want these samples summed-up, not averaged! +Graphite should downsample up to 6 samples representing 10-second mean values into a single value signfying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurences per flush interval, so you want these samples summed-up, not averaged! You would not even notice any problem till you look at a graph for data older than 6 hours ago, since Graphite would need only the high-res 10-second samples to render the first 6 hours, but would have to switch to lower resolution data for rendering a longer timespan. -Two other metric kinds also deserve a note: +Two other metric kinds also deserve a note: -* Counts which are normalized by statsd to signify a per-second count should not be summed, since their meaning does not change when downsampling. +* Counts which are normalized by StatsD to signify a per-second count should not be summed, since their meaning does not change when downsampling. * Metrics for minimum/maximum values should not be averaged but rather preserve the lowest/highest point, respectively. @@ -100,6 +100,6 @@ Similar to retentions, the aggregations in effect for any metric are set once th ### In conclusion -Graphite's handling of your statsd metrics should be verified at least once: is data mysteriously lost at any point? is data downsampled properly? are you defining graphs for counter metrics without knowing what timespan does each y-value actually represent? (admittedly, in some cases you may not even care about the y-values in the graph, as only the trend is of any interest. The coolest graphs seem to always lack y-values...) +Graphite's handling of your StatsD metrics should be verified at least once: is data mysteriously lost at any point? is data downsampled properly? are you defining graphs for counter metrics without knowing what timespan does each y-value actually represent? (admittedly, in some cases you may not even care about the y-values in the graph, as only the trend is of any interest. The coolest graphs seem to always lack y-values...) For more information, see: http://graphite.readthedocs.org/en/latest/config-carbon.html diff --git a/docs/graphite_pickle.md b/docs/graphite_pickle.md index 3ddcfe4e..71122b39 100644 --- a/docs/graphite_pickle.md +++ b/docs/graphite_pickle.md @@ -1,6 +1,6 @@ # Pickling for Graphite -The graphite statsd backend can optionally be configured to use pickle +The graphite StatsD backend can optionally be configured to use pickle for its over-the-wire protocol. ```javascript @@ -30,7 +30,7 @@ This ends up looking like: The graphite receiver `carbon.protocols.MetricPickleReceiver` coerces both the timestamp and measured value into `float`. -The timestamp must be seconds since epoch encoded as a number. +The timestamp must be seconds since epoch encoded as a number. The measured value is encoded as a string. This may change in the future. diff --git a/docs/metric_types.md b/docs/metric_types.md index 6e95554f..f37b19b5 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -8,7 +8,7 @@ This is a simple counter. Add 1 to the "gorets" bucket. At each flush the current count is sent and reset to 0. If the count at flush is 0 then you can opt to send no metric at all for this counter, by setting `config.deleteCounters` (applies only to graphite -backend). Statsd will send both the rate as well as the count at each flush. +backend). StatsD will send both the rate as well as the count at each flush. ## Sampling @@ -31,7 +31,7 @@ generate the following list of stats for each threshold: stats.timers.$KEY.upper_$PCT stats.timers.$KEY.sum_$PCT -Where `$KEY` is the stats key you specify when sending to statsd, and `$PCT` is +Where `$KEY` is the stats key you specify when sending to StatsD, and `$PCT` is the percentile threshold. Note that the `mean` metric is the mean value of all timings recorded during @@ -43,11 +43,11 @@ more detailed explanation of the calculation. If the count at flush is 0 then you can opt to send no metric at all for this timer, by setting `config.deleteTimers`. -Use the `config.histogram` setting to instruct statsd to maintain histograms +Use the `config.histogram` setting to instruct StatsD to maintain histograms over time. Specify which metrics to match and a corresponding list of ordered non-inclusive upper limits of bins (class intervals). (use `inf` to denote infinity; a lower limit of 0 is assumed) -Each `flushInterval`, statsd will store how many values (absolute frequency) +Each `flushInterval`, StatsD will store how many values (absolute frequency) fall within each bin (class interval), for all matching metrics. Examples: @@ -63,7 +63,7 @@ Examples: [ { metric: 'foo', bins: [] }, { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] -Statsd also maintains a counter for each timer metric. The 3rd field +StatsD also maintains a counter for each timer metric. The 3rd field specifies the sample rate for this counter (in this example @0.1). The field is optional and defaults to 1. @@ -77,7 +77,7 @@ i.e. class intervals of different sizes. ## Gauges -StatsD also supports gauges. A gauge will take on the arbitrary value assigned to it, and will maintain it's value until it is next set. +StatsD also supports gauges. A gauge will take on the arbitrary value assigned to it, and will maintain its value until it is next set. gaugor:333|g diff --git a/docs/server_implementations.md b/docs/server_implementations.md index a8a479e7..64186e80 100644 --- a/docs/server_implementations.md +++ b/docs/server_implementations.md @@ -1,12 +1,12 @@ # Server Implementations -The following is a list of projects that re-implement statsd, if the the main project isn't for you, perhaps one of these is. +The following is a list of projects that re-implement StatsD, if the the main project isn't for you, perhaps one of these is. * [brubeck](https://github.com/github/brubeck) - Server in C * [clj-statsd-svr](https://github.com/netmelody/clj-statsd-svr) — Clojure server * [gographite](https://github.com/amir/gographite) — Server in Go * [gostatsd](https://github.com/atlassian/gostatsd) — Server in Go -* [netdata](https://github.com/firehol/netdata) - Embedded statsd server in the netdata server, in C, with visualization +* [netdata](https://github.com/firehol/netdata) - Embedded StatsD server in the netdata server, in C, with visualization * [Net::Statsd::Server](https://github.com/cosimo/perl5-net-statsd-server) — Perl server, also available on [CPAN](https://metacpan.org/module/Net::Statsd::Server) * [Py-Statsd](https://github.com/sivy/py-statsd) — Server and Client * [Ruby-Statsdserver](https://github.com/fetep/ruby-statsdserver) — Ruby server From 27369f4cb93afb61c8bed69e0a30e72d52a4bf0a Mon Sep 17 00:00:00 2001 From: Alfonso Montero Date: Thu, 19 Mar 2020 11:41:28 +0100 Subject: [PATCH 172/207] add Docker Hub image pulls badge to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd7847c0..f6137f02 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +# StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Docker Pulls](https://img.shields.io/docker/pulls/statsd/statsd)](https://hub.docker.com/r/statsd/statsd) A network daemon that runs on the [Node.js][node] platform and listens for statistics, like counters and timers, sent over [UDP][udp] or From 56fc03a560c90d01be7bee34560baa8eaaa535df Mon Sep 17 00:00:00 2001 From: Lorenzo Aiello Date: Tue, 24 Mar 2020 04:37:08 -0700 Subject: [PATCH 173/207] Splitting doc links for Elasticsearch backends (#695) --- docs/backend.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/backend.md b/docs/backend.md index 7839e1db..25c6ecb9 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -37,7 +37,9 @@ queues and third-party services. * [node-bell](https://github.com/eleme/node-bell) * [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) * [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) -* [elasticsearch-backend](https://github.com/markkimsal/statsd-elasticsearch-backend) +* elasticsearch-backend + * [Elasticsearch 5 and 6](https://github.com/markkimsal/statsd-elasticsearch-backend) + * [Elasticsearch 7](https://github.com/lorenzoaiello/statsd-elasticsearch7-backend) * [ganglia-backend](https://github.com/jbuchbinder/statsd-ganglia-backend) * [hosted graphite backend](https://github.com/hostedgraphite/statsdplugin) * [influxdb backend](https://github.com/bernd/statsd-influxdb-backend) From f82317a909cd8070c37e5dc2bbfe91388d04aa21 Mon Sep 17 00:00:00 2001 From: Bruno Adele Date: Fri, 3 Apr 2020 11:14:58 +0200 Subject: [PATCH 174/207] Update Proxy example to prevent crashes --- exampleProxyConfig.js | 1 + 1 file changed, 1 insertion(+) diff --git a/exampleProxyConfig.js b/exampleProxyConfig.js index 040fb32a..6182df90 100644 --- a/exampleProxyConfig.js +++ b/exampleProxyConfig.js @@ -34,6 +34,7 @@ nodes: [ server: './servers/udp', host: '0.0.0.0', port: 8125, +udp_version:'udp4', mgmt_port: 8126, forkCount: 0, checkInterval: 1000, From 22f1d357e93c2db09c01f196c6308eaa6b2f635b Mon Sep 17 00:00:00 2001 From: twocucao Date: Wed, 8 Apr 2020 15:16:25 +0800 Subject: [PATCH 175/207] Update graphite.md not supported date unit https://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf --- docs/graphite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/graphite.md b/docs/graphite.md index 9f4758a5..4da8f194 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -14,7 +14,7 @@ To define retention and downsampling which match your needs, edit Graphite's con [stats] pattern = ^stats.* - retentions = 10s:6h,1min:6d,10min:1800d + retentions = 10s:6h,1m:6d,10m:1800d This translates to: for all metrics starting with 'stats' (i.e. all metrics sent by StatsD), capture: From 406ee1d26c307e73620da2a31faea09b742291fc Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 27 Apr 2020 09:12:05 +0100 Subject: [PATCH 176/207] update docker container to latest version of node 10 (10.20.1) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3fa2dab7..3b9140be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.15.3 +FROM node:10.20.1 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app From 7589c3b60a79888a6fed2b3bf5e71af864093bc0 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Sun, 10 May 2020 17:44:48 +0300 Subject: [PATCH 177/207] Add redistimeseries-backend (#704) --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 25c6ecb9..2083ba65 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -53,6 +53,7 @@ queues and third-party services. * [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) * [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) * [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) +* [redistimeseries ackend](https://github.com/hashedin/statsd-redistimeseries-backend) * [socket.io-backend](https://github.com/Chatham/statsd-socket.io) * [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) * [statsd-backend](https://github.com/dynmeth/statsd-backend) From f433f4ac577495ec4ab885eaa3f61c47e8396f48 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Thu, 11 Jun 2020 23:44:08 +0900 Subject: [PATCH 178/207] Add Swift client implementation to docs --- docs/client_implementations.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/client_implementations.md b/docs/client_implementations.md index 42e70612..f748c808 100644 --- a/docs/client_implementations.md +++ b/docs/client_implementations.md @@ -27,6 +27,9 @@ A number of clients have been made for pushing metrics into StatsD and open sour * [statsd](https://github.com/reinh/statsd/) — Ruby client (needs new maintainer) * [Statsd-Client](https://github.com/dawanda/statsd-client) — Ruby client (not maintained) +**Swift** +* [swift-statsd-client](https://github.com/apple/swift-statsd-client) - Swift client + **Perl** * [Net::Statsd](https://github.com/cosimo/perl5-net-statsd) — Perl client, also available on [CPAN](https://metacpan.org/module/Net::Statsd) * [Net::StatsD::Client](https://github.com/sivy/statsd-client) — Perl client, not available on CPAN From bd85bba3b6dab78de4ff2b1f66650dd51d91ccf3 Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 11 Jun 2020 15:45:42 +0100 Subject: [PATCH 179/207] Change ActorContext to ActorRefFactory (#647) Accepting the superclass ActorRefFactory allows passing either an ActorSystem (often more convenient) or an ActorContext for setup. --- examples/StatsD.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/StatsD.scala b/examples/StatsD.scala index 7830b92d..80d65682 100644 --- a/examples/StatsD.scala +++ b/examples/StatsD.scala @@ -42,7 +42,7 @@ import akka.actor._ * @param multiMetrics If true, multiple stats will be sent in a single UDP packet * @param packetBufferSize If multiMetrics is true, this is the max buffer size before sending the UDP packet */ -class StatsD(context: ActorContext, +class StatsD(context: ActorRefFactory, host: String, port: Int, multiMetrics: Boolean = true, @@ -229,4 +229,4 @@ private class StatsDActor(host: String, } } } -} \ No newline at end of file +} From ab17249217bf7317fb6e4929638cbc804c35cafd Mon Sep 17 00:00:00 2001 From: Hussain Nagri <44284459+nutant-h@users.noreply.github.com> Date: Thu, 2 Jul 2020 17:27:35 +0530 Subject: [PATCH 180/207] Update backend.md Spelling mistake. --- docs/backend.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend.md b/docs/backend.md index 2083ba65..8c143547 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -53,7 +53,7 @@ queues and third-party services. * [netuitive backend](https://github.com/Netuitive/statsd-netuitive-backend) * [opencensus-backend](https://github.com/DazWilkin/statsd-opencensus-backend) * [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) -* [redistimeseries ackend](https://github.com/hashedin/statsd-redistimeseries-backend) +* [redistimeseries backend](https://github.com/hashedin/statsd-redistimeseries-backend) * [socket.io-backend](https://github.com/Chatham/statsd-socket.io) * [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) * [statsd-backend](https://github.com/dynmeth/statsd-backend) From 2041f6fb5e64bbf779a8bcb3e9729e63fe207e2f Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Tue, 25 Aug 2020 08:53:25 -0400 Subject: [PATCH 181/207] Graphite tagged metrics support (#697) * support for tagged metrics * decode datadog tag format * don't replace tagging characters when sanitizing graphite metric names --- backends/graphite.js | 45 +++++++++++++++++++++--------------------- stats.js | 18 ++++++++++++++--- test/graphite_tests.js | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index de23a852..c4a74596 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -14,7 +14,7 @@ * graphitePort: Port for the graphite text collector. Defaults to 2003. * graphitePicklePort: Port for the graphite pickle collector. Defaults to 2004. * graphiteProtocol: Either 'text' or 'pickle'. Defaults to 'text'. - * + * * If graphiteHost is not specified, metrics are processed but discarded. */ @@ -108,7 +108,7 @@ function Metric(key, value, ts) { this.value = value; this.ts = ts; - // return a string representation of this metric appropriate + // return a string representation of this metric appropriate // for sending to the graphite collector. does not include // a trailing newline. this.toText = function() { @@ -167,10 +167,18 @@ var flush_stats = function graphite_flush(ts, metrics) { } else { return key.replace(/\s+/g, '_') .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + .replace(/[^a-zA-Z_\-0-9\.;=]/g, ''); } }; + function format(namespace, key) { + var splitName = key.split(';'); + var keyName = sk(splitName[0]); + var tags = splitName.length > 1 ? (';' + splitName.slice(1).join(';')) : ''; + + return namespace.concat(keyName, [].slice.call(arguments, 2)).join('.') + globalSuffix + tags; + } + // Flatten all the different types of metrics into a single // collection so we can allow serialization to either the graphite // text and pickle formats. @@ -179,18 +187,16 @@ var flush_stats = function graphite_flush(ts, metrics) { for (key in counters) { var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate - var keyName = sk(key); - var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { - stats.add(namespace.join(".") + globalSuffix, valuePerSecond, ts); + stats.add(format(counterNamespace, key), valuePerSecond, ts); if (flush_counts) { - stats.add('stats_counts.' + keyName + globalSuffix, value, ts); + stats.add(format(['stats_counts'], key), value, ts); } } else { - stats.add(namespace.concat('rate').join(".") + globalSuffix, valuePerSecond, ts); + stats.add(format(counterNamespace, key, 'rate'), valuePerSecond, ts); if (flush_counts) { - stats.add(namespace.concat('count').join(".") + globalSuffix, value, ts); + stats.add(format(counterNamespace, key, 'count'), value, ts); } } @@ -198,18 +204,15 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(sk(key)); - var the_key = namespace.join("."); - for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { - stats.add(the_key + '.' + timer_data_key + globalSuffix, timer_data[key][timer_data_key], ts); + stats.add(format(timerNamespace, key, timer_data_key), timer_data[key][timer_data_key], ts); } else { for (var timer_data_sub_key in timer_data[key][timer_data_key]) { if (debug) { l.log(timer_data[key][timer_data_key][timer_data_sub_key].toString()); } - stats.add(the_key + '.' + timer_data_key + '.' + timer_data_sub_key + globalSuffix, + stats.add(format(timerNamespace, key, timer_data_key, timer_data_sub_key), timer_data[key][timer_data_key][timer_data_sub_key], ts); } } @@ -218,14 +221,12 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(sk(key)); - stats.add(namespace.join(".") + globalSuffix, gauges[key], ts); + stats.add(format(gaugesNamespace, key), gauges[key], ts); numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(sk(key)); - stats.add(namespace.join(".") + '.count' + globalSuffix, sets[key].size(), ts); + stats.add(format(setsNamespace, key, 'count'), sets[key].size(), ts); numStats += 1; } @@ -236,12 +237,10 @@ var flush_stats = function graphite_flush(ts, metrics) { stats.add('stats.' + prefixStats + '.' + key + globalSuffix, statsd_metrics[key], ts); } } else { - var namespace = globalNamespace.concat(prefixStats); - stats.add(namespace.join(".") + '.numStats' + globalSuffix, numStats, ts); - stats.add(namespace.join(".") + '.graphiteStats.calculationtime' + globalSuffix, (Date.now() - starttime) , ts); + stats.add(format(globalNamespace, prefixStats, 'numStats'), numStats, ts); + stats.add(format(globalNamespace, prefixStats, 'graphiteStats', 'calculationtime'), (Date.now() - starttime) , ts); for (key in statsd_metrics) { - var the_key = namespace.concat(key); - stats.add(the_key.join(".") + globalSuffix,+ statsd_metrics[key], ts); + stats.add(format(globalNamespace, prefixStats, key), statsd_metrics[key], ts); } } post_stats(stats); diff --git a/stats.js b/stats.js index ceb2ae1b..75f57412 100644 --- a/stats.js +++ b/stats.js @@ -165,7 +165,7 @@ function sanitizeKeyName(key) { if (keyNameSanitize) { return key.replace(/\s+/g, '_') .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + .replace(/[^a-zA-Z_\-0-9\.;=]/g, ''); } else { return key; } @@ -252,8 +252,20 @@ config.configFile(process.argv[2], function (config) { if (config.dumpMessages) { l.log(metrics[midx].toString()); } - const bits = metrics[midx].toString().split(':'); - const key = sanitizeKeyName(bits.shift()); + + let bits = metrics[midx].toString().split('|#'); + let tags = []; + if (bits.length > 1 && bits[1].length > 0) { + tags = bits[1].split(','); + } + bits = bits[0].split(':'); + let key = bits.shift(); + if (tags.length > 0) { + key += ';' + tags.map(function(tag) { + return tag.replace(';', '_').replace(':', '='); + }).join(';'); + } + key = sanitizeKeyName(key); if (keyFlushInterval > 0) { if (! keyCounter[key]) { diff --git a/test/graphite_tests.js b/test/graphite_tests.js index 67f5b9a4..d8893bb2 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -377,5 +377,43 @@ module.exports = { }); }); }); + }, + + graphite_tags_are_supported: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o;tag1=val1:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar;tag1=val1;tag2=val2:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar;tag1=val1;tag3=val3;tag2=val2:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o.count;tag1=val1') !== -1, "Did not map 'fo/o;tag1=val1' => 'fo-o.count;tag1=val1'"); + test.ok(str.indexOf('b_ar.count;tag1=val1;tag2=val2') !== -1, "Did not map 'b ar;tag1=val1;tag2=val2' => 'b_ar.count;tag1=val1;tag2=val2'"); + test.ok(str.indexOf('foobar.count;tag1=val1;tag3=val3;tag2=val2') !== -1, "Did not map 'foo+bar;tag1=val1;tag3=val3;tag2=val2' => 'foobar.count;tag1=val1;tag3=val3;tag2=val2'"); + test.done(); + }); + }); + }); + }); + }); + }, + + dogstatsd_tags_are_supported: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c|#tag1:val1',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c|#tag1:val1,tag2:val2',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c|#tag1:val;1,tag3:val3,tag2:val2',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o.count;tag1=val1') !== -1, "Did not map 'fo/o:250|c|#tag1:val1' => 'fo-o.count;tag1=val1'"); + test.ok(str.indexOf('b_ar.count;tag1=val1;tag2=val2') !== -1, "Did not map 'b ar:250|c|#tag1:val1,tag2:val2' => 'b_ar.count;tag1=val1;tag2=val2'"); + test.ok(str.indexOf('foobar.count;tag1=val_1;tag3=val3;tag2=val2') !== -1, "Did not map 'foo+bar:250|c|#tag1:val;1,tag3:val3,tag2:val2' => 'foobar.count;tag1=val_1;tag3=val3;tag2=val2'"); + test.done(); + }); + }); + }); + }); + }); } } From a6b6cd2b50cc26e201523446ff76a01b02a42689 Mon Sep 17 00:00:00 2001 From: Rishikesh Meena Date: Sat, 23 May 2020 18:49:45 +0530 Subject: [PATCH 182/207] Set last_exception time to 0 on initialisation When this was being set to the startup_time it was causing confusion with users of the admin panel, leading people to believe there had been an exception which wasn't being logged. Since no exceptions are occuring, a 0 initialisation seems reasonable here. --- backends/graphite.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index c4a74596..1e6e462c 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -316,7 +316,7 @@ exports.init = function graphite_init(startup_time, config, events, logger) { } graphiteStats.last_flush = startup_time; - graphiteStats.last_exception = startup_time; + graphiteStats.last_exception = 0; graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; From 3705777aa6552cea3830aa2f3605ac2c36ae15cd Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 27 Aug 2020 16:29:19 +0100 Subject: [PATCH 183/207] Update container image to use current node LTS --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3b9140be..ef7f76c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.20.1 +FROM node:12.8.3 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app From fe835b46ef79a805e316c273d57bd31678957085 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 27 Aug 2020 16:32:16 +0100 Subject: [PATCH 184/207] Update CI to use current node lts versions --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 539886bd..9baa7fe2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,8 @@ language: node_js node_js: -- '8' - '10' -- '11' - '12' +- '14' script: node run_tests.js notifications: email: false From c37feb940f92db10529d6e86cc0bee80a40af010 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 27 Aug 2020 16:34:50 +0100 Subject: [PATCH 185/207] release 0.9.0 --- Changelog.md | 8 ++++++++ debian/changelog | 10 ++++++++++ package.json | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 91cd7b4d..292080d4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,13 @@ # Changelog +## v0.9.0 (08/27/2020) + +- Add support for graphite tagged metrics +- Fix dashboard to 0 last_exception time on startup +- Multiple documentation updates +- Correct some out-dated integration examples +- Update container image to use recent node version + ## v0.8.6 (02/19/2020) - Add an optional max TTL setting for gauges diff --git a/debian/changelog b/debian/changelog index b89d7ae4..b25d0c63 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +statsd (0.9.0-1) unstable; urgency=low + + * Add support for graphite tagged metrics + * Fix dashboard to 0 last_exception time on startup + * Multiple documentation updates + * Correct some out-dated integration examples + * Update container image to use recent node version + + -- Elliot Blackburn Thu, 27 Aug 2020 15:30:29 +0000 + statsd (0.8.6-1) unstable; urgency=low * Add an optional max TTL setting for gauges diff --git a/package.json b/package.json index 0fcbc658..fbe60a9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.8.6", + "version": "0.9.0", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 9cf77d87855bcb69a8663135f59aa23825db9797 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Thu, 27 Aug 2020 16:44:22 +0100 Subject: [PATCH 186/207] Correct dockerfile version to 12.18.3 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef7f76c1..f2cf8233 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.8.3 +FROM node:12.18.3 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app From c47fc4807dc199210f6fac3a5a47dfcda5d45ea0 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 28 Dec 2021 08:43:30 +1100 Subject: [PATCH 187/207] docs: Fix a few typos (#725) There are small typos in: - docs/graphite.md - docs/metric_types.md - exampleConfig.js - examples/python_example.py - proxy.js Fixes: - Should read `occurrences` rather than `occurences`. - Should read `undefined` rather than `undefinded`. - Should read `signifying` rather than `signfying`. - Should read `retrieves` rather than `retreives`. - Should read `retrieved` rather than `retreived`. - Should read `resources` rather than `recources`. - Should read `metric` rather than `mertric`. --- docs/graphite.md | 2 +- docs/metric_types.md | 2 +- exampleConfig.js | 2 +- examples/python_example.py | 2 +- proxy.js | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/graphite.md b/docs/graphite.md index 4da8f194..064fbdf0 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -39,7 +39,7 @@ To fix that, simply ensure your flush interval is at least as long as the highes The next step is ensuring your data isn't corrupted or discarded when downsampled. Continuing with the example above, take for instance the downsampling of .mean values calculated for all StatsD timers: -Graphite should downsample up to 6 samples representing 10-second mean values into a single value signfying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurences per flush interval, so you want these samples summed-up, not averaged! +Graphite should downsample up to 6 samples representing 10-second mean values into a single value signifying the mean for a 1-minute timespan. This is simple: just average all samples to get the new value, and this is exactly the default method applied by Graphite. However, what about the .count metric also sent for timers? Each sample contains the count of occurrences per flush interval, so you want these samples summed-up, not averaged! You would not even notice any problem till you look at a graph for data older than 6 hours ago, since Graphite would need only the high-res 10-second samples to render the first 6 hours, but would have to switch to lower resolution data for rendering a longer timespan. diff --git a/docs/metric_types.md b/docs/metric_types.md index f37b19b5..06c15839 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -99,7 +99,7 @@ without first setting it to zero. ## Sets -StatsD supports counting unique occurences of events between flushes, +StatsD supports counting unique occurrences of events between flushes, using a Set to store all occuring events. uniques:765|s diff --git a/exampleConfig.js b/exampleConfig.js index 271b8e28..0f080c49 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -32,7 +32,7 @@ Optional Variables: address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] port: port to listen for messages on [default: 8125] socket: (only for tcp servers) path to unix domain socket which will be used to receive - metrics [default: undefinded] + metrics [default: undefined] socket_mod: (only for tcp servers) file mode which should be applied to unix domain socket, relevant only if socket option is used [default: undefined] diff --git a/examples/python_example.py b/examples/python_example.py index 28f20410..3b21a1d0 100644 --- a/examples/python_example.py +++ b/examples/python_example.py @@ -150,7 +150,7 @@ def send(_dict, addr): >>> StatsdClient.send({"example.send":"11|c"}, ("127.0.0.1", 8125)) """ # TODO(rbtz@): IPv6 support - # TODO(rbtz@): Creating socket on each send is a waste of recources + # TODO(rbtz@): Creating socket on each send is a waste of resources udp_sock = socket(AF_INET, SOCK_DGRAM) # TODO(rbtz@): Add batch support for item in _dict.items(): diff --git a/proxy.js b/proxy.js index d051d38f..b78c94ac 100644 --- a/proxy.js +++ b/proxy.js @@ -115,7 +115,7 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { if (packet_data.indexOf("\n") > -1) { var metrics; metrics = packet_data.split("\n"); - // Loop through the metrics and split on : to get mertric name for hashing + // Loop through the metrics and split on : to get metric name for hashing for (var midx in metrics) { current_metric = metrics[midx]; bits = current_metric.split(':'); @@ -139,10 +139,10 @@ configlib.configFile(process.argv[2], function (conf, oldConfig) { var client = dgram.createSocket(udp_version); // Listen for the send message, and process the metric key and msg packet.on('send', function(key, msg) { - // retreives the destination for this key + // retrieves the destination for this key var statsd_host = ring.get(key); - // break the retreived host to pass to the send function + // break the retrieved host to pass to the send function if (statsd_host === undefined) { log('Warning: No backend statsd nodes available!', 'WARNING'); } else { From 8e6e29ea1f00062c9be27eebb122fa8c427bc74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zden=C4=9Bk=20H=C5=99eb=C3=AD=C4=8Dek?= <10925068+zhrebicek@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:19:00 +0200 Subject: [PATCH 188/207] Add coralogix to list of third-party backends (#731) --- docs/backend.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backend.md b/docs/backend.md index 8c143547..e9d82dd7 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -35,6 +35,7 @@ queues and third-party services. * [atsd-backend](https://github.com/axibase/atsd-statsd-backend) * [aws-cloudwatch-backend](https://github.com/camitz/aws-cloudwatch-statsd-backend) * [node-bell](https://github.com/eleme/node-bell) +* [coralogix-backend](https://github.com/coralogix/statsd-coralogix-backend) * [couchdb-backend](https://github.com/sysadminmike/couch-statsd-backend) * [datadog-backend](https://github.com/DataDog/statsd-datadog-backend) * elasticsearch-backend From b676a51d71ce4ccc1e7f419f121871f754649df5 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sun, 20 Nov 2022 16:44:47 -0500 Subject: [PATCH 189/207] Fixed docker-compose configuration for 2 The Docker Compose syntax has changed since this file was added. This makes the configuration actually work with modern docker-compose. --- docker-compose.yml | 48 ++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5d702861..7704916f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,27 @@ -statsd: - build: . - links: - - carbon:graphite - ports: - - 8125:8125/udp - - 8126:8126 +version: '2' +services: + statsd: + build: . + links: + - carbon:graphite + ports: + - 8125:8125/udp + - 8126:8126 -graphite-web: - image: dockerana/graphite - links: - - carbon - ports: - - 8000:8000 - volumes_from: - - carbon + graphite-web: + image: dockerana/graphite + links: + - carbon + ports: + - 8000:8000 + volumes_from: + - carbon -carbon: - image: dockerana/carbon - ports: - - 2003:2003 - - 2004:2004 - - 7002:7002 - volumes: - - /opt/graphite + carbon: + image: dockerana/carbon + ports: + - 2003:2003 + - 2004:2004 + - 7002:7002 + volumes: + - /opt/graphite From a2bc0522dae8151565b4bfd4f17e0c1d6b488f8f Mon Sep 17 00:00:00 2001 From: Paul Milbank Date: Wed, 3 May 2023 08:25:08 +1200 Subject: [PATCH 190/207] Update Travis to test current Node LTS releases --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9baa7fe2..49316bce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: node_js node_js: -- '10' -- '12' - '14' +- '16' +- '18' script: node run_tests.js notifications: email: false From d8383d21c0edbd88f3868fbceaf243b3ce8bdd00 Mon Sep 17 00:00:00 2001 From: Paul Milbank Date: Wed, 3 May 2023 08:25:30 +1200 Subject: [PATCH 191/207] Fix test errors by using python3 --- test/graphite_pickle_tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/graphite_pickle_tests.js b/test/graphite_pickle_tests.js index e68051a7..ca653dff 100644 --- a/test/graphite_pickle_tests.js +++ b/test/graphite_pickle_tests.js @@ -95,7 +95,7 @@ var unpickle = function(payload, cb) { fs.close(unpickle_info.fd, function(err) { if (err) throw err; - var cmd = 'python ' + unpickle_info.path + ' ' + payload_info.path; + var cmd = 'python3 ' + unpickle_info.path + ' ' + payload_info.path; var python = cp.exec(cmd, function(err, stdout, stderr) { if (err) throw err; var metrics = JSON.parse(stdout); From f06ebce5b5770a2d2ec5b5a5148bfa102dfcb15f Mon Sep 17 00:00:00 2001 From: Paul Milbank Date: Wed, 3 May 2023 08:25:54 +1200 Subject: [PATCH 192/207] Update the dockerfile to build from Node:18.16.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2cf8233..cd483450 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.18.3 +FROM node:18.16.0 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app From 9fada91b2088da2e6a56a49ca06476a25b9db74e Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 17:11:02 +0100 Subject: [PATCH 193/207] Draft github action for container publishing (#742) --- .../workflows/package-and-publish-image.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/package-and-publish-image.yml diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml new file mode 100644 index 00000000..24955fcf --- /dev/null +++ b/.github/workflows/package-and-publish-image.yml @@ -0,0 +1,53 @@ +# Build, package, and publish a container image to our supported +# image registries. We generate a number of tags in different scenarios. +# +# * Commit pushed on default branch: publish with semver and "latest" tags. +# * Tag on default branch pushed: publish with short commit sha and "edge" tags. +# +# We currently publish to dockerhub and ghcr. + +name: Package and publish container image + +on: + push: + branches: + - master + - docker-publish-test + +jobs: + publish: + name: Build and publish image + runs-on: ubuntu-22.04 + steps: + - name: Generate container image meta tags + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/statsd/statsd + flavour: | + latest=true + # type=semver,pattern=v{{version}},event=tag + # type=semver,pattern=v{{major.minor}},event=tag + # type=semver,pattern=v{{major}},event=tag + # type=edge,branch=$repo.default_branch,event=push + # type=sha,branch=$repo.default_branch,event=push + tag: | + type=edge,event=push + type=sha,event=push + - name: Setup buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GitHub container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish image + id: docker_build + uses: docker/build-and-push-action@v3 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + - name: Echo image digest + run: echo ${{ steps.docker_build.outputs.digest }} From 76ea1576877078c1bd9310b0ba1c80ef39e15ee7 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 17:41:35 +0100 Subject: [PATCH 194/207] GitHub Action for building and publishing container images (#743) Images are published to dockerhub and ghcr as documented in the action file. --- .../workflows/package-and-publish-image.yml | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index 24955fcf..fee5ed25 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -12,7 +12,6 @@ on: push: branches: - master - - docker-publish-test jobs: publish: @@ -24,30 +23,39 @@ jobs: uses: docker/metadata-action@v4 with: images: | + statsd/statsd ghcr.io/statsd/statsd - flavour: | + flavor: | latest=true - # type=semver,pattern=v{{version}},event=tag - # type=semver,pattern=v{{major.minor}},event=tag - # type=semver,pattern=v{{major}},event=tag - # type=edge,branch=$repo.default_branch,event=push - # type=sha,branch=$repo.default_branch,event=push - tag: | - type=edge,event=push - type=sha,event=push + tags: | + type=semver,pattern=v{{version}},event=tag + type=semver,pattern=v{{major.minor}},event=tag + type=semver,pattern=v{{major}},event=tag + type=edge,branch=$repo.default_branch,event=push + type=sha,branch=$repo.default_branch,event=push + - name: Setup buildx uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub container registry + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub container registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ github.token }} + - name: Build and publish image id: docker_build - uses: docker/build-and-push-action@v3 + uses: docker/build-push-action@v4 with: push: true tags: ${{ steps.meta.outputs.tags }} + - name: Echo image digest run: echo ${{ steps.docker_build.outputs.digest }} From eb51abea4a4d0676ccfdcc83bd5e72fbe2296bcb Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 18:00:16 +0100 Subject: [PATCH 195/207] Remove redundant travis-ci file --- .travis.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 49316bce..00000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: node_js -node_js: -- '14' -- '16' -- '18' -script: node run_tests.js -notifications: - email: false - irc: - - irc.freenode.org#statsd -deploy: - provider: npm - email: elliot@lybrary.io - api_key: - secure: deHm/dx0AJoSpkgbi+480BbC8Qxsv5vf4TPfMqM8uGYicj6qMJBjG7AZtCUIcJ0mSEoc9Zf79jOJCeWC53CWziQX+D+AXDC/8RGlqtx1h1zgpM6m1NonceqUt9wN/tVG1eNa/22WrerMA3F2rxo5QtY3rMxGMlJjCYOwyFR++eM= - on: - tags: true - repo: statsd/statsd From c6c6dc1418b31d0cd64fca4f0f73d56a64581922 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 18:00:27 +0100 Subject: [PATCH 196/207] Update CONTRIBUTING.md --- CONTRIBUTING.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c88fd56..40a94306 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,20 +2,19 @@ You're interested in contributing to StatsD? *AWESOME*. Here are the basic steps: -fork StatsD from here: http://github.com/etsy/statsd - -1. Clone your fork -2. Hack away -3. If you are adding new functionality, document it in the [docs][d] -4. If necessary, rebase your commits into logical chunks, without errors -5. Verify your code by running the test suite, and adding additional tests if able. -6. Push the branch up to GitHub -7. Send a pull request to the etsy/statsd project. +1. fork StatsD from here: http://github.com/statsd/statsd +2. Clone your fork +3. Hack away +4. If you are adding new functionality, document it in the [docs][d] +5. If necessary, rebase your commits into logical chunks, without errors +6. Verify your code by running the test suite, and adding additional tests if able. +7. Push the branch up to GitHub +8. Send a pull request to the statsd/statsd project. We'll do our best to get your changes in! # Contributors -In lieu of a list of contributors, check out the [commit history for the project](https://github.com/etsy/statsd/graphs/contributors) +In lieu of a list of contributors, check out the [commit history for the project](https://github.com/statsd/statsd/graphs/contributors) -[d]: https://github.com/etsy/statsd/tree/master/docs +[d]: https://github.com/statsd/statsd/tree/master/docs From 3f0e8c1ccc0ed234a060e80e6ff1652d17c814e9 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 18:00:38 +0100 Subject: [PATCH 197/207] Add reference to GHCR package in README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f6137f02..fd70cf68 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Docker Pulls](https://img.shields.io/docker/pulls/statsd/statsd)](https://hub.docker.com/r/statsd/statsd) +# StatsD [![Join the chat at https://gitter.im/statsd/statsd](https://badges.gitter.im/statsd/statsd.svg)](https://gitter.im/statsd/statsd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Docker Pulls](https://img.shields.io/docker/pulls/statsd/statsd)](https://hub.docker.com/r/statsd/statsd) A network daemon that runs on the [Node.js][node] platform and listens for statistics, like counters and timers, sent over [UDP][udp] or @@ -27,8 +27,9 @@ general values should be integers. ## Installation and Configuration ### Docker -StatsD supports docker in two ways: -* The official docker image on [docker hub](https://hub.docker.com/r/statsd/statsd) +StatsD supports docker in three ways: +* The official container image on [GitHub Container Registry](https://github.com/statsd/statsd/pkgs/container/statsd) +* The official container image on [DockerHub](https://hub.docker.com/r/statsd/statsd) * Building the image from the bundled [Dockerfile](./Dockerfile) ### Manual installation From bdca7994113fc40e88edbef4f34ba3d8343ccd19 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 18:04:47 +0100 Subject: [PATCH 198/207] Upgrade buildx publish action step (#744) --- .github/workflows/package-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index fee5ed25..857b0388 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -35,7 +35,7 @@ jobs: type=sha,branch=$repo.default_branch,event=push - name: Setup buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub container registry uses: docker/login-action@v2 From b2b4b68577107e92fbcaac60566725f16e184637 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Wed, 10 May 2023 18:09:49 +0100 Subject: [PATCH 199/207] Revert "Upgrade buildx publish action step (#744)" This reverts commit bdca7994113fc40e88edbef4f34ba3d8343ccd19. --- .github/workflows/package-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index 857b0388..fee5ed25 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -35,7 +35,7 @@ jobs: type=sha,branch=$repo.default_branch,event=push - name: Setup buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to DockerHub container registry uses: docker/login-action@v2 From d9e53dcbff2f71ea3d8dc5c6d056be3d8a025946 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Thu, 11 May 2023 09:10:50 -0400 Subject: [PATCH 200/207] release 0.10.0 --- Changelog.md | 8 ++++++++ debian/changelog | 10 ++++++++++ package.json | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 292080d4..b9ddc256 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,13 @@ # Changelog +## v0.10.0 (05/11/2023) + +- Multiple documentation updates +- Updated docker-compose syntax +- Upgrade tests to use python3 +- Upgrade node base for Docker image +- Automatic container publishing via GHA + ## v0.9.0 (08/27/2020) - Add support for graphite tagged metrics diff --git a/debian/changelog b/debian/changelog index b25d0c63..51b34677 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +statsd (0.10.0-1) unstable; urgency=low + + * Multiple documentation updates + * Updated docker-compose syntax + * Upgrade tests to use python3 + * Upgrade node base for Docker image + * Automatic container publishing via GHA + + -- Mike Heffner Thu, 11 May 2023 13:03:29 +0000 + statsd (0.9.0-1) unstable; urgency=low * Add support for graphite tagged metrics diff --git a/package.json b/package.json index fbe60a9d..644a1119 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.9.0", + "version": "0.10.0", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 4adb7d49776cfb6da1acb801a46bd56047728e45 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Thu, 11 May 2023 11:58:17 -0400 Subject: [PATCH 201/207] Run on tags as well. --- .github/workflows/package-and-publish-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index fee5ed25..ede0b175 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -12,6 +12,8 @@ on: push: branches: - master + tags: + - "v*" jobs: publish: From 51d821790c29bc930814fe64a1b28d8fdc1d8237 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Thu, 11 May 2023 12:03:15 -0400 Subject: [PATCH 202/207] release 0.10.1 --- Changelog.md | 4 ++++ debian/changelog | 6 ++++++ package.json | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index b9ddc256..e68c94f7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Changelog +## v0.10.1 (05/11/2023) + +- Include tag pushes in docker builds + ## v0.10.0 (05/11/2023) - Multiple documentation updates diff --git a/debian/changelog b/debian/changelog index 51b34677..dccee0d8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +statsd (0.10.1-1) unstable; urgency=low + + * Include tag pushes in docker builds + + -- Mike Heffner Thu, 11 May 2023 16:02:29 +0000 + statsd (0.10.0-1) unstable; urgency=low * Multiple documentation updates diff --git a/package.json b/package.json index 644a1119..90a47764 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.10.0", + "version": "0.10.1", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From b7983094efc9f385c21ef1b4ccd11b7be69f353a Mon Sep 17 00:00:00 2001 From: Namrata Bhave Date: Mon, 24 Jul 2023 21:03:44 +0530 Subject: [PATCH 203/207] Support amd64 and s390x container platforms (#746) --- .github/workflows/package-and-publish-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index ede0b175..b34ad214 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -57,6 +57,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true + platforms: linux/amd64,linux/s390x tags: ${{ steps.meta.outputs.tags }} - name: Echo image digest From 0e20be57d4918dadf5b5fad61b5c61ef382f8860 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 22 Aug 2023 13:15:32 +0100 Subject: [PATCH 204/207] release 0.10.2 --- Changelog.md | 4 ++++ debian/changelog | 6 ++++++ package.json | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index e68c94f7..74549e55 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Changelog +## v0.10.2 (08/22/2023) + +- Support publishing amd64 and s390x container images + ## v0.10.1 (05/11/2023) - Include tag pushes in docker builds diff --git a/debian/changelog b/debian/changelog index dccee0d8..78c9c282 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +statsd (0.10.2-1) unstable; urgency=low + + * Support publishing amd64 and s390x container images + + -- Elliot Blackburn Tue, 22 Aug 2023 13:14:48 +0000 + statsd (0.10.1-1) unstable; urgency=low * Include tag pushes in docker builds diff --git a/package.json b/package.json index 90a47764..30623f08 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "statsd", - "version": "0.10.1", + "version": "0.10.2", "description": "Network daemon for the collection and aggregation of realtime application metrics", "author": { "name": "Etsy", From 7c07eec4e7cebbd376d8313b230cea96c6571423 Mon Sep 17 00:00:00 2001 From: Elliot Blackburn Date: Tue, 22 Aug 2023 13:52:19 +0100 Subject: [PATCH 205/207] Correct container image tagging --- .github/workflows/package-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index b34ad214..8a5f25ba 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -31,7 +31,7 @@ jobs: latest=true tags: | type=semver,pattern=v{{version}},event=tag - type=semver,pattern=v{{major.minor}},event=tag + type=semver,pattern=v{{major}}.{{minor}},event=tag type=semver,pattern=v{{major}},event=tag type=edge,branch=$repo.default_branch,event=push type=sha,branch=$repo.default_branch,event=push From 96d9040fb316c76a32a60c04030d9486342bafc4 Mon Sep 17 00:00:00 2001 From: Danil Titarenko <77471369+danilapog@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:23:55 +0300 Subject: [PATCH 206/207] Update package-and-publish-image.yml (#753) --- .github/workflows/package-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-and-publish-image.yml b/.github/workflows/package-and-publish-image.yml index 8a5f25ba..5675ac01 100644 --- a/.github/workflows/package-and-publish-image.yml +++ b/.github/workflows/package-and-publish-image.yml @@ -57,7 +57,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true - platforms: linux/amd64,linux/s390x + platforms: linux/amd64,linux/s390x,linux/arm64 tags: ${{ steps.meta.outputs.tags }} - name: Echo image digest From f7157b880631152569b04d626f631b60eec366f8 Mon Sep 17 00:00:00 2001 From: Zhang Jingqiang Date: Tue, 20 May 2025 15:22:42 +0800 Subject: [PATCH 207/207] add g3statsd to server implementations (#755) --- docs/server_implementations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/server_implementations.md b/docs/server_implementations.md index 64186e80..b1e0de70 100644 --- a/docs/server_implementations.md +++ b/docs/server_implementations.md @@ -4,6 +4,7 @@ The following is a list of projects that re-implement StatsD, if the the main pr * [brubeck](https://github.com/github/brubeck) - Server in C * [clj-statsd-svr](https://github.com/netmelody/clj-statsd-svr) — Clojure server +* [g3statsd](https://github.com/bytedance/g3/tree/master/g3statsd) - Server in Rust * [gographite](https://github.com/amir/gographite) — Server in Go * [gostatsd](https://github.com/atlassian/gostatsd) — Server in Go * [netdata](https://github.com/firehol/netdata) - Embedded StatsD server in the netdata server, in C, with visualization