diff --git a/Dockerfile b/Dockerfile index d49f24e..9696c0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,11 @@ RUN addgroup nonroot && \ adduser -D nonroot -G nonroot && \ chown nonroot:nonroot /app +RUN apk update && \ + apk add --no-cache iputils + USER nonroot + RUN mkdir -p /home/nonroot/.npm VOLUME /home/nonroot/.npm COPY package.json ./ diff --git a/Gruntfile.js b/Gruntfile.js index b0652e6..1249386 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -35,7 +35,8 @@ const mochaConfig = { const eslintConfig = { options: { - configFile: '.eslintrc.js' + configFile: '.eslintrc.js', + fix: true }, target: config.targets.ts } diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index c535854..dade20e 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,5 +1,6 @@ docker: image: stono/kconmon + tag: latest # Should we run an initContainer that enables core tcp connection setting tweaks enableTcpTweaks: true @@ -37,6 +38,26 @@ config: - www.google.com - kubernetes.default.svc.cluster.local +# ICMP Test configuration + icmp: + enable: true + interval: 5000 + count: 2 + timeout: 5 + hosts: + - www.telekom.de + - www.google.com + - 8.8.4.4 + +# Custom HTTP tests configuration + custom_http: + enable: true + interval: 5000 + timeout: 1000 + hosts: + - www.telekom.de + - www.google.de + resources: agent: # This will scale with the number of nodes in your cluster at loosely 1m per node diff --git a/lib/apps/agent/metrics.ts b/lib/apps/agent/metrics.ts index 54496b9..0e39b57 100644 --- a/lib/apps/agent/metrics.ts +++ b/lib/apps/agent/metrics.ts @@ -1,14 +1,23 @@ export interface IMetrics { handleTCPTestResult(result: ITCPTestResult) + handleCustomHTTPTestResult(result: ICustomHTTPTestResult) handleUDPTestResult(result: IUDPTestResult) handleDNSTestResult(result: IDNSTestResult) + handleICMPTestResult(result: IICMPTestResult) resetTCPTestResults() + resetCustomHTTPTestResults() resetUDPTestResults() toString() } import * as client from 'prom-client' -import { IUDPTestResult, IDNSTestResult, ITCPTestResult } from 'lib/tester' +import { + IICMPTestResult, + IUDPTestResult, + IDNSTestResult, + ITCPTestResult, + ICustomHTTPTestResult +} from 'lib/tester' import { IConfig } from 'lib/config' export default class Metrics implements IMetrics { @@ -24,6 +33,16 @@ export default class Metrics implements IMetrics { private DNS: client.Counter private DNSDuration: client.Gauge + private ICMP: client.Counter + private ICMPDuration: client.Gauge + private ICMPAverage: client.Gauge + private ICMPStddv: client.Gauge + private ICMPLoss: client.Gauge + + private CustomHTTP: client.Counter + private CustomHTTPDuration: client.Gauge + private CustomHTTPConnect: client.Gauge + constructor(config: IConfig) { client.register.clear() this.TCPConnect = new client.Gauge({ @@ -68,6 +87,36 @@ export default class Metrics implements IMetrics { name: `${config.metricsPrefix}_dns_duration_milliseconds` }) + this.ICMP = new client.Counter({ + help: 'ICMP Test Results', + labelNames: ['source', 'source_zone', 'host', 'result'], + name: `${config.metricsPrefix}_icmp_results_total` + }) + + this.ICMPDuration = new client.Gauge({ + help: 'Total time taken to complete the ICMP test', + labelNames: ['source', 'source_zone', 'host'], + name: `${config.metricsPrefix}_icmp_duration_milliseconds` + }) + + this.ICMPAverage = new client.Gauge({ + help: 'ICMP average packet RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_average_rtt_milliseconds` + }) + + this.ICMPStddv = new client.Gauge({ + help: 'ICMP standard deviation of RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_standard_deviation_rtt_milliseconds` + }) + + this.ICMPLoss = new client.Gauge({ + help: 'ICMP packet loss', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_packet_loss` + }) + this.UDP = new client.Counter({ help: 'UDP Test Results', labelNames: [ @@ -91,6 +140,63 @@ export default class Metrics implements IMetrics { ], name: `${config.metricsPrefix}_tcp_results_total` }) + + this.CustomHTTP = new client.Counter({ + help: 'Custom TCP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_custom_http_results_total` + }) + + this.CustomHTTPConnect = new client.Gauge({ + help: 'Time taken to establish the TCP socket for custom test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_http_connect_milliseconds` + }) + + this.CustomHTTPDuration = new client.Gauge({ + help: 'Total time taken to complete the custom TCP test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_http_duration_milliseconds` + }) + } + + public handleICMPTestResult(result: IICMPTestResult): void { + const source = result.source.nodeName + this.ICMP.labels( + source, + result.source.zone, + result.host, + result.result + ).inc(1) + this.ICMPDuration.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.duration) + + this.ICMPAverage.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.avg) + + this.ICMPStddv.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.stddev) + + this.ICMPLoss.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.loss) } public handleDNSTestResult(result: IDNSTestResult): void { @@ -183,6 +289,44 @@ export default class Metrics implements IMetrics { } } + public handleCustomHTTPTestResult(result: ICustomHTTPTestResult): void { + const source = result.source.nodeName + const destination = result.destination + const sourceZone = result.source.zone + const destinationZone = result.destination + this.CustomHTTP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.CustomHTTPConnect.labels( + source, + destination, + sourceZone, + destinationZone + ).set( + ((result.timings.connect || + result.timings.socket || + result.timings.start) - result.timings.start) as number + ) + this.CustomHTTPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.phases.total as number) + } + } + + public resetCustomHTTPTestResults() { + this.CustomHTTPConnect.reset() + this.CustomHTTPDuration.reset() + } + public toString(): string { return client.register.metrics() } diff --git a/lib/config/index.ts b/lib/config/index.ts index c4e0416..c1210f1 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -28,6 +28,19 @@ interface ITestConfiguration { interval: number hosts: string[] } + icmp: { + enable: boolean + interval: number + count: number + timeout: number + hosts: string[] + } + custom_http: { + enable: boolean + interval: number + timeout: number + hosts: string[] + } } export interface IConfig { @@ -60,7 +73,13 @@ export class Config implements IConfig { public readonly testConfig: ITestConfiguration = { tcp: getEnv('tcp', { interval: 5000, timeout: 1000 }), udp: getEnv('udp', { interval: 5000, timeout: 250, packets: 10 }), - dns: getEnv('dns', { interval: 5000, hosts: [] }) + dns: getEnv('dns', { interval: 5000, hosts: [] }), + icmp: getEnv('icmp', { interval: 5000, hosts: [] }), + custom_http: getEnv('custom_http', { + interval: 5000, + timeout: 1000, + hosts: [] + }) } } diff --git a/lib/tester/index.ts b/lib/tester/index.ts index 7a7e45f..d67935f 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -7,6 +7,7 @@ import { IConfig } from 'lib/config' import Logger, { ILogger } from 'lib/logger' import * as dns from 'dns' import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' +import * as ping from 'ping' export interface ITester { start() @@ -14,6 +15,8 @@ export interface ITester { runUDPTests(agents: IAgent[]): Promise runTCPTests(agents: IAgent[]): Promise runDNSTests(): Promise + runICMPTests(): Promise + runCustomHTTPTests(): Promise } interface ITestResult { @@ -29,6 +32,16 @@ export interface IDNSTestResult { result: 'pass' | 'fail' } +export interface IICMPTestResult { + source: IAgent + host: string + duration: number + avg: number + stddev: number + loss: number + result: 'pass' | 'fail' +} + export interface IUDPTestResult extends ITestResult { timings?: IUDPPingResult } @@ -37,6 +50,13 @@ export interface ITCPTestResult extends ITestResult { timings?: PlainResponse['timings'] } +export interface ICustomHTTPTestResult { + source: IAgent + destination: string + result: 'pass' | 'fail' + timings?: PlainResponse['timings'] +} + export default class Tester implements ITester { private got: Got private discovery: IDiscovery @@ -46,6 +66,7 @@ export default class Tester implements ITester { private running = false private config: IConfig private resolver = new dns.promises.Resolver() + // private ping: pingman private readonly udpClientFactory: IUDPClientFactory constructor( @@ -102,16 +123,95 @@ export default class Tester implements ITester { await delay(this.config.testConfig.dns.interval + jitter()) } } + const icmpEventLoop = async () => { + while (this.running) { + await this.runICMPTests() + await delay(this.config.testConfig.icmp.interval + jitter()) + } + } + const httpCustomEventLoop = async () => { + while (this.running) { + this.metrics.resetCustomHTTPTestResults() + await this.runCustomHTTPTests() + await delay(this.config.testConfig.custom_http.interval + jitter()) + } + } + agentUpdateLoop() tcpEventLoop() udpEventLoop() dnsEventLoop() + if (this.config.testConfig.icmp.enable) { + icmpEventLoop() + } + if (this.config.testConfig.custom_http.enable) { + httpCustomEventLoop() + } } public async stop(): Promise { this.running = false } + public async runICMPTests(): Promise { + const promises = this.config.testConfig.icmp.hosts.map( + async (host): Promise => { + const hrstart = process.hrtime() + try { + const result = await ping.promise.probe(host, { + timeout: this.config.testConfig.icmp.timeout, + extra: ['-c', this.config.testConfig.icmp.count] + }) + const hrend = process.hrtime(hrstart) + + if (result.alive) { + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: parseFloat(result.avg), + stddev: parseFloat(result.stddev), + loss: parseFloat(result.packetLoss), + result: 'pass' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } else { + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: parseFloat(result.packetLoss), + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } catch (ex) { + this.logger.error(`icmp test for ${host} failed`, ex) + const hrend = process.hrtime(hrstart) + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: 100.0, + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } + ) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + public async runDNSTests(): Promise { const promises = this.config.testConfig.dns.hosts.map( async (host): Promise => { @@ -225,4 +325,58 @@ export default class Tester implements ITester { .filter((r) => r.status === 'fulfilled') .map((i) => (i as PromiseFulfilledResult).value) } + + public async runCustomHTTPTests(): Promise { + const promises = this.config.testConfig.custom_http.hosts.map( + async (host): Promise => { + try { + const url = `http://${host}` + const result = await this.got(url, { + timeout: this.config.testConfig.custom_http.timeout + }) + const htmlReponseCodes = [200, 301, 302, 304, 401] + if (htmlReponseCodes.includes(result.statusCode)) { + const mappedResult: ICustomHTTPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'pass' + } + this.metrics.handleCustomHTTPTestResult(mappedResult) + return mappedResult + } else { + const mappedResult: ICustomHTTPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'fail' + } + this.metrics.handleCustomHTTPTestResult(mappedResult) + return mappedResult + } + } catch (ex) { + this.logger.warn( + `test failed`, + { + source: this.me, + destination: host + }, + ex + ) + const failResult: ICustomHTTPTestResult = { + source: this.me, + destination: host, + result: 'fail' + } + this.metrics.handleCustomHTTPTestResult(failResult) + return failResult + } + } + ) + + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } } diff --git a/package.json b/package.json index 29170e8..fda9206 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "tsconfig-paths": "3.9.0", "kubernetes-client": "9.0.0", "kubernetes-types": "1.17.0-beta.1", - "ts-xor": "1.0.8" + "ts-xor": "1.0.8", + "ping": "0.2.3" }, "devDependencies": { "@types/express": "4.17.6", diff --git a/test/tester.test.ts b/test/tester.test.ts index 3b58752..66e2096 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -19,6 +19,11 @@ describe('Tester', () => { config.testConfig.udp.timeout = 500 config.testConfig.udp.packets = 1 config.testConfig.tcp.timeout = 500 + config.testConfig.custom_http.enable = true + config.testConfig.custom_http.timeout = 500 + config.testConfig.icmp.enable = true + config.testConfig.icmp.count = 2 + config.testConfig.icmp.timeout = 5 config.port = 8080 }) @@ -31,6 +36,21 @@ describe('Tester', () => { sut = new Tester(config, got, discovery, metrics, me, udpClientFactory) }) + it('should do a icmp test', async () => { + config.testConfig.icmp.hosts = ['www.google.com'] + const result = await sut.runICMPTests() + should(result[0].result).eql('pass') + }) + + it('should do a custom http test', async () => { + config.testConfig.custom_http.hosts = ['www.google.com'] + td.when(got('http://www.google.com', { timeout: 500 })).thenResolve({ + statusCode: 200 + }) + const result = await sut.runCustomHTTPTests() + should(result[0].result).eql('pass') + }) + it('should do a dns test', async () => { config.testConfig.dns.hosts = ['www.google.com'] const result = await sut.runDNSTests()