From b0d0d608604d6e88059d2a23fcd0ccf7d6d13abf Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Wed, 8 Jul 2020 13:41:01 +0000 Subject: [PATCH 1/9] Adding ICMP check for custom hosts Adding custom TCP check for custom hosts Adding mocha tests for the above --- Dockerfile | 4 + Gruntfile.js | 3 +- helmfile/charts/kconmon/values.yaml | 20 +- lib/apps/agent/metrics.ts | 514 +++++++++++++++--------- lib/config/index.ts | 15 +- lib/tester/index.ts | 595 +++++++++++++++++----------- package.json | 3 +- test/tester.test.ts | 15 + 8 files changed, 747 insertions(+), 422 deletions(-) diff --git a/Dockerfile b/Dockerfile index d49f24e..9696c0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,11 @@ RUN addgroup nonroot && \ adduser -D nonroot -G nonroot && \ chown nonroot:nonroot /app +RUN apk update && \ + apk add --no-cache iputils + USER nonroot + RUN mkdir -p /home/nonroot/.npm VOLUME /home/nonroot/.npm COPY package.json ./ diff --git a/Gruntfile.js b/Gruntfile.js index b0652e6..1249386 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -35,7 +35,8 @@ const mochaConfig = { const eslintConfig = { options: { - configFile: '.eslintrc.js' + configFile: '.eslintrc.js', + fix: true }, target: config.targets.ts } diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index c535854..d20d226 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,8 +1,9 @@ docker: - image: stono/kconmon + image: barto92/kubemon + tag: 1.0.30 # Should we run an initContainer that enables core tcp connection setting tweaks -enableTcpTweaks: true +enableTcpTweaks: false config: # What port should the server listen on @@ -36,6 +37,21 @@ config: hosts: - www.google.com - kubernetes.default.svc.cluster.local + + icmp: + interval: 5000 + count: 2 + timeout: 5 + hosts: + - www.telekom.de + - www.google.com + - 8.8.4.4 + custom_tcp: + interval: 5000 + timeout: 1000 + hosts: + - www.telekom.de + - www.google.de resources: agent: diff --git a/lib/apps/agent/metrics.ts b/lib/apps/agent/metrics.ts index 54496b9..894b067 100644 --- a/lib/apps/agent/metrics.ts +++ b/lib/apps/agent/metrics.ts @@ -1,189 +1,325 @@ -export interface IMetrics { - handleTCPTestResult(result: ITCPTestResult) - handleUDPTestResult(result: IUDPTestResult) - handleDNSTestResult(result: IDNSTestResult) - resetTCPTestResults() - resetUDPTestResults() - toString() -} - -import * as client from 'prom-client' -import { IUDPTestResult, IDNSTestResult, ITCPTestResult } from 'lib/tester' -import { IConfig } from 'lib/config' - -export default class Metrics implements IMetrics { - private TCP: client.Counter - private TCPDuration: client.Gauge - private TCPConnect: client.Gauge - - private UDP: client.Counter - private UDPDuration: client.Gauge - private UDPVariance: client.Gauge - private UDPLoss: client.Gauge - - private DNS: client.Counter - private DNSDuration: client.Gauge - - constructor(config: IConfig) { - client.register.clear() - this.TCPConnect = new client.Gauge({ - help: 'Time taken to establish the TCP socket', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_tcp_connect_milliseconds` - }) - - this.TCPDuration = new client.Gauge({ - help: 'Total time taken to complete the TCP test', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_tcp_duration_milliseconds` - }) - - this.UDPDuration = new client.Gauge({ - help: 'Average duration per packet', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_duration_milliseconds` - }) - - this.UDPVariance = new client.Gauge({ - help: 'UDP variance between the slowest and fastest packet', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_duration_variance_milliseconds` - }) - - this.UDPLoss = new client.Gauge({ - help: 'UDP packet loss', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_loss` - }) - - this.DNS = new client.Counter({ - help: 'DNS Test Results', - labelNames: ['source', 'source_zone', 'host', 'result'], - name: `${config.metricsPrefix}_dns_results_total` - }) - - this.DNSDuration = new client.Gauge({ - help: 'Total time taken to complete the DNS test', - labelNames: ['source', 'source_zone', 'host'], - name: `${config.metricsPrefix}_dns_duration_milliseconds` - }) - - this.UDP = new client.Counter({ - help: 'UDP Test Results', - labelNames: [ - 'source', - 'destination', - 'source_zone', - 'destination_zone', - 'result' - ], - name: `${config.metricsPrefix}_udp_results_total` - }) - - this.TCP = new client.Counter({ - help: 'TCP Test Results', - labelNames: [ - 'source', - 'destination', - 'source_zone', - 'destination_zone', - 'result' - ], - name: `${config.metricsPrefix}_tcp_results_total` - }) - } - - public handleDNSTestResult(result: IDNSTestResult): void { - const source = result.source.nodeName - this.DNS.labels(source, result.source.zone, result.host, result.result).inc( - 1 - ) - this.DNSDuration.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.duration) - } - - public resetTCPTestResults() { - this.TCPConnect.reset() - this.TCPDuration.reset() - } - - public resetUDPTestResults() { - this.UDPDuration.reset() - this.UDPLoss.reset() - this.UDPVariance.reset() - } - - public handleUDPTestResult(result: IUDPTestResult): void { - const source = result.source.nodeName - const destination = result.destination.nodeName - const sourceZone = result.source.zone - const destinationZone = result.destination.zone - this.UDP.labels( - source, - destination, - sourceZone, - destinationZone, - result.result - ).inc(1) - - if (result.timings) { - this.UDPDuration.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.average) - - this.UDPVariance.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.variance) - - this.UDPLoss.labels(source, destination, sourceZone, destinationZone).set( - result.timings.loss - ) - } - } - - public handleTCPTestResult(result: ITCPTestResult): void { - const source = result.source.nodeName - const destination = result.destination.nodeName - const sourceZone = result.source.zone - const destinationZone = result.destination.zone - this.TCP.labels( - source, - destination, - sourceZone, - destinationZone, - result.result - ).inc(1) - - if (result.timings) { - this.TCPConnect.labels( - source, - destination, - sourceZone, - destinationZone - ).set( - ((result.timings.connect || - result.timings.socket || - result.timings.start) - result.timings.start) as number - ) - this.TCPDuration.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.phases.total as number) - } - } - - public toString(): string { - return client.register.metrics() - } -} +export interface IMetrics { + handleTCPTestResult(result: ITCPTestResult) + handleCustomTCPTestResult(result: ICustomTCPTestResult) + handleUDPTestResult(result: IUDPTestResult) + handleDNSTestResult(result: IDNSTestResult) + handleICMPTestResult(result: IICMPTestResult) + resetTCPTestResults() + resetCustomTCPTestResults() + resetUDPTestResults() + toString() +} + +import * as client from 'prom-client' +import { IICMPTestResult, IUDPTestResult, IDNSTestResult, ITCPTestResult, ICustomTCPTestResult } from 'lib/tester' +import { IConfig } from 'lib/config' + +export default class Metrics implements IMetrics { + private TCP: client.Counter + private TCPDuration: client.Gauge + private TCPConnect: client.Gauge + + private UDP: client.Counter + private UDPDuration: client.Gauge + private UDPVariance: client.Gauge + private UDPLoss: client.Gauge + + private DNS: client.Counter + private DNSDuration: client.Gauge + + private ICMP: client.Counter + private ICMPDuration: client.Gauge + private ICMPAverage: client.Gauge + private ICMPStddv: client.Gauge + private ICMPLoss: client.Gauge + + private CustomTCP: client.Counter + private CustomTCPDuration: client.Gauge + private CustomTCPConnect: client.Gauge + + constructor(config: IConfig) { + client.register.clear() + this.TCPConnect = new client.Gauge({ + help: 'Time taken to establish the TCP socket', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_tcp_connect_milliseconds` + }) + + this.TCPDuration = new client.Gauge({ + help: 'Total time taken to complete the TCP test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_tcp_duration_milliseconds` + }) + + this.UDPDuration = new client.Gauge({ + help: 'Average duration per packet', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_duration_milliseconds` + }) + + this.UDPVariance = new client.Gauge({ + help: 'UDP variance between the slowest and fastest packet', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_duration_variance_milliseconds` + }) + + this.UDPLoss = new client.Gauge({ + help: 'UDP packet loss', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_loss` + }) + + this.DNS = new client.Counter({ + help: 'DNS Test Results', + labelNames: ['source', 'source_zone', 'host', 'result'], + name: `${config.metricsPrefix}_dns_results_total` + }) + + this.DNSDuration = new client.Gauge({ + help: 'Total time taken to complete the DNS test', + labelNames: ['source', 'source_zone', 'host'], + name: `${config.metricsPrefix}_dns_duration_milliseconds` + }) + + this.ICMP = new client.Counter({ + help: 'ICMP Test Results', + labelNames: ['source', 'source_zone', 'host', 'result'], + name: `${config.metricsPrefix}_icmp_results_total` + }) + + this.ICMPDuration = new client.Gauge({ + help: 'Total time taken to complete the ICMP test', + labelNames: ['source', 'source_zone', 'host'], + name: `${config.metricsPrefix}_icmp_duration_milliseconds` + }) + + this.ICMPAverage = new client.Gauge({ + help: 'ICMP average packet RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_average_rtt_milliseconds` + }) + + this.ICMPStddv = new client.Gauge({ + help: 'ICMP standard deviation of RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_standard_deviation_rtt_milliseconds` + }) + + this.ICMPLoss = new client.Gauge({ + help: 'ICMP packet loss', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_packet_loss` + }) + + this.UDP = new client.Counter({ + help: 'UDP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_udp_results_total` + }) + + this.TCP = new client.Counter({ + help: 'TCP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_tcp_results_total` + }) + + this.CustomTCP = new client.Counter({ + help: 'Custom TCP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_custom_tcp_results_total` + }) + + this.CustomTCPConnect = new client.Gauge({ + help: 'Time taken to establish the TCP socket for custom test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_tcp_connect_milliseconds` + }) + + this.CustomTCPDuration = new client.Gauge({ + help: 'Total time taken to complete the custom TCP test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_tcp_duration_milliseconds` + }) + + } + + public handleICMPTestResult(result: IICMPTestResult): void { + const source = result.source.nodeName + this.ICMP.labels(source, result.source.zone, result.host, result.result).inc( + 1 + ) + this.ICMPDuration.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.duration) + + this.ICMPAverage.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.avg) + + this.ICMPStddv.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.stddev) + + this.ICMPLoss.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.loss) + } + + public handleDNSTestResult(result: IDNSTestResult): void { + const source = result.source.nodeName + this.DNS.labels(source, result.source.zone, result.host, result.result).inc( + 1 + ) + this.DNSDuration.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.duration) + } + + public resetTCPTestResults() { + this.TCPConnect.reset() + this.TCPDuration.reset() + } + + public resetUDPTestResults() { + this.UDPDuration.reset() + this.UDPLoss.reset() + this.UDPVariance.reset() + } + + public handleUDPTestResult(result: IUDPTestResult): void { + const source = result.source.nodeName + const destination = result.destination.nodeName + const sourceZone = result.source.zone + const destinationZone = result.destination.zone + this.UDP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.UDPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.average) + + this.UDPVariance.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.variance) + + this.UDPLoss.labels(source, destination, sourceZone, destinationZone).set( + result.timings.loss + ) + } + } + + public handleTCPTestResult(result: ITCPTestResult): void { + const source = result.source.nodeName + const destination = result.destination.nodeName + const sourceZone = result.source.zone + const destinationZone = result.destination.zone + this.TCP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.TCPConnect.labels( + source, + destination, + sourceZone, + destinationZone + ).set( + ((result.timings.connect || + result.timings.socket || + result.timings.start) - result.timings.start) as number + ) + this.TCPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.phases.total as number) + } + } + + public handleCustomTCPTestResult(result: ICustomTCPTestResult): void { + const source = result.source.nodeName + const destination = result.destination + const sourceZone = result.source.zone + const destinationZone = result.destination + this.CustomTCP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.CustomTCPConnect.labels( + source, + destination, + sourceZone, + destinationZone + ).set( + ((result.timings.connect || + result.timings.socket || + result.timings.start) - result.timings.start) as number + ) + this.CustomTCPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.phases.total as number) + } + } + + public resetCustomTCPTestResults() { + this.CustomTCPConnect.reset() + this.CustomTCPDuration.reset() + } + + public toString(): string { + return client.register.metrics() + } +} diff --git a/lib/config/index.ts b/lib/config/index.ts index c4e0416..8fa2f6e 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -28,6 +28,17 @@ interface ITestConfiguration { interval: number hosts: string[] } + icmp: { + interval: number + count: number + timeout: number + hosts: string[] + } + custom_tcp: { + interval: number + timeout: number + hosts: string[] + } } export interface IConfig { @@ -60,7 +71,9 @@ export class Config implements IConfig { public readonly testConfig: ITestConfiguration = { tcp: getEnv('tcp', { interval: 5000, timeout: 1000 }), udp: getEnv('udp', { interval: 5000, timeout: 250, packets: 10 }), - dns: getEnv('dns', { interval: 5000, hosts: [] }) + dns: getEnv('dns', { interval: 5000, hosts: [] }), + icmp: getEnv('icmp', { interval: 5000, hosts: [] }), + custom_tcp: getEnv('custom_tcp', { interval: 5000, timeout: 1000, hosts: [] }) } } diff --git a/lib/tester/index.ts b/lib/tester/index.ts index 7a7e45f..21a15b2 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -1,228 +1,367 @@ -import { Got } from 'got/dist/source' -import { IDiscovery, IAgent } from 'lib/discovery' -import { PlainResponse } from 'got/dist/source/core' -import { IMetrics } from 'lib/apps/agent/metrics' -import { IUDPPingResult } from 'lib/udp/client' -import { IConfig } from 'lib/config' -import Logger, { ILogger } from 'lib/logger' -import * as dns from 'dns' -import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' - -export interface ITester { - start() - stop() - runUDPTests(agents: IAgent[]): Promise - runTCPTests(agents: IAgent[]): Promise - runDNSTests(): Promise -} - -interface ITestResult { - source: IAgent - destination: IAgent - result: 'pass' | 'fail' -} - -export interface IDNSTestResult { - source: IAgent - host: string - duration: number - result: 'pass' | 'fail' -} - -export interface IUDPTestResult extends ITestResult { - timings?: IUDPPingResult -} - -export interface ITCPTestResult extends ITestResult { - timings?: PlainResponse['timings'] -} - -export default class Tester implements ITester { - private got: Got - private discovery: IDiscovery - private logger: ILogger = new Logger('tester') - private metrics: IMetrics - private me: IAgent - private running = false - private config: IConfig - private resolver = new dns.promises.Resolver() - private readonly udpClientFactory: IUDPClientFactory - - constructor( - config: IConfig, - got: Got, - discovery: IDiscovery, - metrics: IMetrics, - me: IAgent, - udpClientFactory: IUDPClientFactory - ) { - this.got = got - this.discovery = discovery - this.metrics = metrics - this.me = me - this.config = config - this.udpClientFactory = udpClientFactory - } - - public async start(): Promise { - const delay = (ms: number) => { - return new Promise((resolve) => setTimeout(resolve, ms)) - } - - const jitter = () => { - const rand = Math.random() * (500 - 50) - return Math.floor(rand + 100) - } - - this.running = true - let agents = await this.discovery.agents() - const agentUpdateLoop = async () => { - while (this.running) { - agents = await this.discovery.agents() - await delay(5000) - } - } - const tcpEventLoop = async () => { - while (this.running) { - this.metrics.resetTCPTestResults() - await this.runTCPTests(agents) - await delay(this.config.testConfig.tcp.interval + jitter()) - } - } - const udpEventLoop = async () => { - while (this.running) { - this.metrics.resetUDPTestResults() - await this.runUDPTests(agents) - await delay(this.config.testConfig.udp.interval + jitter()) - } - } - const dnsEventLoop = async () => { - while (this.running) { - await this.runDNSTests() - await delay(this.config.testConfig.dns.interval + jitter()) - } - } - agentUpdateLoop() - tcpEventLoop() - udpEventLoop() - dnsEventLoop() - } - - public async stop(): Promise { - this.running = false - } - - public async runDNSTests(): Promise { - const promises = this.config.testConfig.dns.hosts.map( - async (host): Promise => { - const hrstart = process.hrtime() - try { - const result = await this.resolver.resolve4(host) - const hrend = process.hrtime(hrstart) - const mapped: IDNSTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - result: result && result.length > 0 ? 'pass' : 'fail' - } - this.metrics.handleDNSTestResult(mapped) - return mapped - } catch (ex) { - this.logger.error(`dns test for ${host} failed`, ex) - const hrend = process.hrtime(hrstart) - const mapped: IDNSTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - result: 'fail' - } - this.metrics.handleDNSTestResult(mapped) - return mapped - } - } - ) - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } - - public async runUDPTests(agents: IAgent[]): Promise { - const results: IUDPTestResult[] = [] - this.udpClientFactory.generateClientsForAgents(agents) - - const testAgent = async (agent: IAgent): Promise => { - const client = this.udpClientFactory.clientFor(agent) - try { - const result = await client.ping( - this.config.testConfig.udp.timeout, - this.config.testConfig.udp.packets - ) - if (result.loss > 0) { - this.logger.warn('packet loss detected', result) - } - const testResult: IUDPTestResult = { - source: this.me, - destination: agent, - timings: result, - result: result.loss > 0 ? 'fail' : 'pass' - } - results.push(testResult) - this.metrics.handleUDPTestResult(testResult) - } catch (ex) { - this.logger.error('Failed to execute UDP test', ex) - const testResult: IUDPTestResult = { - source: this.me, - destination: agent, - result: 'fail' - } - results.push(testResult) - this.metrics.handleUDPTestResult(testResult) - } - } - const promises = agents.map(testAgent) - await Promise.allSettled(promises) - - return results - } - - public async runTCPTests(agents: IAgent[]): Promise { - const testAgent = async (agent: IAgent): Promise => { - try { - const url = `http://${agent.ip}:${this.config.port}/readiness` - const result = await this.got(url, { - timeout: this.config.testConfig.tcp.timeout - }) - const mappedResult: ITCPTestResult = { - source: this.me, - destination: agent, - timings: result.timings, - result: result.statusCode === 200 ? 'pass' : 'fail' - } - this.metrics.handleTCPTestResult(mappedResult) - return mappedResult - } catch (ex) { - this.logger.warn( - `test failed`, - { - source: this.me, - destination: agent - }, - ex - ) - const failResult: ITCPTestResult = { - source: this.me, - destination: agent, - result: 'fail' - } - this.metrics.handleTCPTestResult(failResult) - return failResult - } - } - const promises = agents.map(testAgent) - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } -} +import { Got } from 'got/dist/source' +import { IDiscovery, IAgent } from 'lib/discovery' +import { PlainResponse } from 'got/dist/source/core' +import { IMetrics } from 'lib/apps/agent/metrics' +import { IUDPPingResult } from 'lib/udp/client' +import { IConfig } from 'lib/config' +import Logger, { ILogger } from 'lib/logger' +import * as dns from 'dns' +import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' +import * as ping from 'ping' + +export interface ITester { + start() + stop() + runUDPTests(agents: IAgent[]): Promise + runTCPTests(agents: IAgent[]): Promise + runDNSTests(): Promise + runICMPTests(): Promise + runCustomTCPTests(): Promise +} + +interface ITestResult { + source: IAgent + destination: IAgent + result: 'pass' | 'fail' +} + +export interface IDNSTestResult { + source: IAgent + host: string + duration: number + result: 'pass' | 'fail' +} + +export interface IICMPTestResult { + source: IAgent + host: string + duration: number, + avg: number, + stddev: number, + loss: number, + result: 'pass' | 'fail' +} + +export interface IUDPTestResult extends ITestResult { + timings?: IUDPPingResult +} + +export interface ITCPTestResult extends ITestResult { + timings?: PlainResponse['timings'] +} + +export interface ICustomTCPTestResult { + source: IAgent + destination: string + result: 'pass' | 'fail' + timings?: PlainResponse['timings'] +} + +export default class Tester implements ITester { + private got: Got + private discovery: IDiscovery + private logger: ILogger = new Logger('tester') + private metrics: IMetrics + private me: IAgent + private running = false + private config: IConfig + private resolver = new dns.promises.Resolver() + // private ping: pingman + private readonly udpClientFactory: IUDPClientFactory + + constructor( + config: IConfig, + got: Got, + discovery: IDiscovery, + metrics: IMetrics, + me: IAgent, + udpClientFactory: IUDPClientFactory + ) { + this.got = got + this.discovery = discovery + this.metrics = metrics + this.me = me + this.config = config + this.udpClientFactory = udpClientFactory + } + + public async start(): Promise { + const delay = (ms: number) => { + return new Promise((resolve) => setTimeout(resolve, ms)) + } + + const jitter = () => { + const rand = Math.random() * (500 - 50) + return Math.floor(rand + 100) + } + + this.running = true + let agents = await this.discovery.agents() + const agentUpdateLoop = async () => { + while (this.running) { + agents = await this.discovery.agents() + await delay(5000) + } + } + const tcpEventLoop = async () => { + while (this.running) { + this.metrics.resetTCPTestResults() + await this.runTCPTests(agents) + await delay(this.config.testConfig.tcp.interval + jitter()) + } + } + const udpEventLoop = async () => { + while (this.running) { + this.metrics.resetUDPTestResults() + await this.runUDPTests(agents) + await delay(this.config.testConfig.udp.interval + jitter()) + } + } + const dnsEventLoop = async () => { + while (this.running) { + await this.runDNSTests() + await delay(this.config.testConfig.dns.interval + jitter()) + } + } + const icmpEventLoop = async () => { + while (this.running) { + await this.runICMPTests() + await delay(this.config.testConfig.icmp.interval + jitter()) + } + } + const tcpCustomEventLoop = async () => { + while (this.running) { + this.metrics.resetCustomTCPTestResults() + await this.runCustomTCPTests() + await delay(this.config.testConfig.custom_tcp.interval + jitter()) + } + } + + agentUpdateLoop() + tcpEventLoop() + udpEventLoop() + dnsEventLoop() + icmpEventLoop() + tcpCustomEventLoop() + } + + public async stop(): Promise { + this.running = false + } + + public async runICMPTests(): Promise { + const promises = this.config.testConfig.icmp.hosts.map( + async (host): Promise => { + const hrstart = process.hrtime() + try { + const result = await ping.promise.probe(host, { + timeout: this.config.testConfig.icmp.timeout, + extra: ['-c', this.config.testConfig.icmp.count], + }); + const hrend = process.hrtime(hrstart) + + if(result.alive){ + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: parseFloat(result.avg), + stddev: parseFloat(result.stddev), + loss: parseFloat(result.packetLoss), + result: 'pass' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } else { + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: parseFloat(result.packetLoss), + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } catch (ex) { + this.logger.error(`icmp test for ${host} failed`, ex) + const hrend = process.hrtime(hrstart) + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: 100.000, + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } + ) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + + public async runDNSTests(): Promise { + const promises = this.config.testConfig.dns.hosts.map( + async (host): Promise => { + const hrstart = process.hrtime() + try { + const result = await this.resolver.resolve4(host) + const hrend = process.hrtime(hrstart) + const mapped: IDNSTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + result: result && result.length > 0 ? 'pass' : 'fail' + } + this.metrics.handleDNSTestResult(mapped) + return mapped + } catch (ex) { + this.logger.error(`dns test for ${host} failed`, ex) + const hrend = process.hrtime(hrstart) + const mapped: IDNSTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + result: 'fail' + } + this.metrics.handleDNSTestResult(mapped) + return mapped + } + } + ) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + public async runUDPTests(agents: IAgent[]): Promise { + const results: IUDPTestResult[] = [] + this.udpClientFactory.generateClientsForAgents(agents) + + const testAgent = async (agent: IAgent): Promise => { + const client = this.udpClientFactory.clientFor(agent) + try { + const result = await client.ping( + this.config.testConfig.udp.timeout, + this.config.testConfig.udp.packets + ) + if (result.loss > 0) { + this.logger.warn('packet loss detected', result) + } + const testResult: IUDPTestResult = { + source: this.me, + destination: agent, + timings: result, + result: result.loss > 0 ? 'fail' : 'pass' + } + results.push(testResult) + this.metrics.handleUDPTestResult(testResult) + } catch (ex) { + this.logger.error('Failed to execute UDP test', ex) + const testResult: IUDPTestResult = { + source: this.me, + destination: agent, + result: 'fail' + } + results.push(testResult) + this.metrics.handleUDPTestResult(testResult) + } + } + const promises = agents.map(testAgent) + await Promise.allSettled(promises) + + return results + } + + public async runTCPTests(agents: IAgent[]): Promise { + const testAgent = async (agent: IAgent): Promise => { + try { + const url = `http://${agent.ip}:${this.config.port}/readiness` + const result = await this.got(url, { + timeout: this.config.testConfig.tcp.timeout + }) + const mappedResult: ITCPTestResult = { + source: this.me, + destination: agent, + timings: result.timings, + result: result.statusCode === 200 ? 'pass' : 'fail' + } + this.metrics.handleTCPTestResult(mappedResult) + return mappedResult + } catch (ex) { + this.logger.warn( + `test failed`, + { + source: this.me, + destination: agent + }, + ex + ) + const failResult: ITCPTestResult = { + source: this.me, + destination: agent, + result: 'fail' + } + this.metrics.handleTCPTestResult(failResult) + return failResult + } + } + const promises = agents.map(testAgent) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + public async runCustomTCPTests(): Promise { + const promises = this.config.testConfig.custom_tcp.hosts.map( + async (host): Promise => { + try { + const url = `http://${host}` + const result = await this.got(url, { + timeout: this.config.testConfig.custom_tcp.timeout + }) + const mappedResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: result.statusCode === 200 ? 'pass' : 'fail' + } + this.metrics.handleCustomTCPTestResult(mappedResult) + return mappedResult + } catch (ex) { + this.logger.warn( + `test failed`, + { + source: this.me, + destination: host + }, + ex + ) + const failResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + result: 'fail' + } + this.metrics.handleCustomTCPTestResult(failResult) + return failResult + } + } + ) + + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } +} diff --git a/package.json b/package.json index 29170e8..fda9206 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "tsconfig-paths": "3.9.0", "kubernetes-client": "9.0.0", "kubernetes-types": "1.17.0-beta.1", - "ts-xor": "1.0.8" + "ts-xor": "1.0.8", + "ping": "0.2.3" }, "devDependencies": { "@types/express": "4.17.6", diff --git a/test/tester.test.ts b/test/tester.test.ts index 3b58752..b210711 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -19,6 +19,9 @@ describe('Tester', () => { config.testConfig.udp.timeout = 500 config.testConfig.udp.packets = 1 config.testConfig.tcp.timeout = 500 + config.testConfig.custom_tcp.timeout = 500 + config.testConfig.icmp.count = 2 + config.testConfig.icmp.timeout = 5 config.port = 8080 }) @@ -31,6 +34,18 @@ describe('Tester', () => { sut = new Tester(config, got, discovery, metrics, me, udpClientFactory) }) + it('should do a icmp test', async () => { + config.testConfig.icmp.hosts = ['www.google.com'] + const result = await sut.runICMPTests() + should(result[0].result).eql('pass') + }) + + it('should do a custom tcp test', async () => { + config.testConfig.custom_tcp.hosts = ['www.google.com'] + const result = await sut.runICMPTests() + should(result[0].result).eql('pass') + }) + it('should do a dns test', async () => { config.testConfig.dns.hosts = ['www.google.com'] const result = await sut.runDNSTests() From a1532ec3af0fc8581d613b85ccddfa9eb1d4d8fc Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Wed, 8 Jul 2020 13:42:34 +0000 Subject: [PATCH 2/9] Updating image tag --- helmfile/charts/kconmon/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index d20d226..b0b196f 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,6 +1,6 @@ docker: image: barto92/kubemon - tag: 1.0.30 + tag: 1.0.40 # Should we run an initContainer that enables core tcp connection setting tweaks enableTcpTweaks: false From ef7284024010122b9f10e304fbd4d246483ff582 Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Thu, 9 Jul 2020 06:22:45 +0000 Subject: [PATCH 3/9] Fix test to test the custom TCP and not ICMP Adding enable value at ICMP and cutstomTCP --- docker-compose.yml | 4 ++-- helmfile/charts/kconmon/values.yaml | 2 +- lib/config/index.ts | 2 ++ lib/tester/index.ts | 8 ++++++-- test/tester.test.ts | 4 +++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index adab234..434d2f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.2' services: controller: - image: stono/kconmon:${VERSION:-latest} + image: barto92/kubemon:${VERSION:-latest} build: context: '.' args: @@ -24,7 +24,7 @@ services: - ~/.kube:/home/nonroot/.kube:ro agent: - image: stono/kconmon:${VERSION:-latest} + image: barto92/kubemon:${VERSION:-latest} command: 'agent' environment: PORT: '80' # Run on port 80 locally in docker-compose to replicate kubernetes service object port magenting diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index b0b196f..3914319 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,6 +1,6 @@ docker: image: barto92/kubemon - tag: 1.0.40 + tag: 1.0.41 # Should we run an initContainer that enables core tcp connection setting tweaks enableTcpTweaks: false diff --git a/lib/config/index.ts b/lib/config/index.ts index 8fa2f6e..2e2e4da 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -29,12 +29,14 @@ interface ITestConfiguration { hosts: string[] } icmp: { + enable: boolean interval: number count: number timeout: number hosts: string[] } custom_tcp: { + enable: boolean interval: number timeout: number hosts: string[] diff --git a/lib/tester/index.ts b/lib/tester/index.ts index 21a15b2..6d2eab2 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -141,8 +141,12 @@ export default class Tester implements ITester { tcpEventLoop() udpEventLoop() dnsEventLoop() - icmpEventLoop() - tcpCustomEventLoop() + if(this.config.testConfig.icmp.enable){ + icmpEventLoop() + } + if(this.config.testConfig.custom_tcp.enable){ + tcpCustomEventLoop() + } } public async stop(): Promise { diff --git a/test/tester.test.ts b/test/tester.test.ts index b210711..c2cd042 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -19,7 +19,9 @@ describe('Tester', () => { config.testConfig.udp.timeout = 500 config.testConfig.udp.packets = 1 config.testConfig.tcp.timeout = 500 + config.testConfig.custom_tcp.enable = true config.testConfig.custom_tcp.timeout = 500 + config.testConfig.icmp.enable = true config.testConfig.icmp.count = 2 config.testConfig.icmp.timeout = 5 config.port = 8080 @@ -42,7 +44,7 @@ describe('Tester', () => { it('should do a custom tcp test', async () => { config.testConfig.custom_tcp.hosts = ['www.google.com'] - const result = await sut.runICMPTests() + const result = await sut.runCustomTCPTests() should(result[0].result).eql('pass') }) From d5538eae6b2f3545d7acfff28adefbc11cb2bb1e Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Thu, 9 Jul 2020 06:43:01 +0000 Subject: [PATCH 4/9] Adding more acceptable HTML response codes --- lib/tester/index.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/tester/index.ts b/lib/tester/index.ts index 6d2eab2..ce02586 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -335,14 +335,27 @@ export default class Tester implements ITester { const result = await this.got(url, { timeout: this.config.testConfig.custom_tcp.timeout }) - const mappedResult: ICustomTCPTestResult = { - source: this.me, - destination: host, - timings: result.timings, - result: result.statusCode === 200 ? 'pass' : 'fail' + const htmlReponseCodes = [200, 301, 302, 304, 401] + if(htmlReponseCodes.includes(result.statusCode)){ + const mappedResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'pass' + } + this.metrics.handleCustomTCPTestResult(mappedResult) + return mappedResult + } else { + const mappedResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'fail' + } + this.metrics.handleCustomTCPTestResult(mappedResult) + return mappedResult } - this.metrics.handleCustomTCPTestResult(mappedResult) - return mappedResult + } catch (ex) { this.logger.warn( `test failed`, From 2f29d92d9de801a8a67e068dfa5bf7ec41fd640b Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Thu, 9 Jul 2020 06:53:26 +0000 Subject: [PATCH 5/9] Fixing mocha test for custom TCP --- test/tester.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/tester.test.ts b/test/tester.test.ts index c2cd042..6f0377e 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -44,6 +44,9 @@ describe('Tester', () => { it('should do a custom tcp test', async () => { config.testConfig.custom_tcp.hosts = ['www.google.com'] + td.when( + got('http://www.google.com', { timeout: 500 }) + ).thenResolve({ statusCode: 200 }) const result = await sut.runCustomTCPTests() should(result[0].result).eql('pass') }) From 595517698381394ad51ab46306792f0221d367e5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Wulff Date: Thu, 9 Jul 2020 06:59:25 +0000 Subject: [PATCH 6/9] Updating helm values --- helmfile/charts/kconmon/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index 3914319..5d3b701 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,6 +1,6 @@ docker: image: barto92/kubemon - tag: 1.0.41 + tag: latest # Should we run an initContainer that enables core tcp connection setting tweaks enableTcpTweaks: false @@ -39,6 +39,7 @@ config: - kubernetes.default.svc.cluster.local icmp: + enable: true interval: 5000 count: 2 timeout: 5 @@ -47,6 +48,7 @@ config: - www.google.com - 8.8.4.4 custom_tcp: + enable: true interval: 5000 timeout: 1000 hosts: From c137735b829ee6244a292b834e0c8cf9fda3ce55 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Thu, 9 Jul 2020 18:27:44 +0200 Subject: [PATCH 7/9] revert changes to the docker-compose and helm change line-endings back to LF --- docker-compose.yml | 4 +- helmfile/charts/kconmon/values.yaml | 9 +- lib/apps/agent/metrics.ts | 658 ++++++++++++------------ lib/tester/index.ts | 766 ++++++++++++++-------------- 4 files changed, 723 insertions(+), 714 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 434d2f3..adab234 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.2' services: controller: - image: barto92/kubemon:${VERSION:-latest} + image: stono/kconmon:${VERSION:-latest} build: context: '.' args: @@ -24,7 +24,7 @@ services: - ~/.kube:/home/nonroot/.kube:ro agent: - image: barto92/kubemon:${VERSION:-latest} + image: stono/kconmon:${VERSION:-latest} command: 'agent' environment: PORT: '80' # Run on port 80 locally in docker-compose to replicate kubernetes service object port magenting diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index 5d3b701..f0aa979 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -1,9 +1,9 @@ docker: - image: barto92/kubemon + image: stono/kconmon tag: latest # Should we run an initContainer that enables core tcp connection setting tweaks -enableTcpTweaks: false +enableTcpTweaks: true config: # What port should the server listen on @@ -37,7 +37,8 @@ config: hosts: - www.google.com - kubernetes.default.svc.cluster.local - + +# ICMP Test configuration icmp: enable: true interval: 5000 @@ -47,6 +48,8 @@ config: - www.telekom.de - www.google.com - 8.8.4.4 + +# Custom TCP tests configuration custom_tcp: enable: true interval: 5000 diff --git a/lib/apps/agent/metrics.ts b/lib/apps/agent/metrics.ts index 894b067..ce2b3d1 100644 --- a/lib/apps/agent/metrics.ts +++ b/lib/apps/agent/metrics.ts @@ -1,325 +1,333 @@ -export interface IMetrics { - handleTCPTestResult(result: ITCPTestResult) - handleCustomTCPTestResult(result: ICustomTCPTestResult) - handleUDPTestResult(result: IUDPTestResult) - handleDNSTestResult(result: IDNSTestResult) - handleICMPTestResult(result: IICMPTestResult) - resetTCPTestResults() - resetCustomTCPTestResults() - resetUDPTestResults() - toString() -} - -import * as client from 'prom-client' -import { IICMPTestResult, IUDPTestResult, IDNSTestResult, ITCPTestResult, ICustomTCPTestResult } from 'lib/tester' -import { IConfig } from 'lib/config' - -export default class Metrics implements IMetrics { - private TCP: client.Counter - private TCPDuration: client.Gauge - private TCPConnect: client.Gauge - - private UDP: client.Counter - private UDPDuration: client.Gauge - private UDPVariance: client.Gauge - private UDPLoss: client.Gauge - - private DNS: client.Counter - private DNSDuration: client.Gauge - - private ICMP: client.Counter - private ICMPDuration: client.Gauge - private ICMPAverage: client.Gauge - private ICMPStddv: client.Gauge - private ICMPLoss: client.Gauge - - private CustomTCP: client.Counter - private CustomTCPDuration: client.Gauge - private CustomTCPConnect: client.Gauge - - constructor(config: IConfig) { - client.register.clear() - this.TCPConnect = new client.Gauge({ - help: 'Time taken to establish the TCP socket', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_tcp_connect_milliseconds` - }) - - this.TCPDuration = new client.Gauge({ - help: 'Total time taken to complete the TCP test', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_tcp_duration_milliseconds` - }) - - this.UDPDuration = new client.Gauge({ - help: 'Average duration per packet', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_duration_milliseconds` - }) - - this.UDPVariance = new client.Gauge({ - help: 'UDP variance between the slowest and fastest packet', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_duration_variance_milliseconds` - }) - - this.UDPLoss = new client.Gauge({ - help: 'UDP packet loss', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_udp_loss` - }) - - this.DNS = new client.Counter({ - help: 'DNS Test Results', - labelNames: ['source', 'source_zone', 'host', 'result'], - name: `${config.metricsPrefix}_dns_results_total` - }) - - this.DNSDuration = new client.Gauge({ - help: 'Total time taken to complete the DNS test', - labelNames: ['source', 'source_zone', 'host'], - name: `${config.metricsPrefix}_dns_duration_milliseconds` - }) - - this.ICMP = new client.Counter({ - help: 'ICMP Test Results', - labelNames: ['source', 'source_zone', 'host', 'result'], - name: `${config.metricsPrefix}_icmp_results_total` - }) - - this.ICMPDuration = new client.Gauge({ - help: 'Total time taken to complete the ICMP test', - labelNames: ['source', 'source_zone', 'host'], - name: `${config.metricsPrefix}_icmp_duration_milliseconds` - }) - - this.ICMPAverage = new client.Gauge({ - help: 'ICMP average packet RTT', - labelNames: ['source', 'destination', 'host'], - name: `${config.metricsPrefix}_icmp_average_rtt_milliseconds` - }) - - this.ICMPStddv = new client.Gauge({ - help: 'ICMP standard deviation of RTT', - labelNames: ['source', 'destination', 'host'], - name: `${config.metricsPrefix}_icmp_standard_deviation_rtt_milliseconds` - }) - - this.ICMPLoss = new client.Gauge({ - help: 'ICMP packet loss', - labelNames: ['source', 'destination', 'host'], - name: `${config.metricsPrefix}_icmp_packet_loss` - }) - - this.UDP = new client.Counter({ - help: 'UDP Test Results', - labelNames: [ - 'source', - 'destination', - 'source_zone', - 'destination_zone', - 'result' - ], - name: `${config.metricsPrefix}_udp_results_total` - }) - - this.TCP = new client.Counter({ - help: 'TCP Test Results', - labelNames: [ - 'source', - 'destination', - 'source_zone', - 'destination_zone', - 'result' - ], - name: `${config.metricsPrefix}_tcp_results_total` - }) - - this.CustomTCP = new client.Counter({ - help: 'Custom TCP Test Results', - labelNames: [ - 'source', - 'destination', - 'source_zone', - 'destination_zone', - 'result' - ], - name: `${config.metricsPrefix}_custom_tcp_results_total` - }) - - this.CustomTCPConnect = new client.Gauge({ - help: 'Time taken to establish the TCP socket for custom test', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_custom_tcp_connect_milliseconds` - }) - - this.CustomTCPDuration = new client.Gauge({ - help: 'Total time taken to complete the custom TCP test', - labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_custom_tcp_duration_milliseconds` - }) - - } - - public handleICMPTestResult(result: IICMPTestResult): void { - const source = result.source.nodeName - this.ICMP.labels(source, result.source.zone, result.host, result.result).inc( - 1 - ) - this.ICMPDuration.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.duration) - - this.ICMPAverage.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.avg) - - this.ICMPStddv.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.stddev) - - this.ICMPLoss.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.loss) - } - - public handleDNSTestResult(result: IDNSTestResult): void { - const source = result.source.nodeName - this.DNS.labels(source, result.source.zone, result.host, result.result).inc( - 1 - ) - this.DNSDuration.labels( - result.source.nodeName, - result.source.zone, - result.host - ).set(result.duration) - } - - public resetTCPTestResults() { - this.TCPConnect.reset() - this.TCPDuration.reset() - } - - public resetUDPTestResults() { - this.UDPDuration.reset() - this.UDPLoss.reset() - this.UDPVariance.reset() - } - - public handleUDPTestResult(result: IUDPTestResult): void { - const source = result.source.nodeName - const destination = result.destination.nodeName - const sourceZone = result.source.zone - const destinationZone = result.destination.zone - this.UDP.labels( - source, - destination, - sourceZone, - destinationZone, - result.result - ).inc(1) - - if (result.timings) { - this.UDPDuration.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.average) - - this.UDPVariance.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.variance) - - this.UDPLoss.labels(source, destination, sourceZone, destinationZone).set( - result.timings.loss - ) - } - } - - public handleTCPTestResult(result: ITCPTestResult): void { - const source = result.source.nodeName - const destination = result.destination.nodeName - const sourceZone = result.source.zone - const destinationZone = result.destination.zone - this.TCP.labels( - source, - destination, - sourceZone, - destinationZone, - result.result - ).inc(1) - - if (result.timings) { - this.TCPConnect.labels( - source, - destination, - sourceZone, - destinationZone - ).set( - ((result.timings.connect || - result.timings.socket || - result.timings.start) - result.timings.start) as number - ) - this.TCPDuration.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.phases.total as number) - } - } - - public handleCustomTCPTestResult(result: ICustomTCPTestResult): void { - const source = result.source.nodeName - const destination = result.destination - const sourceZone = result.source.zone - const destinationZone = result.destination - this.CustomTCP.labels( - source, - destination, - sourceZone, - destinationZone, - result.result - ).inc(1) - - if (result.timings) { - this.CustomTCPConnect.labels( - source, - destination, - sourceZone, - destinationZone - ).set( - ((result.timings.connect || - result.timings.socket || - result.timings.start) - result.timings.start) as number - ) - this.CustomTCPDuration.labels( - source, - destination, - sourceZone, - destinationZone - ).set(result.timings.phases.total as number) - } - } - - public resetCustomTCPTestResults() { - this.CustomTCPConnect.reset() - this.CustomTCPDuration.reset() - } - - public toString(): string { - return client.register.metrics() - } -} +export interface IMetrics { + handleTCPTestResult(result: ITCPTestResult) + handleCustomTCPTestResult(result: ICustomTCPTestResult) + handleUDPTestResult(result: IUDPTestResult) + handleDNSTestResult(result: IDNSTestResult) + handleICMPTestResult(result: IICMPTestResult) + resetTCPTestResults() + resetCustomTCPTestResults() + resetUDPTestResults() + toString() +} + +import * as client from 'prom-client' +import { + IICMPTestResult, + IUDPTestResult, + IDNSTestResult, + ITCPTestResult, + ICustomTCPTestResult +} from 'lib/tester' +import { IConfig } from 'lib/config' + +export default class Metrics implements IMetrics { + private TCP: client.Counter + private TCPDuration: client.Gauge + private TCPConnect: client.Gauge + + private UDP: client.Counter + private UDPDuration: client.Gauge + private UDPVariance: client.Gauge + private UDPLoss: client.Gauge + + private DNS: client.Counter + private DNSDuration: client.Gauge + + private ICMP: client.Counter + private ICMPDuration: client.Gauge + private ICMPAverage: client.Gauge + private ICMPStddv: client.Gauge + private ICMPLoss: client.Gauge + + private CustomTCP: client.Counter + private CustomTCPDuration: client.Gauge + private CustomTCPConnect: client.Gauge + + constructor(config: IConfig) { + client.register.clear() + this.TCPConnect = new client.Gauge({ + help: 'Time taken to establish the TCP socket', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_tcp_connect_milliseconds` + }) + + this.TCPDuration = new client.Gauge({ + help: 'Total time taken to complete the TCP test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_tcp_duration_milliseconds` + }) + + this.UDPDuration = new client.Gauge({ + help: 'Average duration per packet', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_duration_milliseconds` + }) + + this.UDPVariance = new client.Gauge({ + help: 'UDP variance between the slowest and fastest packet', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_duration_variance_milliseconds` + }) + + this.UDPLoss = new client.Gauge({ + help: 'UDP packet loss', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_udp_loss` + }) + + this.DNS = new client.Counter({ + help: 'DNS Test Results', + labelNames: ['source', 'source_zone', 'host', 'result'], + name: `${config.metricsPrefix}_dns_results_total` + }) + + this.DNSDuration = new client.Gauge({ + help: 'Total time taken to complete the DNS test', + labelNames: ['source', 'source_zone', 'host'], + name: `${config.metricsPrefix}_dns_duration_milliseconds` + }) + + this.ICMP = new client.Counter({ + help: 'ICMP Test Results', + labelNames: ['source', 'source_zone', 'host', 'result'], + name: `${config.metricsPrefix}_icmp_results_total` + }) + + this.ICMPDuration = new client.Gauge({ + help: 'Total time taken to complete the ICMP test', + labelNames: ['source', 'source_zone', 'host'], + name: `${config.metricsPrefix}_icmp_duration_milliseconds` + }) + + this.ICMPAverage = new client.Gauge({ + help: 'ICMP average packet RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_average_rtt_milliseconds` + }) + + this.ICMPStddv = new client.Gauge({ + help: 'ICMP standard deviation of RTT', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_standard_deviation_rtt_milliseconds` + }) + + this.ICMPLoss = new client.Gauge({ + help: 'ICMP packet loss', + labelNames: ['source', 'destination', 'host'], + name: `${config.metricsPrefix}_icmp_packet_loss` + }) + + this.UDP = new client.Counter({ + help: 'UDP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_udp_results_total` + }) + + this.TCP = new client.Counter({ + help: 'TCP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_tcp_results_total` + }) + + this.CustomTCP = new client.Counter({ + help: 'Custom TCP Test Results', + labelNames: [ + 'source', + 'destination', + 'source_zone', + 'destination_zone', + 'result' + ], + name: `${config.metricsPrefix}_custom_tcp_results_total` + }) + + this.CustomTCPConnect = new client.Gauge({ + help: 'Time taken to establish the TCP socket for custom test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_tcp_connect_milliseconds` + }) + + this.CustomTCPDuration = new client.Gauge({ + help: 'Total time taken to complete the custom TCP test', + labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], + name: `${config.metricsPrefix}_custom_tcp_duration_milliseconds` + }) + } + + public handleICMPTestResult(result: IICMPTestResult): void { + const source = result.source.nodeName + this.ICMP.labels( + source, + result.source.zone, + result.host, + result.result + ).inc(1) + this.ICMPDuration.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.duration) + + this.ICMPAverage.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.avg) + + this.ICMPStddv.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.stddev) + + this.ICMPLoss.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.loss) + } + + public handleDNSTestResult(result: IDNSTestResult): void { + const source = result.source.nodeName + this.DNS.labels(source, result.source.zone, result.host, result.result).inc( + 1 + ) + this.DNSDuration.labels( + result.source.nodeName, + result.source.zone, + result.host + ).set(result.duration) + } + + public resetTCPTestResults() { + this.TCPConnect.reset() + this.TCPDuration.reset() + } + + public resetUDPTestResults() { + this.UDPDuration.reset() + this.UDPLoss.reset() + this.UDPVariance.reset() + } + + public handleUDPTestResult(result: IUDPTestResult): void { + const source = result.source.nodeName + const destination = result.destination.nodeName + const sourceZone = result.source.zone + const destinationZone = result.destination.zone + this.UDP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.UDPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.average) + + this.UDPVariance.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.variance) + + this.UDPLoss.labels(source, destination, sourceZone, destinationZone).set( + result.timings.loss + ) + } + } + + public handleTCPTestResult(result: ITCPTestResult): void { + const source = result.source.nodeName + const destination = result.destination.nodeName + const sourceZone = result.source.zone + const destinationZone = result.destination.zone + this.TCP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.TCPConnect.labels( + source, + destination, + sourceZone, + destinationZone + ).set( + ((result.timings.connect || + result.timings.socket || + result.timings.start) - result.timings.start) as number + ) + this.TCPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.phases.total as number) + } + } + + public handleCustomTCPTestResult(result: ICustomTCPTestResult): void { + const source = result.source.nodeName + const destination = result.destination + const sourceZone = result.source.zone + const destinationZone = result.destination + this.CustomTCP.labels( + source, + destination, + sourceZone, + destinationZone, + result.result + ).inc(1) + + if (result.timings) { + this.CustomTCPConnect.labels( + source, + destination, + sourceZone, + destinationZone + ).set( + ((result.timings.connect || + result.timings.socket || + result.timings.start) - result.timings.start) as number + ) + this.CustomTCPDuration.labels( + source, + destination, + sourceZone, + destinationZone + ).set(result.timings.phases.total as number) + } + } + + public resetCustomTCPTestResults() { + this.CustomTCPConnect.reset() + this.CustomTCPDuration.reset() + } + + public toString(): string { + return client.register.metrics() + } +} diff --git a/lib/tester/index.ts b/lib/tester/index.ts index ce02586..ff2ba7f 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -1,384 +1,382 @@ -import { Got } from 'got/dist/source' -import { IDiscovery, IAgent } from 'lib/discovery' -import { PlainResponse } from 'got/dist/source/core' -import { IMetrics } from 'lib/apps/agent/metrics' -import { IUDPPingResult } from 'lib/udp/client' -import { IConfig } from 'lib/config' -import Logger, { ILogger } from 'lib/logger' -import * as dns from 'dns' -import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' -import * as ping from 'ping' - -export interface ITester { - start() - stop() - runUDPTests(agents: IAgent[]): Promise - runTCPTests(agents: IAgent[]): Promise - runDNSTests(): Promise - runICMPTests(): Promise - runCustomTCPTests(): Promise -} - -interface ITestResult { - source: IAgent - destination: IAgent - result: 'pass' | 'fail' -} - -export interface IDNSTestResult { - source: IAgent - host: string - duration: number - result: 'pass' | 'fail' -} - -export interface IICMPTestResult { - source: IAgent - host: string - duration: number, - avg: number, - stddev: number, - loss: number, - result: 'pass' | 'fail' -} - -export interface IUDPTestResult extends ITestResult { - timings?: IUDPPingResult -} - -export interface ITCPTestResult extends ITestResult { - timings?: PlainResponse['timings'] -} - -export interface ICustomTCPTestResult { - source: IAgent - destination: string - result: 'pass' | 'fail' - timings?: PlainResponse['timings'] -} - -export default class Tester implements ITester { - private got: Got - private discovery: IDiscovery - private logger: ILogger = new Logger('tester') - private metrics: IMetrics - private me: IAgent - private running = false - private config: IConfig - private resolver = new dns.promises.Resolver() - // private ping: pingman - private readonly udpClientFactory: IUDPClientFactory - - constructor( - config: IConfig, - got: Got, - discovery: IDiscovery, - metrics: IMetrics, - me: IAgent, - udpClientFactory: IUDPClientFactory - ) { - this.got = got - this.discovery = discovery - this.metrics = metrics - this.me = me - this.config = config - this.udpClientFactory = udpClientFactory - } - - public async start(): Promise { - const delay = (ms: number) => { - return new Promise((resolve) => setTimeout(resolve, ms)) - } - - const jitter = () => { - const rand = Math.random() * (500 - 50) - return Math.floor(rand + 100) - } - - this.running = true - let agents = await this.discovery.agents() - const agentUpdateLoop = async () => { - while (this.running) { - agents = await this.discovery.agents() - await delay(5000) - } - } - const tcpEventLoop = async () => { - while (this.running) { - this.metrics.resetTCPTestResults() - await this.runTCPTests(agents) - await delay(this.config.testConfig.tcp.interval + jitter()) - } - } - const udpEventLoop = async () => { - while (this.running) { - this.metrics.resetUDPTestResults() - await this.runUDPTests(agents) - await delay(this.config.testConfig.udp.interval + jitter()) - } - } - const dnsEventLoop = async () => { - while (this.running) { - await this.runDNSTests() - await delay(this.config.testConfig.dns.interval + jitter()) - } - } - const icmpEventLoop = async () => { - while (this.running) { - await this.runICMPTests() - await delay(this.config.testConfig.icmp.interval + jitter()) - } - } - const tcpCustomEventLoop = async () => { - while (this.running) { - this.metrics.resetCustomTCPTestResults() - await this.runCustomTCPTests() - await delay(this.config.testConfig.custom_tcp.interval + jitter()) - } - } - - agentUpdateLoop() - tcpEventLoop() - udpEventLoop() - dnsEventLoop() - if(this.config.testConfig.icmp.enable){ - icmpEventLoop() - } - if(this.config.testConfig.custom_tcp.enable){ - tcpCustomEventLoop() - } - } - - public async stop(): Promise { - this.running = false - } - - public async runICMPTests(): Promise { - const promises = this.config.testConfig.icmp.hosts.map( - async (host): Promise => { - const hrstart = process.hrtime() - try { - const result = await ping.promise.probe(host, { - timeout: this.config.testConfig.icmp.timeout, - extra: ['-c', this.config.testConfig.icmp.count], - }); - const hrend = process.hrtime(hrstart) - - if(result.alive){ - const mapped: IICMPTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - avg: parseFloat(result.avg), - stddev: parseFloat(result.stddev), - loss: parseFloat(result.packetLoss), - result: 'pass' - } - this.metrics.handleICMPTestResult(mapped) - return mapped - } else { - const mapped: IICMPTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - avg: 0, - stddev: 0, - loss: parseFloat(result.packetLoss), - result: 'fail' - } - this.metrics.handleICMPTestResult(mapped) - return mapped - } - } catch (ex) { - this.logger.error(`icmp test for ${host} failed`, ex) - const hrend = process.hrtime(hrstart) - const mapped: IICMPTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - avg: 0, - stddev: 0, - loss: 100.000, - result: 'fail' - } - this.metrics.handleICMPTestResult(mapped) - return mapped - } - } - ) - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } - - - public async runDNSTests(): Promise { - const promises = this.config.testConfig.dns.hosts.map( - async (host): Promise => { - const hrstart = process.hrtime() - try { - const result = await this.resolver.resolve4(host) - const hrend = process.hrtime(hrstart) - const mapped: IDNSTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - result: result && result.length > 0 ? 'pass' : 'fail' - } - this.metrics.handleDNSTestResult(mapped) - return mapped - } catch (ex) { - this.logger.error(`dns test for ${host} failed`, ex) - const hrend = process.hrtime(hrstart) - const mapped: IDNSTestResult = { - source: this.me, - host, - duration: hrend[1] / 1000000, - result: 'fail' - } - this.metrics.handleDNSTestResult(mapped) - return mapped - } - } - ) - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } - - public async runUDPTests(agents: IAgent[]): Promise { - const results: IUDPTestResult[] = [] - this.udpClientFactory.generateClientsForAgents(agents) - - const testAgent = async (agent: IAgent): Promise => { - const client = this.udpClientFactory.clientFor(agent) - try { - const result = await client.ping( - this.config.testConfig.udp.timeout, - this.config.testConfig.udp.packets - ) - if (result.loss > 0) { - this.logger.warn('packet loss detected', result) - } - const testResult: IUDPTestResult = { - source: this.me, - destination: agent, - timings: result, - result: result.loss > 0 ? 'fail' : 'pass' - } - results.push(testResult) - this.metrics.handleUDPTestResult(testResult) - } catch (ex) { - this.logger.error('Failed to execute UDP test', ex) - const testResult: IUDPTestResult = { - source: this.me, - destination: agent, - result: 'fail' - } - results.push(testResult) - this.metrics.handleUDPTestResult(testResult) - } - } - const promises = agents.map(testAgent) - await Promise.allSettled(promises) - - return results - } - - public async runTCPTests(agents: IAgent[]): Promise { - const testAgent = async (agent: IAgent): Promise => { - try { - const url = `http://${agent.ip}:${this.config.port}/readiness` - const result = await this.got(url, { - timeout: this.config.testConfig.tcp.timeout - }) - const mappedResult: ITCPTestResult = { - source: this.me, - destination: agent, - timings: result.timings, - result: result.statusCode === 200 ? 'pass' : 'fail' - } - this.metrics.handleTCPTestResult(mappedResult) - return mappedResult - } catch (ex) { - this.logger.warn( - `test failed`, - { - source: this.me, - destination: agent - }, - ex - ) - const failResult: ITCPTestResult = { - source: this.me, - destination: agent, - result: 'fail' - } - this.metrics.handleTCPTestResult(failResult) - return failResult - } - } - const promises = agents.map(testAgent) - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } - - public async runCustomTCPTests(): Promise { - const promises = this.config.testConfig.custom_tcp.hosts.map( - async (host): Promise => { - try { - const url = `http://${host}` - const result = await this.got(url, { - timeout: this.config.testConfig.custom_tcp.timeout - }) - const htmlReponseCodes = [200, 301, 302, 304, 401] - if(htmlReponseCodes.includes(result.statusCode)){ - const mappedResult: ICustomTCPTestResult = { - source: this.me, - destination: host, - timings: result.timings, - result: 'pass' - } - this.metrics.handleCustomTCPTestResult(mappedResult) - return mappedResult - } else { - const mappedResult: ICustomTCPTestResult = { - source: this.me, - destination: host, - timings: result.timings, - result: 'fail' - } - this.metrics.handleCustomTCPTestResult(mappedResult) - return mappedResult - } - - } catch (ex) { - this.logger.warn( - `test failed`, - { - source: this.me, - destination: host - }, - ex - ) - const failResult: ICustomTCPTestResult = { - source: this.me, - destination: host, - result: 'fail' - } - this.metrics.handleCustomTCPTestResult(failResult) - return failResult - } - } - ) - - const result = await Promise.allSettled(promises) - return result - .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) - } -} +import { Got } from 'got/dist/source' +import { IDiscovery, IAgent } from 'lib/discovery' +import { PlainResponse } from 'got/dist/source/core' +import { IMetrics } from 'lib/apps/agent/metrics' +import { IUDPPingResult } from 'lib/udp/client' +import { IConfig } from 'lib/config' +import Logger, { ILogger } from 'lib/logger' +import * as dns from 'dns' +import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' +import * as ping from 'ping' + +export interface ITester { + start() + stop() + runUDPTests(agents: IAgent[]): Promise + runTCPTests(agents: IAgent[]): Promise + runDNSTests(): Promise + runICMPTests(): Promise + runCustomTCPTests(): Promise +} + +interface ITestResult { + source: IAgent + destination: IAgent + result: 'pass' | 'fail' +} + +export interface IDNSTestResult { + source: IAgent + host: string + duration: number + result: 'pass' | 'fail' +} + +export interface IICMPTestResult { + source: IAgent + host: string + duration: number + avg: number + stddev: number + loss: number + result: 'pass' | 'fail' +} + +export interface IUDPTestResult extends ITestResult { + timings?: IUDPPingResult +} + +export interface ITCPTestResult extends ITestResult { + timings?: PlainResponse['timings'] +} + +export interface ICustomTCPTestResult { + source: IAgent + destination: string + result: 'pass' | 'fail' + timings?: PlainResponse['timings'] +} + +export default class Tester implements ITester { + private got: Got + private discovery: IDiscovery + private logger: ILogger = new Logger('tester') + private metrics: IMetrics + private me: IAgent + private running = false + private config: IConfig + private resolver = new dns.promises.Resolver() + // private ping: pingman + private readonly udpClientFactory: IUDPClientFactory + + constructor( + config: IConfig, + got: Got, + discovery: IDiscovery, + metrics: IMetrics, + me: IAgent, + udpClientFactory: IUDPClientFactory + ) { + this.got = got + this.discovery = discovery + this.metrics = metrics + this.me = me + this.config = config + this.udpClientFactory = udpClientFactory + } + + public async start(): Promise { + const delay = (ms: number) => { + return new Promise((resolve) => setTimeout(resolve, ms)) + } + + const jitter = () => { + const rand = Math.random() * (500 - 50) + return Math.floor(rand + 100) + } + + this.running = true + let agents = await this.discovery.agents() + const agentUpdateLoop = async () => { + while (this.running) { + agents = await this.discovery.agents() + await delay(5000) + } + } + const tcpEventLoop = async () => { + while (this.running) { + this.metrics.resetTCPTestResults() + await this.runTCPTests(agents) + await delay(this.config.testConfig.tcp.interval + jitter()) + } + } + const udpEventLoop = async () => { + while (this.running) { + this.metrics.resetUDPTestResults() + await this.runUDPTests(agents) + await delay(this.config.testConfig.udp.interval + jitter()) + } + } + const dnsEventLoop = async () => { + while (this.running) { + await this.runDNSTests() + await delay(this.config.testConfig.dns.interval + jitter()) + } + } + const icmpEventLoop = async () => { + while (this.running) { + await this.runICMPTests() + await delay(this.config.testConfig.icmp.interval + jitter()) + } + } + const tcpCustomEventLoop = async () => { + while (this.running) { + this.metrics.resetCustomTCPTestResults() + await this.runCustomTCPTests() + await delay(this.config.testConfig.custom_tcp.interval + jitter()) + } + } + + agentUpdateLoop() + tcpEventLoop() + udpEventLoop() + dnsEventLoop() + if (this.config.testConfig.icmp.enable) { + icmpEventLoop() + } + if (this.config.testConfig.custom_tcp.enable) { + tcpCustomEventLoop() + } + } + + public async stop(): Promise { + this.running = false + } + + public async runICMPTests(): Promise { + const promises = this.config.testConfig.icmp.hosts.map( + async (host): Promise => { + const hrstart = process.hrtime() + try { + const result = await ping.promise.probe(host, { + timeout: this.config.testConfig.icmp.timeout, + extra: ['-c', this.config.testConfig.icmp.count] + }) + const hrend = process.hrtime(hrstart) + + if (result.alive) { + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: parseFloat(result.avg), + stddev: parseFloat(result.stddev), + loss: parseFloat(result.packetLoss), + result: 'pass' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } else { + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: parseFloat(result.packetLoss), + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } catch (ex) { + this.logger.error(`icmp test for ${host} failed`, ex) + const hrend = process.hrtime(hrstart) + const mapped: IICMPTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + avg: 0, + stddev: 0, + loss: 100.0, + result: 'fail' + } + this.metrics.handleICMPTestResult(mapped) + return mapped + } + } + ) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + public async runDNSTests(): Promise { + const promises = this.config.testConfig.dns.hosts.map( + async (host): Promise => { + const hrstart = process.hrtime() + try { + const result = await this.resolver.resolve4(host) + const hrend = process.hrtime(hrstart) + const mapped: IDNSTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + result: result && result.length > 0 ? 'pass' : 'fail' + } + this.metrics.handleDNSTestResult(mapped) + return mapped + } catch (ex) { + this.logger.error(`dns test for ${host} failed`, ex) + const hrend = process.hrtime(hrstart) + const mapped: IDNSTestResult = { + source: this.me, + host, + duration: hrend[1] / 1000000, + result: 'fail' + } + this.metrics.handleDNSTestResult(mapped) + return mapped + } + } + ) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + public async runUDPTests(agents: IAgent[]): Promise { + const results: IUDPTestResult[] = [] + this.udpClientFactory.generateClientsForAgents(agents) + + const testAgent = async (agent: IAgent): Promise => { + const client = this.udpClientFactory.clientFor(agent) + try { + const result = await client.ping( + this.config.testConfig.udp.timeout, + this.config.testConfig.udp.packets + ) + if (result.loss > 0) { + this.logger.warn('packet loss detected', result) + } + const testResult: IUDPTestResult = { + source: this.me, + destination: agent, + timings: result, + result: result.loss > 0 ? 'fail' : 'pass' + } + results.push(testResult) + this.metrics.handleUDPTestResult(testResult) + } catch (ex) { + this.logger.error('Failed to execute UDP test', ex) + const testResult: IUDPTestResult = { + source: this.me, + destination: agent, + result: 'fail' + } + results.push(testResult) + this.metrics.handleUDPTestResult(testResult) + } + } + const promises = agents.map(testAgent) + await Promise.allSettled(promises) + + return results + } + + public async runTCPTests(agents: IAgent[]): Promise { + const testAgent = async (agent: IAgent): Promise => { + try { + const url = `http://${agent.ip}:${this.config.port}/readiness` + const result = await this.got(url, { + timeout: this.config.testConfig.tcp.timeout + }) + const mappedResult: ITCPTestResult = { + source: this.me, + destination: agent, + timings: result.timings, + result: result.statusCode === 200 ? 'pass' : 'fail' + } + this.metrics.handleTCPTestResult(mappedResult) + return mappedResult + } catch (ex) { + this.logger.warn( + `test failed`, + { + source: this.me, + destination: agent + }, + ex + ) + const failResult: ITCPTestResult = { + source: this.me, + destination: agent, + result: 'fail' + } + this.metrics.handleTCPTestResult(failResult) + return failResult + } + } + const promises = agents.map(testAgent) + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } + + public async runCustomTCPTests(): Promise { + const promises = this.config.testConfig.custom_tcp.hosts.map( + async (host): Promise => { + try { + const url = `http://${host}` + const result = await this.got(url, { + timeout: this.config.testConfig.custom_tcp.timeout + }) + const htmlReponseCodes = [200, 301, 302, 304, 401] + if (htmlReponseCodes.includes(result.statusCode)) { + const mappedResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'pass' + } + this.metrics.handleCustomTCPTestResult(mappedResult) + return mappedResult + } else { + const mappedResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + timings: result.timings, + result: 'fail' + } + this.metrics.handleCustomTCPTestResult(mappedResult) + return mappedResult + } + } catch (ex) { + this.logger.warn( + `test failed`, + { + source: this.me, + destination: host + }, + ex + ) + const failResult: ICustomTCPTestResult = { + source: this.me, + destination: host, + result: 'fail' + } + this.metrics.handleCustomTCPTestResult(failResult) + return failResult + } + } + ) + + const result = await Promise.allSettled(promises) + return result + .filter((r) => r.status === 'fulfilled') + .map((i) => (i as PromiseFulfilledResult).value) + } +} From 54a79e743fed46982979b24d784df0804113d133 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Thu, 9 Jul 2020 18:33:28 +0200 Subject: [PATCH 8/9] Rename custom TCP checks to HTTP to reflect what they actually are --- helmfile/charts/kconmon/values.yaml | 4 +-- lib/apps/agent/metrics.ts | 38 ++++++++++++++--------------- lib/config/index.ts | 8 ++++-- lib/tester/index.ts | 34 +++++++++++++------------- test/tester.test.ts | 14 +++++------ 5 files changed, 51 insertions(+), 47 deletions(-) diff --git a/helmfile/charts/kconmon/values.yaml b/helmfile/charts/kconmon/values.yaml index f0aa979..dade20e 100644 --- a/helmfile/charts/kconmon/values.yaml +++ b/helmfile/charts/kconmon/values.yaml @@ -49,8 +49,8 @@ config: - www.google.com - 8.8.4.4 -# Custom TCP tests configuration - custom_tcp: +# Custom HTTP tests configuration + custom_http: enable: true interval: 5000 timeout: 1000 diff --git a/lib/apps/agent/metrics.ts b/lib/apps/agent/metrics.ts index ce2b3d1..0e39b57 100644 --- a/lib/apps/agent/metrics.ts +++ b/lib/apps/agent/metrics.ts @@ -1,11 +1,11 @@ export interface IMetrics { handleTCPTestResult(result: ITCPTestResult) - handleCustomTCPTestResult(result: ICustomTCPTestResult) + handleCustomHTTPTestResult(result: ICustomHTTPTestResult) handleUDPTestResult(result: IUDPTestResult) handleDNSTestResult(result: IDNSTestResult) handleICMPTestResult(result: IICMPTestResult) resetTCPTestResults() - resetCustomTCPTestResults() + resetCustomHTTPTestResults() resetUDPTestResults() toString() } @@ -16,7 +16,7 @@ import { IUDPTestResult, IDNSTestResult, ITCPTestResult, - ICustomTCPTestResult + ICustomHTTPTestResult } from 'lib/tester' import { IConfig } from 'lib/config' @@ -39,9 +39,9 @@ export default class Metrics implements IMetrics { private ICMPStddv: client.Gauge private ICMPLoss: client.Gauge - private CustomTCP: client.Counter - private CustomTCPDuration: client.Gauge - private CustomTCPConnect: client.Gauge + private CustomHTTP: client.Counter + private CustomHTTPDuration: client.Gauge + private CustomHTTPConnect: client.Gauge constructor(config: IConfig) { client.register.clear() @@ -141,7 +141,7 @@ export default class Metrics implements IMetrics { name: `${config.metricsPrefix}_tcp_results_total` }) - this.CustomTCP = new client.Counter({ + this.CustomHTTP = new client.Counter({ help: 'Custom TCP Test Results', labelNames: [ 'source', @@ -150,19 +150,19 @@ export default class Metrics implements IMetrics { 'destination_zone', 'result' ], - name: `${config.metricsPrefix}_custom_tcp_results_total` + name: `${config.metricsPrefix}_custom_http_results_total` }) - this.CustomTCPConnect = new client.Gauge({ + this.CustomHTTPConnect = new client.Gauge({ help: 'Time taken to establish the TCP socket for custom test', labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_custom_tcp_connect_milliseconds` + name: `${config.metricsPrefix}_custom_http_connect_milliseconds` }) - this.CustomTCPDuration = new client.Gauge({ + this.CustomHTTPDuration = new client.Gauge({ help: 'Total time taken to complete the custom TCP test', labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], - name: `${config.metricsPrefix}_custom_tcp_duration_milliseconds` + name: `${config.metricsPrefix}_custom_http_duration_milliseconds` }) } @@ -289,12 +289,12 @@ export default class Metrics implements IMetrics { } } - public handleCustomTCPTestResult(result: ICustomTCPTestResult): void { + public handleCustomHTTPTestResult(result: ICustomHTTPTestResult): void { const source = result.source.nodeName const destination = result.destination const sourceZone = result.source.zone const destinationZone = result.destination - this.CustomTCP.labels( + this.CustomHTTP.labels( source, destination, sourceZone, @@ -303,7 +303,7 @@ export default class Metrics implements IMetrics { ).inc(1) if (result.timings) { - this.CustomTCPConnect.labels( + this.CustomHTTPConnect.labels( source, destination, sourceZone, @@ -313,7 +313,7 @@ export default class Metrics implements IMetrics { result.timings.socket || result.timings.start) - result.timings.start) as number ) - this.CustomTCPDuration.labels( + this.CustomHTTPDuration.labels( source, destination, sourceZone, @@ -322,9 +322,9 @@ export default class Metrics implements IMetrics { } } - public resetCustomTCPTestResults() { - this.CustomTCPConnect.reset() - this.CustomTCPDuration.reset() + public resetCustomHTTPTestResults() { + this.CustomHTTPConnect.reset() + this.CustomHTTPDuration.reset() } public toString(): string { diff --git a/lib/config/index.ts b/lib/config/index.ts index 2e2e4da..c1210f1 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -35,7 +35,7 @@ interface ITestConfiguration { timeout: number hosts: string[] } - custom_tcp: { + custom_http: { enable: boolean interval: number timeout: number @@ -75,7 +75,11 @@ export class Config implements IConfig { udp: getEnv('udp', { interval: 5000, timeout: 250, packets: 10 }), dns: getEnv('dns', { interval: 5000, hosts: [] }), icmp: getEnv('icmp', { interval: 5000, hosts: [] }), - custom_tcp: getEnv('custom_tcp', { interval: 5000, timeout: 1000, hosts: [] }) + custom_http: getEnv('custom_http', { + interval: 5000, + timeout: 1000, + hosts: [] + }) } } diff --git a/lib/tester/index.ts b/lib/tester/index.ts index ff2ba7f..60bf4a7 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -16,7 +16,7 @@ export interface ITester { runTCPTests(agents: IAgent[]): Promise runDNSTests(): Promise runICMPTests(): Promise - runCustomTCPTests(): Promise + runCustomHTTPTests(): Promise } interface ITestResult { @@ -50,7 +50,7 @@ export interface ITCPTestResult extends ITestResult { timings?: PlainResponse['timings'] } -export interface ICustomTCPTestResult { +export interface ICustomHTTPTestResult { source: IAgent destination: string result: 'pass' | 'fail' @@ -131,9 +131,9 @@ export default class Tester implements ITester { } const tcpCustomEventLoop = async () => { while (this.running) { - this.metrics.resetCustomTCPTestResults() - await this.runCustomTCPTests() - await delay(this.config.testConfig.custom_tcp.interval + jitter()) + this.metrics.resetCustomHTTPTestResults() + await this.runCustomHTTPTests() + await delay(this.config.testConfig.custom_http.interval + jitter()) } } @@ -144,7 +144,7 @@ export default class Tester implements ITester { if (this.config.testConfig.icmp.enable) { icmpEventLoop() } - if (this.config.testConfig.custom_tcp.enable) { + if (this.config.testConfig.custom_http.enable) { tcpCustomEventLoop() } } @@ -326,32 +326,32 @@ export default class Tester implements ITester { .map((i) => (i as PromiseFulfilledResult).value) } - public async runCustomTCPTests(): Promise { - const promises = this.config.testConfig.custom_tcp.hosts.map( - async (host): Promise => { + public async runCustomHTTPTests(): Promise { + const promises = this.config.testConfig.custom_http.hosts.map( + async (host): Promise => { try { const url = `http://${host}` const result = await this.got(url, { - timeout: this.config.testConfig.custom_tcp.timeout + timeout: this.config.testConfig.custom_http.timeout }) const htmlReponseCodes = [200, 301, 302, 304, 401] if (htmlReponseCodes.includes(result.statusCode)) { - const mappedResult: ICustomTCPTestResult = { + const mappedResult: ICustomHTTPTestResult = { source: this.me, destination: host, timings: result.timings, result: 'pass' } - this.metrics.handleCustomTCPTestResult(mappedResult) + this.metrics.handleCustomHTTPTestResult(mappedResult) return mappedResult } else { - const mappedResult: ICustomTCPTestResult = { + const mappedResult: ICustomHTTPTestResult = { source: this.me, destination: host, timings: result.timings, result: 'fail' } - this.metrics.handleCustomTCPTestResult(mappedResult) + this.metrics.handleCustomHTTPTestResult(mappedResult) return mappedResult } } catch (ex) { @@ -363,12 +363,12 @@ export default class Tester implements ITester { }, ex ) - const failResult: ICustomTCPTestResult = { + const failResult: ICustomHTTPTestResult = { source: this.me, destination: host, result: 'fail' } - this.metrics.handleCustomTCPTestResult(failResult) + this.metrics.handleCustomHTTPTestResult(failResult) return failResult } } @@ -377,6 +377,6 @@ export default class Tester implements ITester { const result = await Promise.allSettled(promises) return result .filter((r) => r.status === 'fulfilled') - .map((i) => (i as PromiseFulfilledResult).value) + .map((i) => (i as PromiseFulfilledResult).value) } } diff --git a/test/tester.test.ts b/test/tester.test.ts index 6f0377e..f9c750a 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -19,8 +19,8 @@ describe('Tester', () => { config.testConfig.udp.timeout = 500 config.testConfig.udp.packets = 1 config.testConfig.tcp.timeout = 500 - config.testConfig.custom_tcp.enable = true - config.testConfig.custom_tcp.timeout = 500 + config.testConfig.custom_http.enable = true + config.testConfig.custom_http.timeout = 500 config.testConfig.icmp.enable = true config.testConfig.icmp.count = 2 config.testConfig.icmp.timeout = 5 @@ -43,11 +43,11 @@ describe('Tester', () => { }) it('should do a custom tcp test', async () => { - config.testConfig.custom_tcp.hosts = ['www.google.com'] - td.when( - got('http://www.google.com', { timeout: 500 }) - ).thenResolve({ statusCode: 200 }) - const result = await sut.runCustomTCPTests() + config.testConfig.custom_http.hosts = ['www.google.com'] + td.when(got('http://www.google.com', { timeout: 500 })).thenResolve({ + statusCode: 200 + }) + const result = await sut.runCustomHTTPTests() should(result[0].result).eql('pass') }) From 9d5c3439b1ec3f0981c763faae4134b2faa8a5f5 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Fri, 10 Jul 2020 12:25:58 +0200 Subject: [PATCH 9/9] Fix missing rename for testing loop --- lib/tester/index.ts | 4 ++-- test/tester.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tester/index.ts b/lib/tester/index.ts index 60bf4a7..d67935f 100644 --- a/lib/tester/index.ts +++ b/lib/tester/index.ts @@ -129,7 +129,7 @@ export default class Tester implements ITester { await delay(this.config.testConfig.icmp.interval + jitter()) } } - const tcpCustomEventLoop = async () => { + const httpCustomEventLoop = async () => { while (this.running) { this.metrics.resetCustomHTTPTestResults() await this.runCustomHTTPTests() @@ -145,7 +145,7 @@ export default class Tester implements ITester { icmpEventLoop() } if (this.config.testConfig.custom_http.enable) { - tcpCustomEventLoop() + httpCustomEventLoop() } } diff --git a/test/tester.test.ts b/test/tester.test.ts index f9c750a..66e2096 100644 --- a/test/tester.test.ts +++ b/test/tester.test.ts @@ -42,7 +42,7 @@ describe('Tester', () => { should(result[0].result).eql('pass') }) - it('should do a custom tcp test', async () => { + it('should do a custom http test', async () => { config.testConfig.custom_http.hosts = ['www.google.com'] td.when(got('http://www.google.com', { timeout: 500 })).thenResolve({ statusCode: 200