From 7e497ffd59ad22de830e420a1346a01a8f722119 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Sat, 23 Apr 2022 21:57:46 +0200 Subject: [PATCH] setup grafana, prometheus, jmx --- .../infra/docker-compose.gradle.kts | 1 + gradle/libs.versions.toml | 2 + modules/infra-kafka-cluster/build.gradle.kts | 21 + modules/infra-kafka-cluster/src/.env | 3 + ...fana_dashboard_broker_hard_disk_usage.json | 378 ++++ .../grafana_dashboard_broker_jvm_os.json | 801 +++++++ .../grafana_dashboard_broker_performance.json | 1532 +++++++++++++ .../grafana_dashboard_broker_zookeeper.json | 688 ++++++ ..._dashboard_client_consumers_fetch_lag.json | 492 +++++ ...grafana_dashboard_cluster_healthcheck.json | 1887 +++++++++++++++++ ...grafana_dashboard_cluster_replication.json | 574 +++++ .../grafana_dashboard_topics_logs.json | 876 ++++++++ .../provisioning/dashboards/kafka.yaml | 12 + .../provisioning/datasources/datasource.yaml | 50 + .../src/config/jmx_exporter_kafka_broker.yml | 505 +++++ .../src/config/prometheus.yml | 29 + .../src/docker-compose.yml | 49 + 17 files changed, 7900 insertions(+) create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_hard_disk_usage.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_jvm_os.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_performance.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_zookeeper.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_client_consumers_fetch_lag.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_healthcheck.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_replication.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_topics_logs.json create mode 100644 modules/infra-kafka-cluster/src/config/grafana/provisioning/dashboards/kafka.yaml create mode 100644 modules/infra-kafka-cluster/src/config/grafana/provisioning/datasources/datasource.yaml create mode 100644 modules/infra-kafka-cluster/src/config/jmx_exporter_kafka_broker.yml create mode 100644 modules/infra-kafka-cluster/src/config/prometheus.yml diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/infra/docker-compose.gradle.kts b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/infra/docker-compose.gradle.kts index 520fa83..7bb535a 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/infra/docker-compose.gradle.kts +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/infra/docker-compose.gradle.kts @@ -53,6 +53,7 @@ val dockerEnv by tasks.registering(WriteProperties::class) { logging.captureStandardOutput(LogLevel.LIFECYCLE) setOutputFile(dockerSrcDir.file(".env")) + properties( "COMPOSE_PROJECT_NAME" to dockerComposeProjectName, "KAFKATORIO_VERSION" to project.version, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ce36119..ad0931d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,8 @@ kotest = "5.2.3" ## infra ## factorio = "1.1.57" confluent-kafkaConnect = "7.0.1" +grafana = "8.5.0" +prometheus = "v2.34.0" ######### [plugins] diff --git a/modules/infra-kafka-cluster/build.gradle.kts b/modules/infra-kafka-cluster/build.gradle.kts index 35b7263..8f52b7e 100644 --- a/modules/infra-kafka-cluster/build.gradle.kts +++ b/modules/infra-kafka-cluster/build.gradle.kts @@ -3,3 +3,24 @@ plugins { } description = "Kafka cluster" + +tasks.dockerEnv { + properties( + "GRAFANA_VERSION" to libs.versions.grafana, + "PROMETHEUS_VERSION" to libs.versions.prometheus, + "KAFKA_VERSION" to libs.versions.kafka, + + // Limit JVM Heap Size +// "KAFKA_BROKER_HEAP_OPTS" to "-XX:MaxRAMPercentage=70.0", +// "KAFKA_CONNECT_HEAP_OPTS" to "-XX:MaxRAMPercentage=70.0", +// "SCHEMA_REGISTRY_HEAP_OPTS" to "-XX:MaxRAMPercentage=70.0", +// "KAFKA_KSQL_HEAP_OPTS" to "-XX:MaxRAMPercentage=70.0", +// "PROMETHEUS_JMX_AGENT_JVM_OPTS" to "-Xmx128M", + +// // Limit container resources +// "KAFKA_BROKER_MEM_LIMIT" to "512m", +// "KAFKA_CONNECT_MEM_LIMIT" to "512m", +// "KAFKA_KSQL_MEM_LIMIT" to "256m", +// "SCHEMA_REGISTRY_MEM_LIMIT" to "256m", + ) +} diff --git a/modules/infra-kafka-cluster/src/.env b/modules/infra-kafka-cluster/src/.env index 469b75e..bb45ded 100644 --- a/modules/infra-kafka-cluster/src/.env +++ b/modules/infra-kafka-cluster/src/.env @@ -1,3 +1,6 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio +GRAFANA_VERSION=8.5.0 KAFKATORIO_VERSION=0.4.0 +KAFKA_VERSION=3.1.0 +PROMETHEUS_VERSION=v2.34.0 diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_hard_disk_usage.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_hard_disk_usage.json new file mode 100644 index 0000000..0ff0182 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_hard_disk_usage.json @@ -0,0 +1,378 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1540328526402, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_log_size{env=\"$env\"}) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Global Topics Size (including replicas)", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(sum(kafka_log_size{env=\"$env\"}) by (topic, partition) / sum(kafka_cluster_partition_replicascount{env=\"$env\"}) by(partition, topic)) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Global Topics Size (without replication)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_log_size{env=\"$env\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Logs Size Per Broker (including replicas)", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "topics", + "disk" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Env", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_log_size, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kafka / Hard Disk Usage", + "uid": "zApgMBbik", + "version": 1 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_jvm_os.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_jvm_os.json new file mode 100644 index 0000000..8fad773 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_jvm_os.json @@ -0,0 +1,801 @@ +{ + "__inputs": [ + { + "name": "prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Java Virtual Machine Memory and OS metrics", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1540307470986, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_non_heap_usage{env=\"$env\", type=\"used\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "JVM Memory Used (Non-Heap)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 7, + "y": 0 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_heap_usage{env=\"$env\", type=\"used\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "JVM Memory Used (Heap)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 9, + "x": 15, + "y": 0 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_gc_collection_time{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} / {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC Process Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_os_systemcpuload{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_os_processcpuload{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Process CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_os_openfiledescriptorcount{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The amount of free physical memory in bytes.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_os_freephysicalmemorysize{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Free Physical Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The amount of virtual memory that is guaranteed to be available to the running process in bytes", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_jvm_os_committedvirtualmemorysize{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Available Virtual Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Env", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_jvm_os_openfiledescriptorcount, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kafka / Brokers JVM & OS", + "uid": "AdG9A1xmk", + "version": 3 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_performance.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_performance.json new file mode 100644 index 0000000..fd95efb --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_performance.json @@ -0,0 +1,1532 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1532113327583, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "repeat": "instance", + "title": "Request Latency : $request ($instance)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"Total\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{ aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"Total\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{ aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"Total\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{ aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Total Time Ms", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 16, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "kafka_network_socket_server_processor_avg_idle_percent{instance=~\"$instance\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "0.3,0.5,0.8", + "title": "Network Socket Avg Idle Percent", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 15, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "kafka_server_request_handler_avg_idle_percent{instance=~\"$instance\", aggregate=~\"OneMinuteRate\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "0.3,0.5,0.8", + "title": "Request Handler Avg Idle Percent", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kafka_network_request_per_sec{aggregate=~\"OneMinuteRate\",instance=~\"$instance\",request=~\"$request\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Requests Per Sec", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 5 + }, + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "kafka_network_request_channel_queue_size{instance=~\"$instance\", queue=~\"Response\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "0,50,1000", + "title": "Response Queue Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 5 + }, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "kafka_network_request_channel_queue_size{instance=~\"$instance\", queue=~\"Request\", env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "0,50,1000", + "title": "Request Queue Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"RequestQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"RequestQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"RequestQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Queue Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"Local\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"Local\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"Local\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Local Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"Remote\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"Remote\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"Remote\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Remote Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"Throttle\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"Throttle\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"Throttle\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Throttle Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"ResponseQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"ResponseQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"ResponseQueue\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Queue Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"Mean\", scope=~\"ResponseSend\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"75thPercentile\", scope=~\"ResponseSend\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_network_request_metrics_time_ms{instance=~\"$instance\", request=~\"$request\", aggregate=~\"99thPercentile\", scope=~\"ResponseSend\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Send Time(Ms)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_network_processor_idle_percent{instance=~\"$instance\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{processor}}", + "metric": "kafka_network_processor_idle_percent", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Processor Idle Percent", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 14, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_request_handler_avg_idle_percent{instance=~\"$instance\", aggregate=~\"OneMinuteRate\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{processor}}", + "metric": "kafka_network_processor_idle_percent", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Handler Avg Idle Percent", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "broker", + "troubleshoot" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Envrionment", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_server_brokerstate, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Broker Host", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(kafka_network_request_metrics_time_ms{env=\"$env\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Request Type", + "multi": false, + "name": "request", + "options": [], + "query": "label_values(kafka_network_request_metrics_time_ms, request)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Broker / Performance & Latency", + "uid": "aRNaJwOmk", + "version": 8 +} diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_zookeeper.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_zookeeper.json new file mode 100644 index 0000000..156b989 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_broker_zookeeper.json @@ -0,0 +1,688 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1539721769054, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Kafka/Zookeeper Session Expire Listener", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeepersyncconnectspersec{aggregate=\"OneMinute\", env=\"$env\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper Sync Connects Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeeperdisconnectspersec{env=\"$env\", aggregate=\"OneMinute\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper Disconnects Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "The ZooKeeper session has expired. **When a session expires, we can have leader changes and even a new controller.**\n\nIt is important to keep an eye on the number of such events across a Kafka cluster and if the overall number is high, then we have a few recommendations:\n\n* Check the health of your network\n* Check for garbage collection issues and tune it accordingly\n * If necessary, increase the session time out by setting the value of zookeeper.connection.timeout.ms.\n\nDocumentation : [monitoring.html#zookeeper-metrics](http://docs.confluent.io/current/kafka/monitoring.html#zookeeper-metrics)\n", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 7, + "isNew": true, + "links": [], + "mode": "markdown", + "title": "The ZooKeeper session has expired ?", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeeperreadonlyconnectspersec{env=\"$env\", aggregate=\"OneMinute\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper Read Only Connects Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeeperexpirespersec{env=\"$env\", aggregate=\"OneMinute\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper Expires Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeeperauthfailurespersec{env=\"$env\", aggregate=\"OneMinute\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper SASL Authentications Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_zookeeper_session_expire_listener_zookeeperauthfailurespersec{env=\"$env\", aggregate=\"OneMinute\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Zookeeper Auth Failures Per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "zookeeper" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Env", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_server_brokerstate, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Broker / Zookeeper Connection", + "uid": "142Xi34mk", + "version": 1 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_client_consumers_fetch_lag.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_client_consumers_fetch_lag.json new file mode 100644 index 0000000..a6703b4 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_client_consumers_fetch_lag.json @@ -0,0 +1,492 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1532113236152, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The latest lag of each client", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_consumer_fetch_manager_metrics_records_lag{env=\"$env\", topic=\"$topic\"}) by(topic, client_id, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{client_id}} ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Consumer Client Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The average number of bytes fetched per request for a topic", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumer_fetch_manager_metrics_fetch_size_avg{env=\"$env\", topic=\"$topic\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{client_id}} ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Fetch Bytes Per Request", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The average number of bytes consumed per second", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumer_fetch_manager_metrics_bytes_consumed_rate{env=\"$env\", topic=\"$topic\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{client_id}} ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Consumed Bytes Rate Per Second", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The average number of records consumed per second", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumer_fetch_manager_metrics_records_consumed_rate{env=\"$env\", topic=\"$topic\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{client_id}} ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Consumed Records Rate Per Second", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "clients" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "env", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_consumer_fetch_manager_metrics_records_consumed_rate, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "topic", + "multi": false, + "name": "topic", + "options": [], + "query": "label_values(kafka_consumer_fetch_manager_metrics_records_consumed_rate, topic)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Consumers / Fetch Rate & Records Lag", + "uid": "Ni95Y3dmz", + "version": 20 +} diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_healthcheck.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_healthcheck.json new file mode 100644 index 0000000..78cae0d --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_healthcheck.json @@ -0,0 +1,1887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1540310392981, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 29, + "title": "Healthcheck", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "description": "Number of active controllers in the cluster. Alert if the aggregated sum across all brokers in the cluster is anything other than 1 because there should be exactly one controller per cluster.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_controller_activecontrollercount{env=\"$env\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,1", + "title": "Active Controller", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Number of partitions that don’t have an active leader and are hence not writable or readable. Alert if value is greater than 0.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 3, + "y": 1 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_controller_offlinepartitionscount{env=\"$env\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Offline Partitions Count", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Number of under-replicated partitions (| ISR | < | all replicas |). Alert if value is greater than 0.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 7, + "y": 1 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_replica_manager_underreplicatedpartitions{env=\"$env\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Under Replicated Partitions", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Number of under-replicated partitions (| ISR | < | min.insync.replicas |). Alert if value is greater than 0.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 11, + "y": 1 + }, + "id": 36, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_underminisr{env=\"$env\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Partitions Under Min ISR", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 15, + "y": 1 + }, + "id": 34, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_controller_preferredreplicaimbalancecount{env=\"$env\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Preferred Replica Imbalance", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "content": "\"Apache\n\n

You're using Apache Kafka, an open-source distributed and fault-tolerant streaming platform originally built at LinkedIn. For more information, check out the kafka.apache.org", + "gridPos": { + "h": 6, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 46, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "decimals": 0, + "description": "Total number of brokers with a state running (i.e 3)", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 44, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count((kafka_server_brokerstate{env=\"$env\"}) == 3 or (kafka_server_brokerstate{env=\"$env\"}) == 4)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,3", + "title": "Brokers Running(>=3)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 0, + "description": "Total number of producer requests per second", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 3, + "y": 4 + }, + "id": 39, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_network_request_per_sec{aggregate=~\"OneMinuteRate\",request=~\"Produce\",env=\"$env\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Produce Req Per Sec", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 7, + "y": 4 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_network_request_per_sec{aggregate=~\"OneMinuteRate\",request=~\"FetchConsumer\",env=\"$env\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Fetch Consumer Req Per Sec", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 0, + "description": "Number of metadata requests per sec", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 11, + "y": 4 + }, + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_network_request_per_sec{aggregate=~\"OneMinuteRate\",request=~\"Metadata\",env=\"$env\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Metadata Req Per Sec", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 15, + "y": 4 + }, + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_network_request_per_sec{aggregate=~\"OneMinuteRate\",request=~\"OffsetCommit\",env=\"$env\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Offset Commit Req Per Sec", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 27, + "panels": [], + "title": "Throughput In/Out", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Aggregate incoming message rate.", + "fill": 1, + "gridPos": { + "h": 12, + "w": 19, + "x": 0, + "y": 8 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_server_broker_topic_metrics_messagesinpersec_rate{topic!=\"\"}) by(topic)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Messages In Per Topics", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "wps", + "label": "Msg/sec", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 2, + "description": "Aggregate incoming message rate.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 8 + }, + "id": 37, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_broker_topic_metrics_messagesinpersec_rate{topic=\"\"}) by (broker)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Messages In per sec", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 2, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 12 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_broker_topic_metrics_bytesinpersec_rate{topic=\"\"}) by (broker)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Bytes In per sec", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 2, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 16 + }, + "id": 25, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_server_broker_topic_metrics_bytesoutpersec_rate{topic=\"\"}) by (broker)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Bytes Out per sec", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 21, + "panels": [], + "title": "In-Sync Replica / Leaders Election", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "When a broker is brought up after a failure, it starts catching up by reading from the leader. Once it is caught up, it gets added back to the ISR.", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_replica_manager_isrexpandspersec{aggregate=\"OneMinute\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Broker={{instance}}", + "metric": "kafka_server_replication_isrexpandspersec", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "In-Sync Replica Expands Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "If a broker goes down, ISR for some of the partitions will shrink. When that broker is up again, ISR will be expanded once the replicas are fully caught up. Other than that, the expected value for both ISR shrink rate and expansion rate is 0.", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_replica_manager_isrshrinkspersec{aggregate=\"OneMinute\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Broker={{instance}}", + "metric": "kafka_server_replication_isrshrinkspersec", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "In-Sync Replica Shrinks Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Leader election rate and latency.", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 8, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_controller_stats_leaderelectionrateandtimems{aggregate=~\"Mean\",env=\"$env\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Broker={{instance}} / {{aggregate}}", + "metric": "kafka_controller_stats_leader_election_rate_and_time_ms", + "refId": "A", + "step": 10 + }, + { + "expr": "kafka_controller_stats_leaderelectionrateandtimems{aggregate=~\"75thPercentile\",env=\"$env\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Broker={{instance}} / {{aggregate}}", + "refId": "B", + "step": 10 + }, + { + "expr": "kafka_controller_stats_leaderelectionrateandtimems{aggregate=~\"99thPercentile\",env=\"$env\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Broker={{instance}} / {{aggregate}}", + "refId": "C", + "step": 10 + }, + { + "expr": "kafka_controller_stats_leaderelectionrateandtimems{aggregate=\"OneMinuteRate\", env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Broker={{instance}} / {{aggregate}}", + "refId": "D", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Leader Election Rate (ms/1MinuteRate)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 9, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_controller_stats_uncleanleaderelectionspersec{aggregate=\"OneMinuteRate\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Broker={{instance}} / {{aggregate}}", + "metric": "kafka_controller_stats_unclean_leader_elections_per_sec", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Unclean Leader Election Rate (should be < 0)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Number of leaders on each broker. This should be mostly even across all brokers. If not, set auto.leader.rebalance.enable to true on all brokers in the cluster.", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 14, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_replica_manager_leadercount{env=\"$env\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "Broker={{instance}}", + "metric": "kafka_server_replication_leadercount", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Leader Count Per Broker", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 15, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_replica_manager_partitioncount{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Broker={{instance}}", + "metric": "kafka_server_replication_leadercount", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of Partitions Per Broker", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "broker", + "healthcheck" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "docker", + "value": "docker" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_server_brokerstate, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Cluster / Global HealthCheck", + "uid": "e-6AJQOik", + "version": 1 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_replication.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_replication.json new file mode 100644 index 0000000..97b06fb --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_cluster_replication.json @@ -0,0 +1,574 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1540330422283, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_broker_topic_metrics_replicationbytesinpersec_rate{aggregate=\"OneMinute\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication BytesIn/Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_broker_topic_metrics_replicationbytesoutpersec_rate{aggregate=\"OneMinute\",env=\"$env\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication BytesOut/Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_replica_fetcher_manager_maxlag_value{env=\"$env\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication Max Lag Per Broker", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 3, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_fetcher_stats_bytespersec{env=\"$env\", aggregate=\"OneMinute\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Host {{instance}}, client_id={{client_id}} / {{broker_host}}:{{broker_port}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Fetch Bytes/sec Per Replication thread", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 4, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_fetcher_stats_requestspersec{env=\"$env\", aggregate=\"OneMinute\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Host {{instance}}, client_id={{client_id}} / {{broker_host}}:{{broker_port}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Fetch Requests/sec Per Replication thread", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "broker", + "replication" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Env", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_server_brokerstate, env)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Cluster / Replication", + "uid": "E9bXmq4ik", + "version": 2 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_topics_logs.json b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_topics_logs.json new file mode 100644 index 0000000..4f91add --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/dashboards/grafana_dashboard_topics_logs.json @@ -0,0 +1,876 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1540331963544, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "repeat": "topic", + "title": "$topic", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_log_size{env=\"$env\", topic=\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Log Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count_values(\"partition\", kafka_log_numlogsegments{topic=\"$topic\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Number of Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "should alert if > 0", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_underminisr{env=\"$env\", topic=\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1", + "title": "Number of Partitions Under Min ISR", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "should alert if > 0", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kafka_cluster_partition_underreplicated{env=\"$env\", topic=\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1", + "title": "Under Replicated Partitions", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 5, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "topic", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "service", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "env", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "kafka_log_numlogsegments{topic=\"$topic\", env=\"$env\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Segment Files Per Broker", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_broker_topic_metrics_messagesinpersec_rate{topic=\"$topic\", env=\"$env\", aggregate=\"OneMinute\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Messages In Per Sec (OneMinuteRate)", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "wps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_broker_topic_metrics_bytesinpersec_rate{topic=\"$topic\", env=\"$env\", aggregate=\"OneMinute\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "BytesInPerSec (OneMinuteRate)", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_server_broker_topic_metrics_bytesoutpersec_rate{topic=\"$topic\", env=\"$env\", aggregate=\"OneMinute\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "BytesOutPerSec (OneMinuteRate)", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kafka", + "topics" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "env", + "options": [], + "query": "label_values(kafka_log_size, env)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Topics", + "multi": true, + "name": "topic", + "options": [], + "query": "label_values(kafka_log_size, topic)", + "refresh": 2, + "regex": "", + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kafka Topics / Logs", + "uid": "1vR8sjAmk", + "version": 1 +} \ No newline at end of file diff --git a/modules/infra-kafka-cluster/src/config/grafana/provisioning/dashboards/kafka.yaml b/modules/infra-kafka-cluster/src/config/grafana/provisioning/dashboards/kafka.yaml new file mode 100644 index 0000000..2053d55 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/provisioning/dashboards/kafka.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: +- name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards diff --git a/modules/infra-kafka-cluster/src/config/grafana/provisioning/datasources/datasource.yaml b/modules/infra-kafka-cluster/src/config/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 0000000..a8984ad --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,50 @@ +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# what's available in the database +datasources: + # name of the datasource. Required +- name: Prometheus + # datasource type. Required + type: prometheus + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 +# # database password, if used +# password: +# # database user, if used +# user: +# # database name, if used +# database: +# # enable/disable basic auth +# basicAuth: +# # basic auth username +# basicAuthUser: +# # basic auth password +# basicAuthPassword: +# # enable/disable with credentials headers +# withCredentials: + # mark as default datasource. Max one per org + isDefault: true +# # fields that will be converted to json and stored in json_data +# jsonData: +# graphiteVersion: "1.1" +# tlsAuth: true +# tlsAuthWithCACert: true +# # json object of data that will be encrypted. +# secureJsonData: +# tlsCACert: "..." +# tlsClientCert: "..." +# tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true diff --git a/modules/infra-kafka-cluster/src/config/jmx_exporter_kafka_broker.yml b/modules/infra-kafka-cluster/src/config/jmx_exporter_kafka_broker.yml new file mode 100644 index 0000000..da74442 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/jmx_exporter_kafka_broker.yml @@ -0,0 +1,505 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#--------------------------------------------------------------------------------------------------------- +# Kafka Broker JMX Metrics : +# Compatible with version Apache Kafka v2.x.x / Confluent Platform 5.x.x +#--------------------------------------------------------------------------------------------------------- +hostPort: kafka:9991 +lowercaseOutputName: true +lowercaseOutputLabelNames: true + +#Whitelist is used to reduce scrapping time +whitelistObjectNames: + - java.lang:* + - kafka.cluster:* + - kafka.controller:* + - kafka.log:* + - kafka.server:type=KafkaServer,name=BrokerState + - kafka.server:type=KafkaRequestHandlerPool,* + - kafka.server:type=BrokerTopicMetrics,* +# - kafka.server:type=DelayedFetchMetrics,* +# - kafka.server:type=DelayedOperationPurgatory,* + - kafka.server:type=FetcherLagMetrics,* + - kafka.server:type=FetcherStats,* + - kafka.server:type=Request,* + - kafka.server:type=Fetch,* + - kafka.server:type=Produce,* + - kafka.server:type=ReplicaManager,* + - kafka.server:type=ReplicaFetcherManager,* + - kafka.server:type=SessionExpireListener,* + - kafka.server:type=controller-channel-metrics,* + - kafka.server:type=socket-server-metrics,* + - kafka.network:type=RequestChannel,* + - kafka.network:type=Processor,* + - kafka.network:type=SocketServer,* + - kafka.network:type=RequestMetrics,* +# - kafka.network:type=RequestMetrics,name=*,request=OffsetCommit,version=* +# - kafka.network:type=RequestMetrics,name=*,request=FetchConsumer,version=* +# - kafka.network:type=RequestMetrics,name=*,request=FetchFollower,version=* +# - kafka.network:type=RequestMetrics,name=*,request=Produce,version=* +# - kafka.network:type=RequestMetrics,name=*,request=Metadata,version=* + - kafka.coordinator.group:* +blacklistObjectNames: + - java.lang:type=ClassLoading,* + - java.lang:type=Compilation,* + - java.lang:type=MemoryManager,* + - kafka.utils:* + - kafka.controller:type=ControllerChannelManager,name=QueueSize,* + # Following metrics are exposed per topics + - kafka.log:type=Log,name=LogEndOffset,* + - kafka.log:type=Log,name=LogStartOffset,* + - kafka.cluster:type=Partition,name=InSyncReplicasCount,* + - kafka.cluster:type=Partition,name=LastStableOffsetLag,* + - kafka.cluster:type=Partition,name=ReplicasCounts,* + - kafka.cluster:type=Partition,name=UnderReplicated,* + - kafka.server:type=BrokerTopicMetrics,name=TotalFetchRequestsPerSec,* + - kafka.server:type=BrokerTopicMetrics,name=TotalProduceRequestsPerSec,* + - kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec,* + - kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec,* + - kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec,* + +rules: + #------------------------------------------------------------------------------------------------------- + # KafkaServers : State of broker server + # + # - BrokerState + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>Value + name: kafka_server_brokerstate + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Partition : Number of partitions for each broker + # - InSyncReplicasCount + # - LastStableOffsetLag + # - ReplicasCount + # - UnderReplicated + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.cluster<>Value + name: kafka_cluster_partition_$1 + labels: + topic: $2 + partition: $3 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaController : + # + # - ActiveControllerCount, OfflinePartitionsCount, PreferredReplicaImbalanceCount + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.controller<>Value + name: kafka_controller_$1 + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # ControllerStats : The event that is currently being processed by the elected broker controller. + # + # - LeaderElectionRateAndtimeMs, UncleanLeaderElectionsPerSec, AutoLeaderBalanceRateAndTimeMs, ManualLeaderBalanceRateAndTimeMs + # - ControllerChangeRateAndTimeMs, + # - TopicChangeRateAndTimeMs, TopicDeletionRateAndTimeMs, PartitionReassignmentRateAndTimeMs + # - IsrChangeRateAndTimeMs + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.controller<>(OneMinuteRate|Mean|75thPercentile|99thPercentile) + name: kafka_controller_stats_$1 + labels: + aggregate: $2 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # Coordinator : GroupMetadataManager + # + # - NumGroups, NumOffsets + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.coordinator.group<>(Value) + name: kafka_coordinator_group_metadata_manager_$1 + labels: + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # Logs : + # + # - LogEndOffset, LogStartOffset, NumLogSegments, Size + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.log<>Value + name: kafka_log_$1 + labels: + topic: $2 + partition: $3 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # LogCleaner : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.log<>(Value) + name: kafka_log_cleaner_recopy_percent + labels: + service: kafka-broker + env: kafkatorio + + - pattern: kafka.log<>(Value) + name: kafka_log_cleaner_max_clean_time_secs + labels: + service: kafka-broker + env: kafkatorio + + - pattern: kafka.log<>(Value) + name: kafka_log_cleaner_max_buffer_utilization_percent + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # LogCleanerManager : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.log<>(Value) + name: kafka_log_cleaner_manager_max_dirty_percent + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # LogFlushStats : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.log<>(\w+) + name: kafka_log_flush_stats_rate_and_time_ms + labels: + aggregate: $1 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaRequestHandlerPool : Latency + # + # - KafkaRequestHandlerPool + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(\w+) + name: kafka_server_request_handler_avg_idle_percent + labels: + aggregate: $1 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Network Socket Server : Latency + # + # - NetworkProcessorAvgIdlePercent + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.network<>(Value) + name: kafka_network_socket_server_processor_avg_idle_percent + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Network Processor : Latency + # + # - IdlePercent + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.network<>(Value) + name: kafka_network_processor_idle_percent + labels: + processor: $1 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Network KafkaRequestChannel : + # + # - RequestQueueSize, ResponseQueueSize + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.network<>Value + name: kafka_network_request_channel_queue_size + labels: + queue: $1 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Network KafkaRequest : + # + # - RequestPerSec, + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.network<>(OneMinuteRate|Mean) + name: kafka_network_request_per_sec + labels: + request: $1 + version: $2 + aggregate: $3 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Network KafkaRequestMetrics : + # + # - LocalTimeMs, RemoteTimeMs, + # - RequestQueueTimeMs, + # - ResponseQueueTimeMs, ResponseSendTimeMs + # - ThrottleTimeMs + # - TotalTimeMs + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.network<>(OneMinuteRate|Mean|75thPercentile|99thPercentile) + name: kafka_network_request_metrics_time_ms + labels: + scope: $1 + request: $2 + aggregate: $3 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # KafkaServer / BrokerTopicMetrics - I/O metrics : + # + # - BytesInPerSec, BytesOutPerSec, BytesRejectedPerSec, + # - FailedFetchRequestsPerSec, FailedProduceRequestsPerSec,MessagesInPerSec, + # - TotalFetchRequestPerSec, TotalProduceRequestPerSec, ReplicationBytesInPerSec, ReplicationBytesOutPerSec + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(OneMinute)Rate + name: kafka_server_broker_topic_metrics_$1_rate + labels: + aggregate: $2 + service: kafka-broker + env: kafkatorio + + - pattern: kafka.server<>(OneMinute)Rate + name: kafka_server_broker_topic_metrics_$1_rate + labels: + topic: $2 + aggregate: $3 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaServer / DelayedFetchMetrics : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>([^,]+)Rate + name: kafka_server_delayed_fetch_expires_per_sec + labels: + fetcher_type: $1 + aggregate: $2 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaServer / DelayedOperationPurgatory : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>Value + name: kafka_server_delayed_operation_purgatory_$1 + labels: + operation: $2 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # FetcherLagMetrics : Lag in number of messages per follower replica + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>Value + name: kafka_server_fetcher_lag_$1 + labels: + client_id: $2 + topic: $3 + partition: $4 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # FetcherStats : Replica Fetcher Thread stats + # - BytesPerSec / RequestsPerSec + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>([^,]+)Rate + name: kafka_server_fetcher_stats_$1 + labels: + client_id: $2 + broker_host: $3 + broker_port: $4 + aggregate: $5 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaServer Request : + # - request-time - Tracking request-time per user/client-id + # - throttle-time - Tracking average throttle-time per user/client-id + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(request-time|throttle-time) + name: kafka_server_request_$2 + labels: + client_id: $1 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # KafkaServer Fetcher/Producer : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(byte-rate|throttle-time) + name: kafka_server_fetch_client_$2 + labels: + client_id: $1 + service: kafka-broker + env: kafkatorio + + - pattern: kafka.server<>(byte-rate|throttle-time) + name: kafka_server_produce_client_$2 + labels: + client_id: $1 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # ReplicaManager : + # - IsrExpandsPerSec, IsrShrinksPerSec, FailedIsrUpdatesPerSec + # - LeaderCount, PartitionCount, UnderReplicatedPartitions) + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>([^,]+)Rate + name: kafka_server_replica_manager_$1 + labels: + aggregate: $2 + service: kafka-broker + env: kafkatorio + + - pattern: kafka.server<>(Value) + name: kafka_server_replica_manager_$1 + labels: + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # ReplicaFetcherManager : + # - MaxLag, MinFetchRate + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(Value) + name: kafka_server_replica_fetcher_manager_$1_value + labels: + client_id: $2 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # Zookeeper / SessionExpireListener : + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>([^,]+)Rate + name: kafka_zookeeper_session_expire_listener_$1 + labels: + aggregate: $2 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # ControllerChannelMetrics: + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(\w*) + name: kafka_server_controller_channel_metrics_$2 + labels: + broker_id: $1 + service: kafka-broker + env: kafkatorio + + #------------------------------------------------------------------------------------------------------- + # KafkaServer / Socket Server Metrics + #------------------------------------------------------------------------------------------------------- + - pattern: kafka.server<>(\w*) + name: kafka_server_socket_server_metrics_$2 + labels: + network_processor: $1 + service: kafka-broker + env: kafkatorio + #------------------------------------------------------------------------------------------------------- + # + # Broker / JVM + #------------------------------------------------------------------------------------------------------- + # JVM GarbageCollector + # + - pattern: 'java.lang<>CollectionCount' + name: kafka_jvm_gc_collection_count + labels: + name: $1 + service: kafka-broker + env: kafkatorio + + - pattern: 'java.lang<>CollectionTime' + name: kafka_jvm_gc_collection_time + labels: + name: $1 + service: kafka-broker + env: kafkatorio + + - pattern: java.lang + name: kafka_jvm_last_gc_duration + labels: + name: $1 + service: kafka-broker + env: kafkatorio + attrNameSnakeCase: true + + - pattern: 'java.lang(\w+)' + name: kafka_jvm_last_gc_memory_usage_$4 + labels: + name: $1 + space: $2 + type: $4 + service: kafka-broker + env: kafkatorio + attrNameSnakeCase: true + + # JVM Memory + - pattern: java.lang(\w*) + name: kafka_jvm_heap_usage + labels: + type: $1 + service: kafka-broker + env: kafkatorio + attrNameSnakeCase: true + + - pattern: java.lang(\w*) + name: kafka_jvm_non_heap_usage + labels: + type: $1 + service: kafka-broker + env: kafkatorio + attrNameSnakeCase: true + + - pattern: 'java.lang(\w*)' + name: kafka_jvm_memory_pool_collection_usage + labels: + name: $1 + type: $2 + service: kafka-broker + env: kafkatorio + - pattern: 'java.lang(\w*)' + name: kafka_jvm_memory_pool_usage + labels: + name: $1 + type: $2 + service: kafka-broker + env: kafkatorio + - pattern: 'java.lang(\w*)' + name: kafka_jvm_memory_pool_peak_usage + labels: + name: $1 + type: $2 + service: kafka-broker + env: kafkatorio + + # JVM Thread + - pattern: java.lang<>(\w*thread_count) + name: kafka_jvm_$1 + labels: + service: kafka-broker + env: kafkatorio + attrNameSnakeCase: true + + # Operating System + - pattern: java.lang<>(\w*) + name: kafka_jvm_os_$1 + labels: + service: kafka-broker + env: kafkatorio diff --git a/modules/infra-kafka-cluster/src/config/prometheus.yml b/modules/infra-kafka-cluster/src/config/prometheus.yml new file mode 100644 index 0000000..dae03d5 --- /dev/null +++ b/modules/infra-kafka-cluster/src/config/prometheus.yml @@ -0,0 +1,29 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: +# - "first_rules.yml" +# - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'kafka' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: [ 'jmx-kafka:5556' ] diff --git a/modules/infra-kafka-cluster/src/docker-compose.yml b/modules/infra-kafka-cluster/src/docker-compose.yml index 1d0d4b5..c881589 100644 --- a/modules/infra-kafka-cluster/src/docker-compose.yml +++ b/modules/infra-kafka-cluster/src/docker-compose.yml @@ -12,6 +12,13 @@ services: target: "kafka-kraft" args: KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.1.0/kafka_2.13-3.1.0.tgz" + volumes: + - "./kafka-server.properties:/kafka/server.properties" + environment: + JMX_PORT: "9991" +# KAFKA_JMX_OPTS: "9991" + KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=9991" + networks: - "kafka-nw" - "kafkatorio-nw" @@ -49,6 +56,48 @@ services: # - "8083:8083" # + grafana: + image: "grafana/grafana:${GRAFANA_VERSION}" + container_name: "grafana" + ports: + - "3000:3000" + environment: + GF_PATHS_DATA : /var/lib/grafana + GF_SECURITY_ADMIN_PASSWORD : kafka + volumes: + - "./config/grafana/provisioning:/etc/grafana/provisioning" + - "./config/grafana/dashboards:/var/lib/grafana/dashboards" + depends_on: + - prometheus + networks: + - "kafka-nw" + + prometheus: + image: "prom/prometheus:${PROMETHEUS_VERSION}" + container_name: "prometheus" + ports: + - "9090:9090" + volumes: + - "./config/prometheus.yml:/etc/prometheus/prometheus.yml" + command: "--config.file=/etc/prometheus/prometheus.yml" + networks: + - "kafka-nw" + + jmx-kafka: + image: "sscaling/jmx-prometheus-exporter" + container_name: "jmx-kafka" + ports: + - "5556:5556" + environment: + CONFIG_YML : "/etc/jmx_exporter/config.yml" +# JVM_OPTS: "-XX:MaxRAMPercentage=70.0" + volumes: + - "./config/jmx_exporter_kafka_broker.yml:/etc/jmx_exporter/config.yml" + depends_on: + - kafka + networks: + - "kafka-nw" + networks: kafka-nw: name: "kafka-nw"