metrics.yaml

# Metrics collected by this check. You should not have to modify this.
jmx_metrics:
  #
  # Producers (only v0.8.2.x)
  #
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.producer.request_rate
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.producer.request_latency_avg
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=BytesPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.producer.bytes_out
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=ProducerTopicMetrics,name=MessagesPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.producer.message_rate

    #
    # Producers (v0.9.0.x to v0.10.2.x)
    #
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
        attribute:
          response-rate:
            metric_type: gauge
            alias: kafka.producer.response_rate
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
        attribute:
          request-rate:
            metric_type: gauge
            alias: kafka.producer.request_rate
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
        attribute:
          request-latency-avg:
            metric_type: gauge
            alias: kafka.producer.request_latency_avg
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
        attribute:
          outgoing-byte-rate:
            metric_type: gauge
            alias: kafka.producer.bytes_out
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
        attribute:
          io-wait-time-ns-avg:
            metric_type: gauge
            alias: kafka.producer.io_wait

    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-metrics,client-id=([-.\w]+)'
        attribute:
          waiting-threads:
            metric_type: gauge
            alias: kafka.producer.waiting_threads
          buffer-total-bytes:
            metric_type: gauge
            alias: kafka.producer.buffer_bytes_total
          buffer-available-bytes:
            metric_type: gauge
            alias: kafka.producer.available_buffer_bytes
          bufferpool-wait-time:
            metric_type: gauge
            alias: kafka.producer.bufferpool_wait_time
          batch-size-avg:
            metric_type: gauge
            alias: kafka.producer.batch_size_avg
          batch-size-max:
            metric_type: gauge
            alias: kafka.producer.batch_size_max
          compression-rate-avg:
            metric_type: gauge
            alias: kafka.producer.compression_rate_avg
          record-queue-time-avg:
            metric_type: gauge
            alias: kafka.producer.record_queue_time_avg
          record-queue-time-max:
            metric_type: gauge
            alias: kafka.producer.record_queue_time_max
          request-latency-avg:
            metric_type: gauge
            alias: kafka.producer.request_latency_avg
          request-latency-max:
            metric_type: gauge
            alias: kafka.producer.request_latency_max
          record-send-rate:
            metric_type: gauge
            alias: kafka.producer.records_send_rate
          records-per-request-avg:
            metric_type: gauge
            alias: kafka.producer.records_per_request
          record-retry-rate:
            metric_type: gauge
            alias: kafka.producer.record_retry_rate
          record-error-rate:
            metric_type: gauge
            alias: kafka.producer.record_error_rate
          record-size-max:
            metric_type: gauge
            alias: kafka.producer.record_size_max
          record-size-avg:
            metric_type: gauge
            alias: kafka.producer.record_size_avg
          requests-in-flight:
            metric_type: gauge
            alias: kafka.producer.requests_in_flight
          metadata-age:
            metric_type: gauge
            alias: kafka.producer.metadata_age
          produce-throttle-time-max:
            metric_type: gauge
            alias: kafka.producer.throttle_time_max
          produce-throttle-time-avg:
            metric_type: gauge
            alias: kafka.producer.throttle_time_avg


    #
    # Producers: Per Topic Metrics
    #
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          byte-rate:
            metric_type: gauge
            alias: kafka.producer.bytes_out
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          record-send-rate:
            metric_type: gauge
            alias: kafka.producer.record_send_rate
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          compression-rate:
            metric_type: gauge
            alias: kafka.producer.compression_rate
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          record-retry-rate:
            metric_type: gauge
            alias: kafka.producer.record_retry_rate
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.producer'
        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          record-error-rate:
            metric_type: gauge
            alias: kafka.producer.record_error_rate
        tags:
          client: $1
          topic: $2

    #
    # Consumers (only v0.8.2.x)
    #
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=.*'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.consumer.max_lag
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MinFetchRate,clientId=.*'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.consumer.fetch_rate
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.consumer.bytes_in
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ConsumerTopicMetrics,name=MessagesPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.consumer.messages_in
    - include:
        # Offsets committed to ZooKeeper
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=ZooKeeperCommitsPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.consumer.zookeeper_commits
    - include:
        # Offsets committed to Kafka
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=KafkaCommitsPerSec,clientId=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.consumer.kafka_commits

    #
    # Consumers (v0.9.0.x to v0.10.2.x)
    #
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*'
        attribute:
          bytes-consumed-rate:
            metric_type: gauge
            alias: kafka.consumer.bytes_in
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*'
        attribute:
          records-consumed-rate:
            metric_type: gauge
            alias: kafka.consumer.messages_in

    #
    # Consumers: Per Topic Metrics
    #

    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          fetch-size-avg:
            metric_type: gauge
            alias: kafka.consumer.fetch_size_avg
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          fetch-size-max:
            metric_type: gauge
            alias: kafka.consumer.fetch_size_max
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          bytes-consumed-rate:
            metric_type: gauge
            alias: kafka.consumer.bytes_consumed
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          records-per-request-avg:
            metric_type: gauge
            alias: kafka.consumer.records_per_request_avg
        tags:
          client: $1
          topic: $2
    - include:
        domain: 'kafka.consumer'
        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
        attribute:
          records-consumed-rate:
            metric_type: gauge
            alias: kafka.consumer.records_consumed
        tags:
          client: $1
          topic: $2

    #
    # Aggregate cluster stats
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.net.bytes_out.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.net.bytes_in.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.messages_in.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.net.bytes_rejected.rate

    #
    # Per Topic Broker Stats (only v0.8.x)
    #
    - include:
        domain: '"kafka.server"'
        bean_regex: '"kafka.server":type="BrokerTopicMetrics",name="(.*?)-MessagesInPerSec"'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.topic.messages_in.rate
        tags:
          topic: $1

    #
    # Brokers: Per Topic Metrics (v0.9.0.x+)
    #
    - include:
        domain: 'kafka.server'
        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.topic.messages_in.rate
    - include:
        domain: 'kafka.server'
        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.topic.net.bytes_out.rate
    - include:
        domain: 'kafka.server'
        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.topic.net.bytes_in.rate
    - include:
        domain: 'kafka.server'
        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec,topic=.*'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.topic.net.bytes_rejected.rate

    #
    # Request timings
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.request.fetch.failed.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.request.produce.failed.rate
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.request.produce.rate
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.produce.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.produce.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.request.fetch_consumer.rate
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.request.fetch_follower.rate
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.fetch_consumer.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.fetch_consumer.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.fetch_follower.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.fetch_follower.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.update_metadata.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.update_metadata.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.metadata.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.metadata.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets'
        attribute:
          Mean:
            metric_type: gauge
            alias: kafka.request.offsets.time.avg
          99thPercentile:
            metric_type: gauge
            alias: kafka.request.offsets.time.99percentile
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=RequestChannel,name=RequestQueueSize'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.request.channel.queue.size
    - include:
        domain: 'kafka.network'
        bean: 'kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.net.processor.avg.idle.pct.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
        attribute:
          OneMinuteRate:
            metric_type: gauge
            alias: kafka.request.handler.avg.idle.pct.rate

    #
    # Request Purgatory (only v0.8.2.x)
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.request.producer_request_purgatory.size
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=FetchRequestPurgatory,name=PurgatorySize'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.request.fetch_request_purgatory.size

    #
    # Request Purgatory (v0.9.0.x onwards)
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Produce'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.request.producer_request_purgatory.size
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Fetch'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.request.fetch_request_purgatory.size

    #
    # Replication stats
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.under_replicated_partitions
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.replication.isr_shrinks.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.replication.isr_expands.rate
    - include:
        domain: 'kafka.controller'
        bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.replication.leader_elections.rate
    - include:
        domain: 'kafka.controller'
        bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.replication.unclean_leader_elections.rate
    - include:
        domain: 'kafka.controller'
        bean: 'kafka.controller:type=KafkaController,name=OfflinePartitionsCount'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.offline_partitions_count
    - include:
        domain: 'kafka.controller'
        bean: 'kafka.controller:type=KafkaController,name=ActiveControllerCount'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.active_controller_count
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaManager,name=PartitionCount'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.partition_count
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaManager,name=LeaderCount'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.leader_count
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=ReplicaFetcherManager,name=MaxLag,clientId=Replica'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.replication.max_lag

    #
    # Log flush stats
    #
    - include:
        domain: 'kafka.log'
        bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.log.flush_rate.rate

    #
    # Zookeeper stats
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperDisconnectsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.session.zookeeper.disconnect.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperExpiresPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.session.zookeeper.expire.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperReadOnlyConnectsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.session.zookeeper.readonly.rate
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperSyncConnectsPerSec'
        attribute:
          Count:
            metric_type: rate
            alias: kafka.session.zookeeper.sync.rate

    #
    # Session stats
    #
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=FetchSessionCache,name=NumIncrementalFetchSessions'
        attribute:
          Value:
            metric_type: gauge
            alias: kafka.session.fetch.count
    - include:
        domain: 'kafka.server'
        bean: 'kafka.server:type=FetchSessionCache,name=IncrementalFetchSessionEvictionsPerSec'
        attribute:
          Count:
            alias: kafka.session.fetch.eviction
            metric_type: rate
    #
    # Listeners stats
    #
    - include:
        domain: 'kafka.server'
        bean_regex: 'kafka\.server:type=socket-server-metrics,listener=(.*?),networkProcessor=.*'
        attribute:
          connection-count:
            metric_type: gauge
            alias: kafka.server.socket.connection_count
        tags:
          listener: $1