zabbix_export:
  version: '7.0'
  template_groups:
    - uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6
      name: Templates/Applications
  templates:
    - uuid: b25b8b517a4743c48037bfa10af3dc3c
      template: 'Etcd by HTTP'
      name: 'Etcd by HTTP'
      description: |
        This template is designed to monitor 'etcd' by Zabbix that works without any external scripts.
        Most of the metrics are collected in one go, thanks to Zabbix bulk data collection.
        
        The template 'Etcd by HTTP' — collects metrics by help of the HTTP agent from '/metrics' endpoint.
        
        Refer to the vendor documentation:
        https://etcd.io/docs/v3.5/op-guide/monitoring/#metrics-endpoint
        
        For the users of 'etcd version <= 3.4':
        
        In 'etcd v3.5' some metrics have been deprecated. See more details on 'Upgrade etcd from 3.4 to 3.5':
        https://etcd.io/docs/v3.4/upgrades/upgrade_3_5/
        Please upgrade your 'etcd' instance, or use older 'Etcd by HTTP' template version.
        
        Setup:
        
        1. Make sure that 'etcd' allows the collection of metrics. You can test it by running:
        curl -L http://localhost:2379/metrics
        
        2. Check if 'etcd' is accessible from Zabbix proxy or Zabbix server depending on where you are planning to do the monitoring. To verify it, run:
        curl -L  http://<etcd_node_address>:2379/metrics
        
        3. Add the template to the 'etcd' node. Set the hostname or IP address of the 'etcd' host in the '{$ETCD.HOST}' macro. By default, the template uses a client's port.
        You can configure metrics endpoint location by adding '--listen-metrics-urls' flag.
        
        For more details, see the etcd documentation:
        https://etcd.io/docs/v3.5/op-guide/configuration/#profiling-and-monitoring
        
        Additional points to consider:
        
        - If you have specified a non-standard port for 'etcd', don't forget to change macros: '{$ETCD.SCHEME}' and '{$ETCD.PORT}'.
        - You can set '{$ETCD.USERNAME}' and '{$ETCD.PASSWORD}' macros in the template to use on a host level if necessary.
        - To test availability, run : 'zabbix_get -s etcd-host -k etcd.health'.
        
        Generated by official Zabbix template tool "Templator"
      vendor:
        name: Zabbix
        version: 7.0-0
      groups:
        - name: Templates/Applications
      items:
        - uuid: 32a59c8e93e141d6a471266df6dbfbd3
          name: 'Cluster version'
          type: DEPENDENT
          key: etcd.cluster.version
          value_type: CHAR
          description: 'The version of the `etcd cluster`.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $.etcdcluster
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1d
          master_item:
            key: etcd.get_version
          tags:
            - tag: component
              value: application
          triggers:
            - uuid: 7c87424c4fb34c56bc9b976755b4ec02
              expression: 'last(/Etcd by HTTP/etcd.cluster.version,#1)<>last(/Etcd by HTTP/etcd.cluster.version,#2) and length(last(/Etcd by HTTP/etcd.cluster.version))>0'
              name: 'Cluster version has changed'
              event_name: 'Cluster version has changed (new version: {ITEM.VALUE})'
              priority: INFO
              description: 'Etcd version has changed. Acknowledge to close the problem manually.'
              manual_close: 'YES'
              tags:
                - tag: scope
                  value: notice
        - uuid: 04b0fa552b7d4267b4c5b67ee82ef5f1
          name: CPU
          type: DEPENDENT
          key: etcd.cpu.util
          value_type: FLOAT
          units: s
          description: 'The total user and system CPU time spent in seconds.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_cpu_seconds_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: cpu
        - uuid: bfa5bd42637642808802f7b2485a0c4d
          name: 'DB size'
          type: DEPENDENT
          key: etcd.db.size
          value_type: FLOAT
          units: B
          description: 'The total size of the underlying database.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_mvcc_db_total_size_in_bytes
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: database
        - uuid: a5bc7ffb090641ab92f537b38b6055e5
          name: 'Deletes per second'
          type: DEPENDENT
          key: etcd.delete.rate
          value_type: FLOAT
          units: rps
          description: 'The number of deletes seen by this member per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_mvcc_delete_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: operations
        - uuid: 183843bd93f84dc887a03fb638b2d323
          name: 'Pending events'
          type: DEPENDENT
          key: etcd.events.sent.rate
          description: 'The total number of pending events to be sent.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_mvcc_pending_events_total
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: events
        - uuid: 34ffab33275a400ab88e5217dee5ef96
          name: 'Get node metrics'
          type: HTTP_AGENT
          key: etcd.get_metrics
          history: '0'
          value_type: TEXT
          authtype: BASIC
          username: '{$ETCD.USER}'
          password: '{$ETCD.PASSWORD}'
          url: '{$ETCD.SCHEME}://{$ETCD.HOST}:{$ETCD.PORT}/metrics'
          tags:
            - tag: component
              value: raw
        - uuid: fdf7593420ab42b2a5af8f8b8030b517
          name: 'Get version'
          type: HTTP_AGENT
          key: etcd.get_version
          history: '0'
          value_type: TEXT
          url: '{$ETCD.SCHEME}://{$ETCD.HOST}:{$ETCD.PORT}/version'
          tags:
            - tag: component
              value: application
        - uuid: d23baf75628043e193ba0a607e1b4215
          name: 'RPCs received per second'
          type: DEPENDENT
          key: etcd.grpc.received.rate
          value_type: FLOAT
          units: rps
          description: 'The number of RPC stream messages received on the server.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - grpc_server_msg_received_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: eda81182710e47e1b5f2f21bb05b4775
          name: 'RPCs sent per second'
          type: DEPENDENT
          key: etcd.grpc.sent.rate
          value_type: FLOAT
          units: rps
          description: 'The number of gRPC stream messages sent by the server.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - grpc_server_msg_sent_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: bf59a130b20d480d93eb9330750e8e28
          name: 'RPCs started per second'
          type: DEPENDENT
          key: etcd.grpc.started.rate
          value_type: FLOAT
          units: rps
          description: 'The number of RPCs started on the server.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - grpc_server_started_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: e03575f4c472410eb6fbcf731ac6aab2
          name: 'Server has a leader'
          type: DEPENDENT
          key: etcd.has.leader
          description: |
            It defines - whether or not a leader exists:
            1 - it exists;
            0 - it does not.
          valuemap:
            name: 'Etcd leader'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_has_leader
                - value
                - ''
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 10m
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: leader
          triggers:
            - uuid: 20165719d521453bb239d818ac57805c
              expression: 'last(/Etcd by HTTP/etcd.has.leader)=0'
              name: 'Member has no leader'
              priority: AVERAGE
              description: 'If a member does not have a leader, it is totally unavailable.'
              tags:
                - tag: scope
                  value: availability
        - uuid: 3fde4db8b9684ba4b56ba915e48957b5
          name: 'Node health'
          type: HTTP_AGENT
          key: etcd.health
          authtype: BASIC
          username: '{$ETCD.USER}'
          password: '{$ETCD.PASSWORD}'
          valuemap:
            name: 'Etcd healthcheck'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $.health
            - type: BOOL_TO_DECIMAL
              error_handler: CUSTOM_VALUE
              error_handler_params: '0'
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 10m
          url: '{$ETCD.SCHEME}://{$ETCD.HOST}:{$ETCD.PORT}/health'
          tags:
            - tag: component
              value: health
          triggers:
            - uuid: 6acda7bdc9df4a4ab5b7cca76c6369f0
              expression: 'last(/Etcd by HTTP/etcd.health)=0'
              name: 'Node healthcheck failed'
              opdata: 'Current state: {ITEM.LASTVALUE1}'
              priority: AVERAGE
              description: 'See more details on https://etcd.io/docs/v3.5/op-guide/monitoring/#health-check.'
              dependencies:
                - name: 'Service is unavailable'
                  expression: 'last(/Etcd by HTTP/net.tcp.service["{$ETCD.SCHEME}","{$ETCD.HOST}","{$ETCD.PORT}"])=0'
              tags:
                - tag: scope
                  value: availability
        - uuid: 923a408dd4514e808b6e2137a94f8140
          name: 'HTTP 4XX'
          type: DEPENDENT
          key: etcd.http.requests.4xx.rate
          value_type: FLOAT
          units: rps
          description: 'The number of handled failures of requests (non-watches), by the method (`GET/PUT` etc.), and the code `4XX`.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - 'etcd_http_failed_total{code=~"4.+"}'
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: http
            - tag: http-code
              value: 4xx
        - uuid: c0f27d4bfba344079a31ce8c10b22683
          name: 'HTTP 5XX'
          type: DEPENDENT
          key: etcd.http.requests.5xx.rate
          value_type: FLOAT
          units: rps
          description: 'The number of handled failures of requests (non-watches), by the method (`GET/PUT` etc.), and the code `5XX`.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - 'etcd_http_failed_total{code=~"5.+"}'
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: http
            - tag: http-code
              value: 5xx
          triggers:
            - uuid: 0302957e0f6b43389546e1cfb958ed9c
              expression: 'min(/Etcd by HTTP/etcd.http.requests.5xx.rate,5m)>{$ETCD.HTTP.FAIL.MAX.WARN}'
              name: 'Too many HTTP requests failures'
              event_name: 'Too many HTTP requests failures (over {$ETCD.HTTP.FAIL.MAX.WARN} for 5m)'
              priority: WARNING
              description: 'Too many requests failed on `etcd` instance with the `5xx HTTP code`.'
              tags:
                - tag: scope
                  value: availability
        - uuid: 2a19db1c58ee4a509061fcb1b557c1a3
          name: 'HTTP requests received'
          type: DEPENDENT
          key: etcd.http.requests.rate
          value_type: FLOAT
          units: rps
          description: 'The number of requests received into the system (successfully parsed and `authd`).'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - etcd_http_received_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: http
        - uuid: b3760811472440baad6a338f481ba13a
          name: 'Server is a leader'
          type: DEPENDENT
          key: etcd.is.leader
          description: |
            It defines - whether or not this member is a leader:
            1 - it is;
            0 - otherwise.
          valuemap:
            name: 'Etcd leader'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_is_leader
                - value
                - ''
              error_handler: CUSTOM_VALUE
              error_handler_params: '0'
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 10m
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: leader
          triggers:
            - uuid: 44d66eb1a332418daf4c3a1110db5458
              expression: 'nodata(/Etcd by HTTP/etcd.is.leader,30m)=1'
              name: 'Failed to fetch info data'
              event_name: 'Failed to fetch info data (or no data for 30m)'
              priority: WARNING
              description: 'Zabbix has not received any data for items for the last 30 minutes.'
              manual_close: 'YES'
              dependencies:
                - name: 'Service is unavailable'
                  expression: 'last(/Etcd by HTTP/net.tcp.service["{$ETCD.SCHEME}","{$ETCD.HOST}","{$ETCD.PORT}"])=0'
              tags:
                - tag: scope
                  value: notice
        - uuid: ecd1ae9c038f4fc2b720ad562ced0191
          name: 'Keys compacted per second'
          type: DEPENDENT
          key: etcd.keys.compacted.rate
          value_type: FLOAT
          description: 'The number of DB keys compacted per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_mvcc_db_compaction_keys_total
                - value
                - ''
              error_handler: CUSTOM_VALUE
              error_handler_params: '0'
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: keys
        - uuid: a3f910efb0a04cc494c07b8703f9d2ec
          name: 'Keys expired per second'
          type: DEPENDENT
          key: etcd.keys.expired.rate
          value_type: FLOAT
          description: 'The number of expired keys per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_store_expires_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: keys
        - uuid: fbda737014544cf1bcf544a48aa6e48b
          name: 'Keys total'
          type: DEPENDENT
          key: etcd.keys.total
          description: 'The total number of keys.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_mvcc_keys_total
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: keys
        - uuid: e45ba61d99b8432b86f5797a2cfdb416
          name: 'Leader changes'
          type: DEPENDENT
          key: etcd.leader.changes
          description: 'The number of leader changes the member has seen since its start.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_leader_changes_seen_total
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: leader
          triggers:
            - uuid: 1ea623927179489890a5a73eeb8177f7
              expression: '(max(/Etcd by HTTP/etcd.leader.changes,15m)-min(/Etcd by HTTP/etcd.leader.changes,15m))>{$ETCD.LEADER.CHANGES.MAX.WARN}'
              name: 'Instance has seen too many leader changes'
              event_name: 'Instance has seen too many leader changes (over {$ETCD.LEADER.CHANGES.MAX.WARN} for 15m)'
              priority: WARNING
              description: 'Rapid leadership changes impact the performance of `etcd` significantly. It also signals that the leader is unstable, perhaps due to network connectivity issues or excessive load hitting the `etcd cluster`.'
              tags:
                - tag: scope
                  value: availability
        - uuid: 348e15d2ec3a4bb88e2ca371f96c2f00
          name: 'Maximum open file descriptors'
          type: DEPENDENT
          key: etcd.max.fds
          value_type: FLOAT
          description: 'The Maximum number of open file descriptors.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_max_fds
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: fds
        - uuid: d016b8674ebd4251943f2e94b22f5ff2
          name: 'Client gRPC received bytes per second'
          type: DEPENDENT
          key: etcd.network.grpc.received.rate
          value_type: FLOAT
          units: Bps
          description: 'The number of bytes received from gRPC clients per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_network_client_grpc_received_bytes_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: e50d2d088c6448dbb3ecaeebc3b2b8f1
          name: 'Client gRPC sent bytes per second'
          type: DEPENDENT
          key: etcd.network.grpc.sent.rate
          value_type: FLOAT
          units: Bps
          description: 'The number of bytes sent from gRPC clients per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_network_client_grpc_sent_bytes_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: d5099d4cdb3044ba95935c2aea2b6352
          name: 'Open file descriptors'
          type: DEPENDENT
          key: etcd.open.fds
          value_type: FLOAT
          description: 'The number of open file descriptors.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_open_fds
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: fds
        - uuid: a2927b1e85af41cab9c28b1b79c229ea
          name: 'Proposals applied per second'
          type: DEPENDENT
          key: etcd.proposals.applied.rate
          value_type: FLOAT
          description: 'The number of consensus proposals applied.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_proposals_applied_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: proposals
        - uuid: e829f3df055e42dfbce5f27eb7ca487c
          name: 'Proposals committed per second'
          type: DEPENDENT
          key: etcd.proposals.committed.rate
          value_type: FLOAT
          description: 'The number of consensus proposals committed.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_proposals_committed_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: proposals
        - uuid: 6b45b99526394a219d31b5c22cb98c85
          name: 'Proposals failed per second'
          type: DEPENDENT
          key: etcd.proposals.failed.rate
          value_type: FLOAT
          description: 'The number of failed proposals seen.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_proposals_failed_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: proposals
          triggers:
            - uuid: 432cea8bddd742ca98528be2fbc7e35e
              expression: 'min(/Etcd by HTTP/etcd.proposals.failed.rate,5m)>{$ETCD.PROPOSAL.FAIL.MAX.WARN}'
              name: 'Too many proposal failures'
              event_name: 'Too many proposal failures (over {$ETCD.PROPOSAL.FAIL.MAX.WARN} for 5m)'''
              priority: WARNING
              description: 'Normally related to two issues: temporary failures related to a leader election or longer downtime caused by a loss of quorum in the cluster.'
              tags:
                - tag: scope
                  value: performance
        - uuid: 1c506ff69e7b4564a6d95fd35b1a11fd
          name: 'Proposals pending'
          type: DEPENDENT
          key: etcd.proposals.pending
          description: 'The current number of pending proposals to commit.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_server_proposals_pending
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: proposals
          triggers:
            - uuid: 5feefc4dd5d14fe2b56dd63029b57026
              expression: 'min(/Etcd by HTTP/etcd.proposals.pending,5m)>{$ETCD.PROPOSAL.PENDING.MAX.WARN}'
              name: 'Too many proposals are queued to commit'
              event_name: 'Too many proposals are queued to commit (over {$ETCD.PROPOSAL.PENDING.MAX.WARN} for 5m)'''
              priority: WARNING
              description: 'Rising pending proposals suggests there is a high client load, or the member cannot commit proposals.'
              tags:
                - tag: scope
                  value: performance
        - uuid: bd7398507c274bfab53339380df16761
          name: 'PUT per second'
          type: DEPENDENT
          key: etcd.put.rate
          value_type: FLOAT
          units: rps
          description: 'The number of puts seen by this member per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_mvcc_put_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: operations
        - uuid: b744c07f3290467b96b21ea38ad5d497
          name: 'Range per second'
          type: DEPENDENT
          key: etcd.range.rate
          value_type: FLOAT
          units: rps
          description: 'The number of ranges seen by this member per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_mvcc_range_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: operations
        - uuid: 88c91b36eca94fd2b357a67d171dc621
          name: 'Reads per second'
          type: DEPENDENT
          key: etcd.reads.rate
          value_type: FLOAT
          units: rps
          description: 'The number of read actions by `get/getRecursive`, local to this member.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - etcd_debugging_store_reads_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: store
        - uuid: 4b881e32094e4f478c5d0849cb5d07a7
          name: 'Resident memory'
          type: DEPENDENT
          key: etcd.res.bytes
          value_type: FLOAT
          units: B
          description: 'The size of resident memory expressed in bytes.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_resident_memory_bytes
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: memory
        - uuid: dee9ed8897cf4d3582957707ea09cdf8
          name: 'Server version'
          type: DEPENDENT
          key: etcd.server.version
          value_type: CHAR
          description: 'The version of the `etcd server`.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $.etcdserver
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1d
          master_item:
            key: etcd.get_version
          tags:
            - tag: component
              value: application
          triggers:
            - uuid: cfb2fc467b224ef694d59b5c081ed965
              expression: 'last(/Etcd by HTTP/etcd.server.version,#1)<>last(/Etcd by HTTP/etcd.server.version,#2) and length(last(/Etcd by HTTP/etcd.server.version))>0'
              name: 'Server version has changed'
              event_name: 'Server version has changed (new version: {ITEM.VALUE})'
              priority: INFO
              description: 'Etcd version has changed. Acknowledge to close the problem manually.'
              manual_close: 'YES'
              tags:
                - tag: scope
                  value: notice
        - uuid: b14c787c716146e990bc388d277a2803
          name: 'Transaction per second'
          type: DEPENDENT
          key: etcd.txn.rate
          value_type: FLOAT
          units: rps
          description: 'The number of transactions seen by this member per second.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - etcd_debugging_mvcc_range_total
                - value
                - ''
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: transactions
        - uuid: 98ec9085d621446aa462efc86cf93905
          name: Uptime
          type: DEPENDENT
          key: etcd.uptime
          value_type: FLOAT
          units: s
          description: '`Etcd` server uptime.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_start_time_seconds
                - value
                - ''
            - type: JAVASCRIPT
              parameters:
                - 'return (Math.floor(Date.now() / 1000) - Number(value));'
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: application
          triggers:
            - uuid: 6843369969f5410a840494104d71fe1f
              expression: 'last(/Etcd by HTTP/etcd.uptime)<10m'
              name: 'Host has been restarted'
              event_name: '{HOST.NAME} has been restarted (uptime < 10m)'
              priority: INFO
              description: 'Uptime is less than 10 minutes.'
              manual_close: 'YES'
              tags:
                - tag: scope
                  value: notice
        - uuid: c35810b8b7bc4a62970b5293fb2d8fb2
          name: 'Virtual memory'
          type: DEPENDENT
          key: etcd.virtual.bytes
          value_type: FLOAT
          units: B
          description: 'The size of virtual memory expressed in bytes.'
          preprocessing:
            - type: PROMETHEUS_PATTERN
              parameters:
                - process_virtual_memory_bytes
                - value
                - ''
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: memory
        - uuid: 16c041fc189248bfaaa5826ffaf38459
          name: 'Writes per second'
          type: DEPENDENT
          key: etcd.writes.rate
          value_type: FLOAT
          units: rps
          description: 'The number of writes (e.g., `set/compareAndDelete`) seen by this member.'
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - etcd_debugging_store_writes_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var valueArr = JSON.parse(value);
                  return valueArr.reduce(function (acc, obj) {
                  	return acc + parseFloat(obj['value']);
                  }, 0);
            - type: CHANGE_PER_SECOND
          master_item:
            key: etcd.get_metrics
          tags:
            - tag: component
              value: store
        - uuid: a0f94f429b99432e86d15ffa74d6eada
          name: 'Service''s TCP port state'
          type: SIMPLE
          key: 'net.tcp.service["{$ETCD.SCHEME}","{$ETCD.HOST}","{$ETCD.PORT}"]'
          valuemap:
            name: 'Service state'
          preprocessing:
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 10m
          tags:
            - tag: component
              value: health
            - tag: component
              value: network
          triggers:
            - uuid: 74164f0783ae4227ba44f3e865fee3bd
              expression: 'last(/Etcd by HTTP/net.tcp.service["{$ETCD.SCHEME}","{$ETCD.HOST}","{$ETCD.PORT}"])=0'
              name: 'Service is unavailable'
              priority: AVERAGE
              manual_close: 'YES'
              tags:
                - tag: scope
                  value: availability
      discovery_rules:
        - uuid: 5e6121383e5d4f3eb1150a2068a4633b
          name: 'gRPC codes discovery'
          type: DEPENDENT
          key: etcd.grpc_code.discovery
          filter:
            evaltype: AND
            conditions:
              - macro: '{#GRPC.CODE}'
                value: '{$ETCD.GRPC_CODE.MATCHES}'
              - macro: '{#GRPC.CODE}'
                value: '{$ETCD.GRPC_CODE.NOT_MATCHES}'
                operator: NOT_MATCHES_REGEX
          item_prototypes:
            - uuid: 7d316cbec2ce4718ac133d90b7a89585
              name: 'RPCs completed with code {#GRPC.CODE}'
              type: DEPENDENT
              key: 'etcd.grpc.handled.rate[{#GRPC.CODE}]'
              value_type: FLOAT
              units: rps
              description: 'The number of RPCs completed on the server with grpc_code {#GRPC.CODE}.'
              preprocessing:
                - type: PROMETHEUS_TO_JSON
                  parameters:
                    - 'grpc_server_handled_total{grpc_method="{#GRPC.CODE}"}'
                - type: JAVASCRIPT
                  parameters:
                    - |
                      var valueArr = JSON.parse(value);
                      return valueArr.reduce(function (acc, obj) {
                      	return acc + parseFloat(obj['value']);
                      }, 0);
                - type: CHANGE_PER_SECOND
              master_item:
                key: etcd.get_metrics
              tags:
                - tag: component
                  value: grpc
                - tag: grpc-code
                  value: '{#GRPC.CODE}'
              trigger_prototypes:
                - uuid: 459b6ee5735047d597a6a4ab41b76e21
                  expression: 'min(/Etcd by HTTP/etcd.grpc.handled.rate[{#GRPC.CODE}],5m)>{$ETCD.GRPC.ERRORS.MAX.WARN}'
                  name: 'Too many failed gRPC requests with code: {#GRPC.CODE}'
                  event_name: 'Too many failed gRPC requests with code: {#GRPC.CODE} (over {$ETCD.GRPC.ERRORS.MAX.WARN} in 5m)'
                  priority: WARNING
                  tags:
                    - tag: scope
                      value: availability
          master_item:
            key: etcd.get_metrics
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - grpc_server_handled_total
            - type: JAVASCRIPT
              parameters:
                - |
                  var data = JSON.parse(value),
                  	lookup = {},
                  	result = [];
                  for (var item, i = 0; item = data[i++];) {
                  	var code = item.labels.grpc_code;
                  	if (!(code in lookup)) {
                  		lookup[code] = 1;
                  		result.push({ "{#GRPC.CODE}": code });
                  	}
                  }
                  return JSON.stringify(result);
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1h
          overrides:
            - name: trigger
              step: '1'
              filter:
                conditions:
                  - macro: '{#GRPC.CODE}'
                    value: '{$ETCD.GRPC_CODE.TRIGGER.MATCHES}'
              operations:
                - operationobject: TRIGGER_PROTOTYPE
                  operator: LIKE
                  value: 'Too many failed gRPC requests'
                  discover: DISCOVER
                  status: ENABLED
        - uuid: b7b527ee30b84a569afcd1f85b705810
          name: 'Peers discovery'
          type: DEPENDENT
          key: etcd.peer.discovery
          item_prototypes:
            - uuid: 4129aa7b8acf4ca3b5476461fe5275c9
              name: 'Etcd peer {#ETCD.PEER}: Bytes received'
              type: DEPENDENT
              key: 'etcd.bytes.received.rate[{#ETCD.PEER}]'
              value_type: FLOAT
              units: Bps
              description: 'The number of bytes received from a peer with the ID `{#ETCD.PEER}`.'
              preprocessing:
                - type: PROMETHEUS_PATTERN
                  parameters:
                    - 'etcd_network_peer_received_bytes_total{From="{#ETCD.PEER}"}'
                    - value
                    - ''
                  error_handler: CUSTOM_VALUE
                  error_handler_params: '0'
                - type: CHANGE_PER_SECOND
              master_item:
                key: etcd.get_metrics
              tags:
                - tag: component
                  value: network
                - tag: component
                  value: peers
                - tag: peer
                  value: '{#ETCD.PEER}'
            - uuid: 8f5fecbabe474baaab40df46879401af
              name: 'Etcd peer {#ETCD.PEER}: Bytes sent'
              type: DEPENDENT
              key: 'etcd.bytes.sent.rate[{#ETCD.PEER}]'
              value_type: FLOAT
              units: Bps
              description: 'The number of bytes sent to a peer with the ID `{#ETCD.PEER}`.'
              preprocessing:
                - type: PROMETHEUS_PATTERN
                  parameters:
                    - 'etcd_network_peer_sent_bytes_total{To="{#ETCD.PEER}"}'
                    - value
                    - ''
                  error_handler: CUSTOM_VALUE
                  error_handler_params: '0'
                - type: CHANGE_PER_SECOND
              master_item:
                key: etcd.get_metrics
              tags:
                - tag: component
                  value: network
                - tag: component
                  value: peers
                - tag: peer
                  value: '{#ETCD.PEER}'
            - uuid: 2521ccfc16fc43069001883b85aa0243
              name: 'Etcd peer {#ETCD.PEER}: Receive failures'
              type: DEPENDENT
              key: 'etcd.received.fail.rate[{#ETCD.PEER}]'
              value_type: FLOAT
              units: rps
              description: 'The number of received failures from a peer with the ID `{#ETCD.PEER}`.'
              preprocessing:
                - type: PROMETHEUS_PATTERN
                  parameters:
                    - 'etcd_network_peer_received_failures_total{To="{#ETCD.PEER}"}'
                    - value
                    - ''
                  error_handler: CUSTOM_VALUE
                  error_handler_params: '0'
                - type: CHANGE_PER_SECOND
              master_item:
                key: etcd.get_metrics
              tags:
                - tag: component
                  value: network
                - tag: component
                  value: peers
                - tag: peer
                  value: '{#ETCD.PEER}'
            - uuid: 5756f1a16e5c42b79f6d6225c5382599
              name: 'Etcd peer {#ETCD.PEER}: Send failures'
              type: DEPENDENT
              key: 'etcd.sent.fail.rate[{#ETCD.PEER}]'
              value_type: FLOAT
              units: rps
              description: 'The number of sent failures from a peer with the ID `{#ETCD.PEER}`.'
              preprocessing:
                - type: PROMETHEUS_PATTERN
                  parameters:
                    - 'etcd_network_peer_sent_failures_total{To="{#ETCD.PEER}"}'
                    - value
                    - ''
                  error_handler: CUSTOM_VALUE
                  error_handler_params: '0'
                - type: CHANGE_PER_SECOND
              master_item:
                key: etcd.get_metrics
              tags:
                - tag: component
                  value: network
                - tag: component
                  value: peers
                - tag: peer
                  value: '{#ETCD.PEER}'
          master_item:
            key: etcd.get_metrics
          lld_macro_paths:
            - lld_macro: '{#ETCD.PEER}'
              path: $.labels.To
          preprocessing:
            - type: PROMETHEUS_TO_JSON
              parameters:
                - etcd_network_peer_sent_bytes_total
      tags:
        - tag: class
          value: application
        - tag: target
          value: etcd
      macros:
        - macro: '{$ETCD.GRPC.ERRORS.MAX.WARN}'
          value: '1'
          description: 'The maximum number of gRPC request failures.'
        - macro: '{$ETCD.GRPC_CODE.MATCHES}'
          value: '.*'
          description: 'The filter of discoverable gRPC codes. See more details on https://github.com/grpc/grpc/blob/master/doc/statuscodes.md.'
        - macro: '{$ETCD.GRPC_CODE.NOT_MATCHES}'
          value: CHANGE_IF_NEEDED
          description: 'The filter to exclude discovered gRPC codes. See more details on https://github.com/grpc/grpc/blob/master/doc/statuscodes.md.'
        - macro: '{$ETCD.GRPC_CODE.TRIGGER.MATCHES}'
          value: Aborted|Unavailable
          description: 'The filter of discoverable gRPC codes, which will create triggers.'
        - macro: '{$ETCD.HOST}'
          value: '<SET ETCD HOST>'
          description: 'The hostname or IP address of the `etcd` API endpoint.'
        - macro: '{$ETCD.HTTP.FAIL.MAX.WARN}'
          value: '2'
          description: 'The maximum number of HTTP request failures.'
        - macro: '{$ETCD.LEADER.CHANGES.MAX.WARN}'
          value: '5'
          description: 'The maximum number of leader changes.'
        - macro: '{$ETCD.OPEN.FDS.MAX.WARN}'
          value: '90'
          description: 'The maximum percentage of used file descriptors.'
        - macro: '{$ETCD.PASSWORD}'
        - macro: '{$ETCD.PORT}'
          value: '2379'
          description: 'The port of the `etcd` API endpoint.'
        - macro: '{$ETCD.PROPOSAL.FAIL.MAX.WARN}'
          value: '2'
          description: 'The maximum number of proposal failures.'
        - macro: '{$ETCD.PROPOSAL.PENDING.MAX.WARN}'
          value: '5'
          description: 'The maximum number of proposals in queue.'
        - macro: '{$ETCD.SCHEME}'
          value: http
          description: 'The request scheme which may be `http` or `https`.'
        - macro: '{$ETCD.USER}'
      dashboards:
        - uuid: 5b0ffbb731cd4415a2edbc74978e0276
          name: 'Etcd Overview'
          pages:
            - widgets:
                - type: graph
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: Proposals rate'
                    - type: STRING
                      name: reference
                      value: AAAAA
                - type: graph
                  'y': '5'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: gRPC client traffic'
                    - type: STRING
                      name: reference
                      value: AAAAC
                - type: graph
                  'y': '10'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: gRPC requests rate'
                    - type: STRING
                      name: reference
                      value: AAAAE
                - type: graph
                  'y': '15'
                  width: '72'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: File descriptors'
                    - type: STRING
                      name: reference
                      value: AAAAG
                - type: graph
                  x: '36'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: Read/Write rate'
                    - type: STRING
                      name: reference
                      value: AAAAB
                - type: graph
                  x: '36'
                  'y': '5'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: HTTP requests rate'
                    - type: STRING
                      name: reference
                      value: AAAAD
                - type: graph
                  x: '36'
                  'y': '10'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'Etcd by HTTP'
                        name: 'Etcd: Memory usage'
                    - type: STRING
                      name: reference
                      value: AAAAF
      valuemaps:
        - uuid: f25e21a70baa4e009bdbcb44acb1a22e
          name: 'Etcd healthcheck'
          mappings:
            - value: '0'
              newvalue: Failed
            - value: '1'
              newvalue: Ok
        - uuid: 7bcaf8a520e24613a96d49e63a91a55b
          name: 'Etcd leader'
          mappings:
            - value: '0'
              newvalue: 'No'
            - value: '1'
              newvalue: 'Yes'
        - uuid: 1735a8d251b24c3fbab32e766064536b
          name: 'Service state'
          mappings:
            - value: '0'
              newvalue: Down
            - value: '1'
              newvalue: Up
  triggers:
    - uuid: c45583928d204c04ad8884115e1e35c5
      expression: 'min(/Etcd by HTTP/etcd.open.fds,5m)/last(/Etcd by HTTP/etcd.max.fds)*100>{$ETCD.OPEN.FDS.MAX.WARN}'
      name: 'Current number of open files is too high'
      event_name: 'Current number of open files is too high (over {$ETCD.OPEN.FDS.MAX.WARN}% for 5m)'
      priority: WARNING
      description: |
        Heavy usage of a file descriptor (i.e., near the limit of the process's file descriptor) indicates a potential file descriptor exhaustion issue.
        If the file descriptors are exhausted, `etcd` may panic because it cannot create new WAL files.
      tags:
        - tag: scope
          value: capacity
  graphs:
    - uuid: 18baccd03c0f4814a42d32b51334787d
      name: 'Etcd: File descriptors'
      graph_items:
        - drawtype: GRADIENT_LINE
          color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.open.fds
        - sortorder: '1'
          drawtype: BOLD_LINE
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.max.fds
    - uuid: eefd07cf30d84cc4b84f802468363200
      name: 'Etcd: gRPC client traffic'
      graph_items:
        - color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.network.grpc.received.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.network.grpc.sent.rate
    - uuid: c53ee0dba42d4a1f8afedbe0f6e42785
      name: 'Etcd: gRPC requests rate'
      graph_items:
        - color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.grpc.received.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.grpc.sent.rate
        - sortorder: '2'
          color: 00611C
          item:
            host: 'Etcd by HTTP'
            key: etcd.grpc.started.rate
    - uuid: 520ff92815d84e0f84e9296d249c04ae
      name: 'Etcd: HTTP requests rate'
      graph_items:
        - color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.http.requests.4xx.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.http.requests.5xx.rate
        - sortorder: '2'
          color: 00611C
          item:
            host: 'Etcd by HTTP'
            key: etcd.http.requests.rate
    - uuid: 90af5b2f75b7402693bad7a8f371ab8e
      name: 'Etcd: Memory usage'
      graph_items:
        - drawtype: GRADIENT_LINE
          color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.res.bytes
        - sortorder: '1'
          drawtype: GRADIENT_LINE
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.virtual.bytes
    - uuid: 59cd15292ad04ebd902a7d3080b53838
      name: 'Etcd: Proposals rate'
      graph_items:
        - color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.proposals.failed.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.proposals.committed.rate
        - sortorder: '2'
          color: 00611C
          item:
            host: 'Etcd by HTTP'
            key: etcd.proposals.applied.rate
        - sortorder: '3'
          drawtype: BOLD_LINE
          color: F7941D
          yaxisside: RIGHT
          item:
            host: 'Etcd by HTTP'
            key: etcd.proposals.pending
    - uuid: b374fab55bcc452e9279214ddb2c8024
      name: 'Etcd: Read/Write rate'
      graph_items:
        - color: 199C0D
          item:
            host: 'Etcd by HTTP'
            key: etcd.reads.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'Etcd by HTTP'
            key: etcd.writes.rate
