zabbix_export:
  version: '7.0'
  template_groups:
    - uuid: 748ad4d098d447d492bb935c907f652f
      name: Templates/Databases
  templates:
    - uuid: 3a0bbbb2ec0a4c58bba3ba3a3d6ce660
      template: 'TiDB TiKV by HTTP'
      name: 'TiDB TiKV by HTTP'
      description: |
        The template to monitor TiKV server of TiDB cluster by Zabbix that works without any external scripts.
        Most of the metrics are collected in one go, thanks to Zabbix bulk data collection.
        Don't forget to change the macros {$TIKV.URL}, {$TIKV.PORT}.
        
        Template `TiDB TiKV by HTTP` — collects metrics by HTTP agent from TiKV /metrics endpoint.
        
        You can discuss this template or leave feedback on our forum https://www.zabbix.com/forum/zabbix-suggestions-and-feedback
        
        Generated by official Zabbix template tool "Templator"
      vendor:
        name: Zabbix
        version: 7.0-0
      groups:
        - name: Templates/Databases
      items:
        - uuid: 449a7ba908884db3af4ec8ccc0d7ea43
          name: 'Scheduler: High priority commands total, rate'
          type: DEPENDENT
          key: tikv.commands_pri.high.rate
          value_type: FLOAT
          description: 'Total count of high priority commands per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "high")].value.first()'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: scheduler
        - uuid: b5ce79ee79804e76bb7b91e17915159c
          name: 'Scheduler: Low priority commands total, rate'
          type: DEPENDENT
          key: tikv.commands_pri.low.rate
          value_type: FLOAT
          description: 'Total count of low priority commands per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "low")].value.first()'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: scheduler
        - uuid: df3fa02457ac47168fad251727b300ff
          name: 'Scheduler: Normal priority commands total, rate'
          type: DEPENDENT
          key: tikv.commands_pri.normal.rate
          value_type: FLOAT
          description: 'Total count of normal priority commands per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "normal")].value.first()'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: scheduler
        - uuid: dbf8288b5cab4e8a95b5e7c4355676bd
          name: 'Coprocessor: Requests, rate'
          type: DEPENDENT
          key: tikv.coprocessor_request.rate
          value_type: FLOAT
          units: Ops
          description: 'Total number of coprocessor requests per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $..value.sum()
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.coprocessor_requests.metrics
          tags:
            - tag: component
              value: coprocessor
        - uuid: a4166b4e9f454e0388ee813bcf9ed429
          name: 'Get coprocessor requests metrics'
          type: DEPENDENT
          key: tikv.coprocessor_requests.metrics
          history: '0'
          value_type: TEXT
          description: 'Get metrics of coprocessor requests.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_coprocessor_request_duration_seconds_count")]'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: raw
        - uuid: d6185382a1974b9194e68bea4f14eb75
          name: 'Coprocessor: Errors, rate'
          type: DEPENDENT
          key: tikv.coprocessor_request_error.rate
          value_type: FLOAT
          units: Ops
          description: 'Total number of push down request error per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_coprocessor_request_error")].value.sum()'
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: coprocessor
          triggers:
            - uuid: 31eca27ff6ce4ed78ee428ed3b8d8806
              expression: 'min(/TiDB TiKV by HTTP/tikv.coprocessor_request_error.rate,5m)>{$TIKV.COPROCESSOR.ERRORS.MAX.WARN}'
              name: 'Too many coprocessor request error'
              event_name: 'Too many coprocessor request error (over {$TIKV.COPROCESSOR.ERRORS.MAX.WARN} in 5m)'
              priority: WARNING
              tags:
                - tag: scope
                  value: performance
        - uuid: 5567ebf02fb6441dbb6672e1bc77b3ab
          name: 'Coprocessor: Response size, rate'
          type: DEPENDENT
          key: tikv.coprocessor_response_bytes.rate
          value_type: FLOAT
          units: Bps
          description: 'The total size of coprocessor response per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_coprocessor_response_bytes")].value.first()'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: coprocessor
        - uuid: 237f6ba2973e472d8c20de1d61b4695c
          name: 'Coprocessor: RocksDB ops, rate'
          type: DEPENDENT
          key: tikv.coprocessor_rocksdb_perf.rate
          value_type: FLOAT
          units: Ops
          description: 'Total number of RocksDB internal operations from PerfContext per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_coprocessor_rocksdb_perf")].value.sum()'
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: coprocessor
        - uuid: bfcec759c0894cf89c5267fc6b3862d7
          name: 'Coprocessor: Scan keys, rate'
          type: DEPENDENT
          key: tikv.coprocessor_scan_keys_sum.rate
          value_type: FLOAT
          units: Ops
          description: 'Total number of scan keys observed per request per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_coprocessor_scan_keys")].value.sum()'
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: coprocessor
        - uuid: a76665b7a9924f8cbd4a455bc248d790
          name: 'CPU util'
          type: DEPENDENT
          key: tikv.cpu.util
          value_type: FLOAT
          units: '%'
          description: 'The CPU usage ratio on TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_thread_cpu_seconds_total")].value.sum()'
            - type: CHANGE_PER_SECOND
            - type: MULTIPLIER
              parameters:
                - '100'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: cpu
        - uuid: 4558a5a11c304584bf24a7c3d2eace23
          name: 'Bytes read'
          type: DEPENDENT
          key: tikv.engine_flow_bytes.read
          value_type: FLOAT
          units: Bps
          description: 'The total bytes of read in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_engine_flow_bytes" && @.labels.db == "kv" && @.labels.type =~ "bytes_read|iter_bytes_read")].value.sum()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: storage
        - uuid: 0d44c3c9a7f34905b0404ae39a3bc881
          name: 'Bytes write'
          type: DEPENDENT
          key: tikv.engine_flow_bytes.write
          value_type: FLOAT
          units: Bps
          description: 'The total bytes of write in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_engine_flow_bytes" && @.labels.db == "kv" && @.labels.type == "wal_file_bytes")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: storage
        - uuid: f79d5daea61842b18e39c7289657ddf5
          name: 'Store size'
          type: DEPENDENT
          key: tikv.engine_size
          value_type: FLOAT
          units: B
          description: 'The storage size of TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_engine_size_bytes")].value.sum()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: storage
        - uuid: 418bcc8c0bc440468efe833bef02929d
          name: 'Get instance metrics'
          type: HTTP_AGENT
          key: tikv.get_metrics
          history: '0'
          value_type: TEXT
          description: 'Get TiKV instance metrics.'
          preprocessing:
            - type: CHECK_NOT_SUPPORTED
              parameters:
                - '-1'
            - type: PROMETHEUS_TO_JSON
              parameters:
                - ''
          url: '{$TIKV.URL}:{$TIKV.PORT}/metrics'
          tags:
            - tag: component
              value: raw
        - uuid: fed3db7222fa41a8bf51d2ad860a22d4
          name: 'Total query, rate'
          type: DEPENDENT
          key: tikv.grpc_msg.rate
          value_type: FLOAT
          units: Ops
          description: 'The total QPS in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $..value.sum()
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.grpc_msgs.metrics
          tags:
            - tag: component
              value: queries
        - uuid: bb14162b05c84458a66996c08deffb1e
          name: 'Get QPS metrics'
          type: DEPENDENT
          key: tikv.grpc_msgs.metrics
          history: '0'
          value_type: TEXT
          description: 'Get QPS metrics in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_grpc_msg_duration_seconds_count")]'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: raw
        - uuid: c68db41c184d44d98445cc66489ef39c
          name: 'Total query errors, rate'
          type: DEPENDENT
          key: tikv.grpc_msg_fail.rate
          value_type: FLOAT
          units: Ops
          description: 'The total number of gRPC message handling failure per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_grpc_msg_fail_total")].value.sum()'
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: grpc
        - uuid: 3ef3b7bf8c2645e8a953e68fdf246274
          name: 'Get failure msg metrics'
          type: DEPENDENT
          key: tikv.messages.failure.metrics
          history: '0'
          value_type: TEXT
          description: 'Get metrics of reporting failure messages.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_server_report_failure_msg_total")]'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: raw
        - uuid: 7afa5147a1e247a5b8914d8739d31f15
          name: 'Server: failure messages total, rate'
          type: DEPENDENT
          key: tikv.messages.failure.rate
          value_type: FLOAT
          description: 'Total number of reporting failure messages per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $..value.sum()
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.messages.failure.metrics
          tags:
            - tag: component
              value: application
        - uuid: e6137dd87faa496497b23cadaa37dca9
          name: 'Regions, count'
          type: DEPENDENT
          key: tikv.region_count
          description: 'The number of regions collected in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_raftstore_region_count" && @.labels.type == "region" )].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: regions
        - uuid: 2ab7643435794ec6b5a2c3eb1dd7e913
          name: 'Regions, leader'
          type: DEPENDENT
          key: tikv.region_leader
          description: 'The number of leaders in TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_raftstore_region_count" && @.labels.type == "leader" )].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: regions
        - uuid: 1c480fa23eaa42bcb367bb5afdcb65b9
          name: 'RSS memory usage'
          type: DEPENDENT
          key: tikv.rss_bytes
          value_type: FLOAT
          units: B
          description: 'Resident memory size in bytes.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "process_resident_memory_bytes")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: memory
        - uuid: c99a6d9591674723a45de6a0ace68088
          name: 'Get scheduler metrics'
          type: DEPENDENT
          key: tikv.scheduler.metrics
          history: '0'
          value_type: TEXT
          description: 'Get metrics of scheduler commands.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_stage_total")]'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: raw
        - uuid: 62f856cf22804262aef64e9369049332
          name: 'Scheduler: Commands total, rate'
          type: DEPENDENT
          key: tikv.scheduler_commands.rate
          value_type: FLOAT
          description: 'Total number of commands per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - $..value.sum()
              error_handler: CUSTOM_VALUE
              error_handler_params: '0'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.scheduler.metrics
          tags:
            - tag: component
              value: scheduler
        - uuid: 33a1b2f676134cf1b81771fbaf152f59
          name: 'Scheduler: Pending commands'
          type: DEPENDENT
          key: tikv.scheduler_contex
          description: 'The total number of pending commands. The scheduler receives commands from clients, executes them against the MVCC layer storage engine.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_contex_total")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: scheduler
          triggers:
            - uuid: 0f21c02b8e1c45d9bbe0c3313cea1a23
              expression: 'min(/TiDB TiKV by HTTP/tikv.scheduler_contex,5m)>{$TIKV.PENDING_COMMANDS.MAX.WARN}'
              name: 'Too many pending commands'
              event_name: 'Too many pending commands (over {$TIKV.PENDING_COMMANDS.MAX.WARN} for 5m)'
              priority: AVERAGE
              tags:
                - tag: scope
                  value: performance
        - uuid: 8f5c9d2b9eca4a489d764420ee06e4b6
          name: 'Scheduler: Busy, rate'
          type: DEPENDENT
          key: tikv.scheduler_too_busy.rate
          value_type: FLOAT
          description: 'The total count of too busy schedulers per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_scheduler_too_busy_total")].value.sum()'
              error_handler: DISCARD_VALUE
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: scheduler
        - uuid: 0a2d0a119bcd406aadfc5c014186c5d3
          name: 'Snapshot: Applying'
          type: DEPENDENT
          key: tikv.snapshot.applying
          description: 'The total amount of raftstore snapshot traffic.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "applying")].value.first()'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: snapshot
        - uuid: 58559be859d64887a1b919e36aa1e336
          name: 'Snapshot: Receiving'
          type: DEPENDENT
          key: tikv.snapshot.receiving
          description: 'The total amount of raftstore snapshot traffic.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "receiving")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: snapshot
        - uuid: 0ae6a2024ed3488498e22bc542d27619
          name: 'Snapshot: Sending'
          type: DEPENDENT
          key: tikv.snapshot.sending
          description: 'The total amount of raftstore snapshot traffic.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "sending")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: snapshot
        - uuid: ce31318a420d40668e62a47fe19d3d40
          name: 'Storage: commands total, rate'
          type: DEPENDENT
          key: tikv.storage_command.rate
          value_type: FLOAT
          description: 'Total number of commands received per second.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_storage_command_total")].value.sum()'
            - type: CHANGE_PER_SECOND
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: commands
        - uuid: fbfaa0d967c049c0a1bbcec5afea4bf2
          name: 'Available size'
          type: DEPENDENT
          key: tikv.store_size.available
          value_type: FLOAT
          units: B
          description: 'The available capacity of TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.labels.type == "available")].value.first()'
          master_item:
            key: tikv.store_size.metrics
          tags:
            - tag: component
              value: storage
        - uuid: 1e0c1c071b604e77998797c2f9d41dfc
          name: 'Capacity size'
          type: DEPENDENT
          key: tikv.store_size.capacity
          value_type: FLOAT
          units: B
          description: 'The capacity size of TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.labels.type == "capacity")].value.first()'
          master_item:
            key: tikv.store_size.metrics
          tags:
            - tag: component
              value: storage
        - uuid: fd4b786239d9426d9a882af47a5abe01
          name: 'Get store size metrics'
          type: DEPENDENT
          key: tikv.store_size.metrics
          history: '0'
          value_type: TEXT
          description: 'Get capacity metrics of TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_store_size_bytes")]'
              error_handler: DISCARD_VALUE
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: raw
        - uuid: f2632dbd773140398a7d61608d85f392
          name: Uptime
          type: DEPENDENT
          key: tikv.uptime
          value_type: FLOAT
          units: uptime
          description: 'The runtime of each TiKV instance.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name=="process_start_time_seconds")].value.first()'
            - type: JAVASCRIPT
              parameters:
                - 'return (Math.floor(Date.now() / 1000) - Number(value));'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: application
          triggers:
            - uuid: ca06602fa0b64a2ba0c51ed4835c52b2
              expression: 'last(/TiDB TiKV by HTTP/tikv.uptime)<10m'
              name: 'has been restarted'
              event_name: 'has been restarted (uptime < 10m)'
              priority: INFO
              description: 'Uptime is less than 10 minutes.'
              manual_close: 'YES'
              tags:
                - tag: scope
                  value: notice
        - uuid: 8b7c2dc8c60e491db76cafd2d0e1234b
          name: 'Snapshot: Pending tasks'
          type: DEPENDENT
          key: tikv.worker_pending_task
          description: 'The number of tasks currently running by the worker or pending.'
          preprocessing:
            - type: JSONPATH
              parameters:
                - '$[?(@.name == "tikv_worker_pending_task_total")].value.first()'
          master_item:
            key: tikv.get_metrics
          tags:
            - tag: component
              value: snapshot
          triggers:
            - uuid: f26874f910e34933983685ae43a90bee
              expression: 'min(/TiDB TiKV by HTTP/tikv.worker_pending_task,5m)>{$TIKV.PENDING_TASKS.MAX.WARN}'
              name: 'Too many pending tasks'
              event_name: 'Too many pending tasks (over {$TIKV.PENDING_TASKS.MAX.WARN} for 5m)'
              priority: AVERAGE
              tags:
                - tag: scope
                  value: performance
      discovery_rules:
        - uuid: f1c7de94679e40a4ac6f569e05ad61d0
          name: 'Coprocessor metrics discovery'
          type: DEPENDENT
          key: tikv.coprocessor.discovery
          description: 'Discovery coprocessor metrics.'
          item_prototypes:
            - uuid: e5176f11fe694efba01c4a2b14f5929d
              name: 'Coprocessor: {#REQ_TYPE} metrics'
              type: DEPENDENT
              key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]'
              history: '0'
              value_type: TEXT
              description: 'Get metrics of {#REQ_TYPE} requests.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.labels.req == "{#REQ_TYPE}")]'
                  error_handler: DISCARD_VALUE
              master_item:
                key: tikv.get_metrics
              tags:
                - tag: component
                  value: raw
                - tag: request
                  value: '{#REQ_TYPE}'
            - uuid: ecbe08549fef42d5aad0369e861b4c20
              name: 'Coprocessor: {#REQ_TYPE} requests, rate'
              type: DEPENDENT
              key: 'tikv.coprocessor_request.rate[{#REQ_TYPE}]'
              value_type: FLOAT
              units: Ops
              description: 'Total number of coprocessor requests per second.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.name == "tikv_coprocessor_request_duration_seconds_count")].value.first()'
                - type: CHANGE_PER_SECOND
              master_item:
                key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]'
              tags:
                - tag: component
                  value: coprocessor
                - tag: request
                  value: '{#REQ_TYPE}'
            - uuid: 08060401409b4fff97430c595fec7b82
              name: 'Coprocessor: {#REQ_TYPE} errors, rate'
              type: DEPENDENT
              key: 'tikv.coprocessor_request_error.rate[{#REQ_TYPE}]'
              value_type: FLOAT
              units: Ops
              description: 'Total number of push down request error per second.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.name == "tikv_coprocessor_request_error")].value.first()'
                  error_handler: DISCARD_VALUE
                - type: CHANGE_PER_SECOND
              master_item:
                key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]'
              tags:
                - tag: component
                  value: coprocessor
                - tag: request
                  value: '{#REQ_TYPE}'
            - uuid: bff6ee970f7349edba6cac1cd0da70ed
              name: 'Coprocessor: {#REQ_TYPE} RocksDB ops, rate'
              type: DEPENDENT
              key: 'tikv.coprocessor_rocksdb_perf.rate[{#REQ_TYPE}]'
              value_type: FLOAT
              units: Ops
              description: 'Total number of RocksDB internal operations from PerfContext per second.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.name == "tikv_coprocessor_rocksdb_perf")].value.sum()'
                  error_handler: DISCARD_VALUE
                - type: CHANGE_PER_SECOND
              master_item:
                key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]'
              tags:
                - tag: component
                  value: coprocessor
                - tag: request
                  value: '{#REQ_TYPE}'
            - uuid: 8ef93de8a25b4f35b2e54f6ced4a5bd2
              name: 'Coprocessor: {#REQ_TYPE} scan keys, rate'
              type: DEPENDENT
              key: 'tikv.coprocessor_scan_keys.rate[{#REQ_TYPE}]'
              value_type: FLOAT
              units: Ops
              description: 'Total number of scan keys observed per request per second.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.name == "tikv_coprocessor_scan_keys_count")].value.first()'
                - type: CHANGE_PER_SECOND
              master_item:
                key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]'
              tags:
                - tag: component
                  value: coprocessor
                - tag: request
                  value: '{#REQ_TYPE}'
          master_item:
            key: tikv.coprocessor_requests.metrics
          preprocessing:
            - type: JAVASCRIPT
              parameters:
                - |
                  output = JSON.parse(value).map(function (item) {
                  	return {
                  		"{#REQ_TYPE}": item.labels.req,
                  	};
                  });
                  return JSON.stringify({ "data": output });
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1h
        - uuid: 39e4031ee41143ed9d0c84492f817125
          name: 'QPS metrics discovery'
          type: DEPENDENT
          key: tikv.qps.discovery
          description: 'Discovery QPS metrics.'
          item_prototypes:
            - uuid: 3b1d1b10cee6465c822152072189ebbe
              name: 'Query: {#TYPE}, rate'
              type: DEPENDENT
              key: 'tikv.grpc_msg.rate[{#TYPE}]'
              value_type: FLOAT
              units: Ops
              description: 'The QPS per command in TiKV instance.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.labels.type == "{#TYPE}")].value.first()'
                  error_handler: CUSTOM_VALUE
              master_item:
                key: tikv.grpc_msgs.metrics
              tags:
                - tag: component
                  value: grpc
                - tag: type
                  value: '{#TYPE}'
          master_item:
            key: tikv.grpc_msgs.metrics
          preprocessing:
            - type: JAVASCRIPT
              parameters:
                - |
                  output = JSON.parse(value).map(function (item) {
                  	return {
                  		"{#TYPE}": item.labels.type,
                  	};
                  });
                  return JSON.stringify({ "data": output });
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1h
        - uuid: 7168e631146546129284cbf9c52b2025
          name: 'Scheduler metrics discovery'
          type: DEPENDENT
          key: tikv.scheduler.discovery
          description: 'Discovery scheduler metrics.'
          item_prototypes:
            - uuid: c4f902a0be5444b2babb1775b89a6827
              name: 'Scheduler: commands {#STAGE}, rate'
              type: DEPENDENT
              key: 'tikv.scheduler_stage.rate[{#STAGE}]'
              value_type: FLOAT
              description: 'Total number of commands on each stage per second.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.labels.stage == "{#STAGE}")].value.sum()'
                  error_handler: CUSTOM_VALUE
                  error_handler_params: '0'
                - type: CHANGE_PER_SECOND
              master_item:
                key: tikv.scheduler.metrics
              tags:
                - tag: component
                  value: scheduler
                - tag: stage
                  value: '{#STAGE}'
          master_item:
            key: tikv.scheduler.metrics
          preprocessing:
            - type: JAVASCRIPT
              parameters:
                - |
                  var lookup = {},
                  	result = [];
                  
                  JSON.parse(value).forEach(function (item) {
                  	var stage = item.labels.stage;
                  	if (!(lookup[stage])) {
                  		lookup[stage] = 1;
                  		result.push({ "{#STAGE}": stage });
                  	}
                  });
                  
                  return JSON.stringify(result);
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1h
        - uuid: 5ebd2e2131ab4a9f8ca63f0e490b04a3
          name: 'Server errors discovery'
          type: DEPENDENT
          key: tikv.server_report_failure.discovery
          description: 'Discovery server errors metrics.'
          item_prototypes:
            - uuid: 064e74b5484e42e4acf3122c8ead4826
              name: 'Store_id {#STORE_ID}: failure messages "{#TYPE}", rate'
              type: DEPENDENT
              key: 'tikv.messages.failure.rate[{#STORE_ID},{#TYPE}]'
              value_type: FLOAT
              description: 'Total number of reporting failure messages. The metric has two labels: type and store_id. type represents the failure type, and store_id represents the destination peer store id.'
              preprocessing:
                - type: JSONPATH
                  parameters:
                    - '$[?(@.labels.store_id == "{#STORE_ID}"  && @.labels.type == "{#TYPE}")].value.sum()'
                - type: CHANGE_PER_SECOND
              master_item:
                key: tikv.messages.failure.metrics
              tags:
                - tag: component
                  value: stores
                - tag: message-type
                  value: '{#TYPE}'
                - tag: store
                  value: '{#STORE_ID}'
              trigger_prototypes:
                - uuid: 1f65b64824d64852aaa609df5f3e27fd
                  expression: 'min(/TiDB TiKV by HTTP/tikv.messages.failure.rate[{#STORE_ID},{#TYPE}],5m)>{$TIKV.STORE.ERRORS.MAX.WARN}'
                  name: 'Store_id {#STORE_ID}: Too many failure messages "{#TYPE}"'
                  event_name: 'Store_id {#STORE_ID}: Too many failure messages "{#TYPE}" (over {$TIKV.STORE.ERRORS.MAX.WARN} in 5m)'
                  discover: NO_DISCOVER
                  priority: WARNING
                  description: 'Indicates that the remote TiKV cannot be connected.'
                  tags:
                    - tag: scope
                      value: availability
          master_item:
            key: tikv.messages.failure.metrics
          preprocessing:
            - type: JAVASCRIPT
              parameters:
                - |
                  output = JSON.parse(value).map(function (item) {
                  	return {
                  		"{#STORE_ID}": item.labels.store_id,
                  		"{#TYPE}": item.labels.type,
                  
                  	};
                  });
                  return JSON.stringify({ "data": output });
            - type: DISCARD_UNCHANGED_HEARTBEAT
              parameters:
                - 1h
          overrides:
            - name: 'Too many unreachable messages trigger'
              step: '1'
              filter:
                conditions:
                  - macro: '{#TYPE}'
                    value: unreachable
              operations:
                - operationobject: TRIGGER_PROTOTYPE
                  operator: LIKE
                  value: 'Too many failure messages'
                  discover: DISCOVER
                  status: ENABLED
      tags:
        - tag: class
          value: database
        - tag: target
          value: tidb
        - tag: target
          value: tikv
      macros:
        - macro: '{$TIKV.COPROCESSOR.ERRORS.MAX.WARN}'
          value: '1'
          description: 'Maximum number of coprocessor request errors'
        - macro: '{$TIKV.PENDING_COMMANDS.MAX.WARN}'
          value: '1'
          description: 'Maximum number of pending commands'
        - macro: '{$TIKV.PENDING_TASKS.MAX.WARN}'
          value: '1'
          description: 'Maximum number of tasks currently running by the worker or pending'
        - macro: '{$TIKV.PORT}'
          value: '20180'
          description: 'The port of TiKV server metrics web endpoint'
        - macro: '{$TIKV.STORE.ERRORS.MAX.WARN}'
          value: '1'
          description: 'Maximum number of failure messages'
        - macro: '{$TIKV.URL}'
          value: localhost
          description: 'TiKV server URL'
      dashboards:
        - uuid: 5d700978ddb447e59b2536732377d27b
          name: 'TiKV: Overview'
          pages:
            - name: Main
              widgets:
                - type: item
                  name: Uptime
                  width: '12'
                  fields:
                    - type: ITEM
                      name: itemid.0
                      value:
                        host: 'TiDB TiKV by HTTP'
                        key: tikv.uptime
                    - type: INTEGER
                      name: show.0
                      value: '2'
                    - type: INTEGER
                      name: show.1
                      value: '4'
                - type: graph
                  'y': '2'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'TiDB TiKV by HTTP'
                        name: 'TiKV: Snapshot state count'
                    - type: STRING
                      name: reference
                      value: AAAAA
                - type: graph
                  x: '36'
                  'y': '2'
                  width: '36'
                  height: '5'
                  fields:
                    - type: GRAPH
                      name: graphid.0
                      value:
                        host: 'TiDB TiKV by HTTP'
                        name: 'TiKV: Scheduler priority commands rate'
                    - type: STRING
                      name: reference
                      value: AAAAB
  graphs:
    - uuid: cdca8a2bb2bb486da8c57740acc431d6
      name: 'TiKV: Scheduler priority commands rate'
      graph_items:
        - color: 199C0D
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.commands_pri.normal.rate
        - sortorder: '1'
          color: F63100
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.commands_pri.high.rate
        - sortorder: '2'
          color: 00611C
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.commands_pri.low.rate
    - uuid: 93a38153ed8b44a49849cb1f920f50a4
      name: 'TiKV: Snapshot state count'
      graph_items:
        - color: 199C0D
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.snapshot.applying
        - sortorder: '1'
          color: F63100
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.snapshot.receiving
        - sortorder: '2'
          color: 00611C
          item:
            host: 'TiDB TiKV by HTTP'
            key: tikv.snapshot.sending
