Buy Percona ServicesBuy Now!

some errors are ouccured in prometheus log

Collapse
X
  • Filter
  • Time
  • Show
Clear All
new posts

  • some errors are ouccured in prometheus log

    hi, all

    I am using prometheus, but there are some error ouccured in prometheus log.


    -- error log

    time="2017-12-12T08:19:16Z" level=warning msg="Scrape health sample discarded" error="sample timestamp out of order" sample=up{instance="malldb25", job="mysql"} => 1 @[1234] source="scrape.go:587"
    time="2017-12-12T08:19:16Z" level=warning msg="Scrape duration sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25" , job="mysql"} => 4.728769441 @[1234] source="scrape.go:590"
    time="2017-12-12T08:19:16Z" level=warning msg="Scrape sample count sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25" , job="mysql"} => 4.728769441 @[1234] source="scrape.go:593"
    time="2017-12-12T08:19:16Z" level=warning msg="Scrape sample count post-relabeling sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25" , job="mysql"} => 4.728769441 @[1234] source="scrape.go:596"


    too many logs are storing now.

    please let me know why these errors are ouccured.




    -- docker run command

    docker run -d \
    -p 80:80 \
    -e METRICS_MEMORY=48318382080 -e METRICS_RETENTION=2160h -e QUERIES_RETENTION=2160h -e METRICS_RESOLUTION=5s \
    --volumes-from pmm-data \
    --name pmm-server \
    --restart always \
    percona/pmm-server:latest



    -- prometheus.yml

    # Managed by pmm-managed. DO NOT EDIT.
    ---
    global:
    scrape_interval: 5s
    scrape_timeout: 5s
    evaluation_interval: 5s
    scrape_configs:
    - job_name: prometheus
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /prometheus/metrics
    scheme: http
    static_configs:
    - targets:
    - localhost:9090
    labels:
    instance: prometheus
    - job_name: linux
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /metrics
    scheme: http
    static_configs:
    - targets:
    - localhost:9100
    labels:
    instance: pmm-server
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - linux:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - job_name: proxysql
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /metrics
    scheme: http
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - proxysql:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - job_name: mongodb
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /metrics
    scheme: http
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - mongodb:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,cluster_([-\w:\.]+),.*
    target_label: cluster
    replacement: $1
    action: replace
    - job_name: mysql-hr
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /metrics-hr
    scheme: http
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - mysql:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - separator: ;
    regex: (.*)
    target_label: job
    replacement: mysql
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - job_name: mysql-mr
    scrape_interval: 5s
    scrape_timeout: 5s
    metrics_path: /metrics-mr
    scheme: http
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - mysql:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - separator: ;
    regex: (.*)
    target_label: job
    replacement: mysql
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - job_name: mysql-lr
    scrape_interval: 1m
    scrape_timeout: 5s
    metrics_path: /metrics-lr
    scheme: http
    consul_sd_configs:
    - server: localhost:8500
    datacenter: dc1
    tag_separator: ','
    scheme: http
    services:
    - mysql:metrics
    basic_auth:
    username: pmm
    password: pmm
    tls_config:
    insecure_skip_verify: true
    relabel_configs:
    - separator: ;
    regex: (.*)
    target_label: job
    replacement: mysql
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,alias_([-\w:\.]+),.*
    target_label: instance
    replacement: $1
    action: replace
    - source_labels: [__meta_consul_tags]
    separator: ;
    regex: .*,scheme_https,.*
    target_label: __scheme__
    replacement: https
    action: replace
    - job_name: rds-mysql-hr
    scrape_interval: 1s
    scrape_timeout: 1s
    metrics_path: /metrics-hr
    relabel_configs:
    - target_label: job
    replacement: mysql
    - job_name: rds-mysql-mr
    scrape_interval: 5s
    scrape_timeout: 1s
    metrics_path: /metrics-mr
    relabel_configs:
    - target_label: job
    replacement: mysql
    - job_name: rds-mysql-lr
    scrape_interval: 1m
    scrape_timeout: 5s
    metrics_path: /metrics-lr
    relabel_configs:
    - target_label: job
    replacement: mysql




    -- top result

    top - 18:05:17 up 22 days, 1:04, 1 user, load average: 0.68, 1.16, 1.25
    Tasks: 444 total, 1 running, 443 sleeping, 0 stopped, 0 zombie
    %Cpu(s): 6.9 us, 0.2 sy, 0.0 ni, 92.9 id, 0.1 wa, 0.0 hi, 0.0 si, 0.0 st
    KiB Mem : 65692272 total, 33868000 free, 2573836 used, 29250436 buff/cache
    KiB Swap: 8191996 total, 8164396 free, 27600 used. 62573256 avail Mem

    PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
    29289 1000 20 0 5564340 1.605g 12892 S 220.2 2.6 94:18.48 prometheus
    29520 1000 20 0 4151424 43404 6504 S 2.3 0.1 0:30.39 percona-qan-api
    29299 root 20 0 2427432 30316 5048 S 1.3 0.0 0:20.82 node_exporter
    5813 root 20 0 148464 2452 1464 R 0.7 0.0 0:00.03 top
    29283 polkitd 20 0 10.344g 115144 6864 S 0.7 0.2 0:17.54 mysqld



    thanks

  • #2
    Unless excessive you should not be concerned. "Prometheus" Dashboard has a graph to show you how many samples are being discarded every second

    Comment

    Working...
    X