Announcement

Announcement Module
Collapse
No announcement yet.

All Node is crash!!!!

Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • All Node is crash!!!!

    -I use:
    +OS: Centos 5.5
    +Node number: 3
    +Package: Percona-XtraDB-Cluster-5.5.34.tar.gz, Percona-XtraDB-Cluster-galera-2-2.8-1.157.rhel5.rpm,percona-xtrabackup-2.1.7-721.rhel5

    File my.cnf:

    Code:
    [client]
    port            = 3306
    socket          = /usr/local/webserver/percona-server2/data/mysql.sock
    [mysqld]
    port            = 3306
    socket          = /usr/local/webserver/percona-server2/data/mysql.sock
    skip-external-locking
    datadir = /usr/local/webserver/percona-server2/data
    log-slow-queries = /data0/percona/3307/slow.log
    long_query_time = 10
    wsrep_provider=/usr/lib64/libgalera_smm.so
    wsrep_cluster_address=gcomm://xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx
    binlog_format=ROW
    default_storage_engine=InnoDB
    innodb_autoinc_lock_mode=2
    wsrep_node_address=xxx.xxx.xxx.xxx
    wsrep_sst_method=xtrabackup
    wsrep_cluster_name=my_centos_cluster
    wsrep_sst_auth="sstuser:s3cret"
    open_files_limit    = 10240
    back_log = 600
    max_connections = 5000
    max_connect_errors = 6000
    table_cache = 614
    external-locking = FALSE
    max_allowed_packet = 32M
    sort_buffer_size = 1M
    join_buffer_size = 1M
    thread_cache_size = 300
    query_cache_size = 0
    query_cache_limit = 0
    query_cache_min_res_unit = 2k
    default-storage-engine = MyISAM
    thread_stack = 192K
    transaction_isolation = READ-COMMITTED
    tmp_table_size = 246M
    max_heap_table_size = 256M
    long_query_time = 3
    log-slave-updates
    binlog_cache_size = 256M
    max_binlog_cache_size = 1G
    max_binlog_size = 1G
    interactive_timeout = 288000
    wait_timeout = 288000
    key_buffer_size = 4M
    table_open_cache = 64
    sort_buffer_size = 128K
    net_buffer_length = 8K
    read_buffer_size = 256K
    read_rnd_buffer_size = 512K
    myisam_sort_buffer_size = 8M
    loose_handlersocket_port = 9998
    loose_handlersocket_port_wr = 9999
    loose_handlersocket_threads = 16
    loose_handlersocket_threads_wr = 1
    open_files_limit = 65535
    log-bin=mysql-bin
    server-id       = 1
    innodb_data_home_dir = /usr/local/webserver/percona-server2/data
    innodb_data_file_path = ibdata1:10M:autoextend
    innodb_log_group_home_dir = /usr/local/webserver/percona-server2/data
    innodb_buffer_pool_size = 32M
    innodb_additional_mem_pool_size = 2M
    innodb_log_file_size = 5M
    innodb_log_buffer_size = 8M
    innodb_flush_log_at_trx_commit = 1
    innodb_lock_wait_timeout = 50
    [mysqldump]
    quick
    max_allowed_packet = 16M
    [mysql]
    no-auto-rehash
    [myisamchk]
    key_buffer_size = 20M
    sort_buffer_size = 20M
    read_buffer = 2M
    write_buffer = 2M
    [mysqlhotcopy]
    interactive-timeout

    File error Log:

    Code:
    140223 10:45:38 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:45:38 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: tcp://210.211.105.169:4567
    140223 10:45:39 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') reconnecting to 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e (tcp://210.211.105.169:4567), attempt 0
    140223 10:45:40 [Note] WSREP: evs::proto(d283e7e2-9c12-11e3-a79d-da979c1a3636, OPERATIONAL, view_id(REG,5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e,11)) suspecting node: 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e
    140223 10:45:41 [Note] WSREP: evs::proto(d283e7e2-9c12-11e3-a79d-da979c1a3636, GATHER, view_id(REG,5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e,11)) suspecting node: 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e
    140223 10:45:42 [Note] WSREP: declaring a36fdba8-9c0f-11e3-a3c9-d64488c384ab stable
    140223 10:45:42 [Note] WSREP: Node a36fdba8-9c0f-11e3-a3c9-d64488c384ab state prim
    140223 10:45:42 [Note] WSREP: view(view_id(PRIM,a36fdba8-9c0f-11e3-a3c9-d64488c384ab,12) memb {
            a36fdba8-9c0f-11e3-a3c9-d64488c384ab,
            d283e7e2-9c12-11e3-a79d-da979c1a3636,
    } joined {
    } left {
    } partitioned {
            5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e,
    })
    140223 10:45:42 [Note] WSREP: forgetting 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e (tcp://210.211.105.169:4567)
    140223 10:45:42 [Note] WSREP: deleting entry tcp://210.211.105.169:4567
    140223 10:45:42 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 1, memb_num = 2
    140223 10:45:42 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:45:42 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
    140223 10:45:42 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') turning message relay requesting off
    140223 10:45:42 [Note] WSREP: STATE EXCHANGE: sent state msg: 087d4e2c-9c3d-11e3-ad3e-d602c44f07e2
    140223 10:45:42 [Note] WSREP: STATE EXCHANGE: got state msg: 087d4e2c-9c3d-11e3-ad3e-d602c44f07e2 from 0 (SRV07)
    140223 10:45:42 [Note] WSREP: STATE EXCHANGE: got state msg: 087d4e2c-9c3d-11e3-ad3e-d602c44f07e2 from 1 (SRV02)
    140223 10:45:42 [Note] WSREP: Quorum results:
            version    = 2,
            component  = PRIMARY,
            conf_id    = 11,
            members    = 2/2 (joined/total),
            act_id     = 113055,
            last_appl. = 112893,
            protocols  = 0/4/2 (gcs/repl/appl),
            group UUID = ea27b198-9c0e-11e3-ac62-6ea9475c693f
    140223 10:45:42 [Note] WSREP: Flow-control interval: [23, 23]
    140223 10:45:42 [Note] WSREP: New cluster view: global state: ea27b198-9c0e-11e3-ac62-6ea9475c693f:113055, view# 12: Primary, number of nodes: 2, my index: 1, protocol version 2
    140223 10:45:42 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    140223 10:45:42 [Note] WSREP: Assign initial position for certification: 113055, protocol version: 2
    140223 10:45:42 [ERROR] WSREP: Certification failed for TO isolated action: source: a36fdba8-9c0f-11e3-a3c9-d64488c384ab version: 2 local: 0 state: CERTIFYING flags: 65 conn_id: 194 trx_id: -1 seqnos (l: 113238, g: 113056, s: 113053, d: -1, ts: 1393127170063119000)
    140223 10:45:47 [Note] WSREP:  cleaning up 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e (tcp://210.211.105.169:4567)
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:34 [Note] WSREP: (d283e7e2-9c12-11e3-a79d-da979c1a3636, 'tcp://0.0.0.0:4567') address 'tcp://210.211.105.224:4567' pointing to uuid d283e7e2-9c12-11e3-a79d-da979c1a3636 is blacklisted, skipping
    140223 10:47:35 [Note] WSREP: declaring 5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e stable
    140223 10:47:35 [Note] WSREP: declaring a36fdba8-9c0f-11e3-a3c9-d64488c384ab stable
    140223 10:47:35 [Note] WSREP: Node a36fdba8-9c0f-11e3-a3c9-d64488c384ab state prim
    140223 10:47:35 [Note] WSREP: view(view_id(PRIM,5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e,13) memb {
            5a9dcbfc-9c56-11e3-9c5a-8354a546ed5e,
            a36fdba8-9c0f-11e3-a3c9-d64488c384ab,
            d283e7e2-9c12-11e3-a79d-da979c1a3636,
    } joined {
    } left {
    } partitioned {
    })
    140223 10:47:35 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 2, memb_num = 3
    140223 10:47:35 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
    140223 10:47:35 [Note] WSREP: STATE EXCHANGE: sent state msg: 7ee39754-9c80-11e3-9ffe-42e9fbeb5d74
    140223 10:47:35 [Note] WSREP: STATE EXCHANGE: got state msg: 7ee39754-9c80-11e3-9ffe-42e9fbeb5d74 from 0 (SRV71)
    140223 10:47:35 [Note] WSREP: STATE EXCHANGE: got state msg: 7ee39754-9c80-11e3-9ffe-42e9fbeb5d74 from 1 (SRV07)
    140223 10:47:35 [Note] WSREP: STATE EXCHANGE: got state msg: 7ee39754-9c80-11e3-9ffe-42e9fbeb5d74 from 2 (SRV02)
    140223 10:47:35 [Note] WSREP: Quorum results:
            version    = 2,
            component  = PRIMARY,
            conf_id    = 12,
            members    = 2/3 (joined/total),
            act_id     = 114999,
            last_appl. = 114052,
            protocols  = 0/4/2 (gcs/repl/appl),
            group UUID = ea27b198-9c0e-11e3-ac62-6ea9475c693f
    140223 10:47:35 [Note] WSREP: Flow-control interval: [28, 28]
    140223 10:47:35 [Note] WSREP: New cluster view: global state: ea27b198-9c0e-11e3-ac62-6ea9475c693f:114999, view# 13: Primary, number of nodes: 3, my index: 2, protocol version 2
    140223 10:47:35 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    140223 10:47:35 [Note] WSREP: Assign initial position for certification: 114999, protocol version: 2
    140223 10:47:36 [ERROR] WSREP: Certification failed for TO isolated action: source: a36fdba8-9c0f-11e3-a3c9-d64488c384ab version: 2 local: 0 state: CERTIFYING flags: 65 conn_id: 193 trx_id: -1 seqnos (l: 115186, g: 115000, s: 114998, d: -1, ts: 1393127284523619000)
    140223 10:47:37 [Note] WSREP: Node 0 (SRV71) requested state transfer from '*any*'. Selected 1 (SRV07)(SYNCED) as donor.
Working...
X