Xtrabackup cannot keep up with binlogs

  • Filter
  • Time
  • Show
Clear All
new posts

  • Xtrabackup cannot keep up with binlogs


    We're having a problem with backups failing on one of our slaves. The backups fail soon after beginning with the following error message about not being able to find a binlog file. Our guess was that we're replicating so much data (over a terabyte, 6000+ qps) that percona can't keep up with it and the binlog file rotates before it can copy it. Our work around has been to stop the slave for about 10 minutes and then restart after the log file shows files are being copied. Does anyone have suggestions about tuning or some feature I'm not implementing?

    xtrabackup_51 version 2.0.6 for MySQL server 5.1.59 unknown-linux-gnu (x86_64) (revision id: undefined)
    xtrabackup: uses posix_fadvise().
    xtrabackup: cd to /var/lib/mysql
    xtrabackup: Target instance is assumed as followings.
    xtrabackup: innodb_data_home_dir = ./
    xtrabackup: innodb_data_file_path = ibdata1:10M:autoextend
    xtrabackup: innodb_log_group_home_dir = ./
    xtrabackup: innodb_log_files_in_group = 2
    xtrabackup: innodb_log_file_size = 268435456
    xtrabackup: using O_DIRECT
    130808 16:09:24 InnoDB: Operating system error number 2 in a file operation.
    InnoDB: The error means the system cannot find the path specified.
    InnoDB: File name ./mysqld-relay-bin.119788
    InnoDB: File operation call: 'stat'.
    InnoDB: Error: os_file_readdir_next_file() returned -1 in
    InnoDB: directory .
    InnoDB: Crash recovery may have failed for some .ibd files!
    130808 16:09:41 InnoDB: Operating system error number 2 in a file operation.
    InnoDB: The error means the system cannot find the path specified.
    InnoDB: File name ./mysqld-relay-bin.119785
    InnoDB: File operation call: 'stat'.
    InnoDB: Error: os_file_readdir_next_file() returned -1 in
    InnoDB: directory .

  • #2
    Hi sysmonkey,

    Could you provide us with the command you are running and the my.cnf active for your environment? It sounds quite weird that xtrabackup is currently failing to stat those files even with that amount of transactions coming through.



    • #3
      Could you place the relay-logs outside the datadirectory as a workaround. It seems xtrabackup exits during the relay-log rotation in the datadirectory. xtrabackup keeps an eye on the files which are in the datadir and it currently doesn't like the relay-logs being removed during a backup.

      Anyway, I have created a bug for you on https://bugs.launchpad.net/percona-x...p/+bug/1211669 , so we can have a look on how to change this type of behaviour.


      • #4

        Thank you for the workaround, it worked.

        Here is the start command

        /usr/bin/ionice -c2 -n4 /usr/bin/innobackupex-1.5.1 --slave-info --password=xxxxx

        Here is our my.cnf

        # The MySQL database server configuration file.
        # You can copy this to one of:
        # - "/etc/mysql/my.cnf" to set global options,
        # - "~/.my.cnf" to set user-specific options.
        # One can use all long options that the program supports.
        # Run program with --help to get a list of available options and with
        # --print-defaults to see which it would actually understand and use.
        # For explanations see
        # http://dev.mysql.com/doc/mysql/en/se...variables.html

        # This will be passed to all mysql clients
        # It has been reported that passwords should be enclosed with ticks/quotes
        # escpecially if they contain "#" chars...
        # Remember to edit /etc/mysql/debian.cnf when changing the socket location.
        port = 3306
        socket = /var/run/mysqld/mysqld.sock

        # Here is entries for some specific programs
        # The following values assume you have at least 32M ram

        # This was formally known as [safe_mysqld]. Both versions are currently parsed.
        socket = /var/run/mysqld/mysqld.sock
        nice = 0

        # * Basic Settings

        user = mysql
        pid-file = /var/run/mysqld/mysqld.pid
        socket = /var/run/mysqld/mysqld.sock
        port = 3306
        basedir = /usr
        datadir = /var/lib/mysql
        tmpdir = /data/tmp
        default-storage-engine = innodb

        # Instead of skip-networking the default is now to listen only on
        # localhost which is more compatible and is not less secure.
        #bind-address =
        bind-address = [xxx.xxx.xxx.xxx]

        # * Fine Tuning
        key_buffer = 16M
        max_allowed_packet = 64M
        thread_stack = 192K
        thread_cache_size = 8
        # This replaces the startup script and checks MyISAM tables if needed
        # the first time they are touched
        myisam-recover = BACKUP
        max_connections = 2048
        table_open_cache = 4000
        table_definition_cache = 4000
        open-files-limit = 12000
        #thread_concurrency = 10
        # * Query Cache Configuration
        query_cache_limit = 1M
        query_cache_size = 16M
        max_heap_table_size = 100M
        tmp_table_size = 100M
        # * Logging and Replication
        # Both location gets rotated by the cronjob.
        # Be aware that this log type is a performance killer.
        # As of 5.1 you can enable the log at runtime!
        #general_log_file = /var/log/mysql/mysql.log
        #general_log = 1
        # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf.
        # Here you can see queries with especially long duration
        slow_query_log = 1
        slow_query_log_file = /var/log/mysql/mysql-slow.log
        long_query_time = 2

        log-error = /var/log/mysql/mysql-error.log
        # The following can be used as easy to replay backup logs or for replication.
        # note: if you are setting up a replication slave, see README.Debian about
        # other settings you may need to change.
        # Replications settings
        server-id = 60000313
        master-host = [xxx.xxx.xxx.xxx]
        master-user = username
        master-password = password
        relay-log = /var/log/mysql/relay/mysqld-relay-bin

        log_bin = /var/log/mysql/mysql-bin.log
        binlog_format = row
        expire_logs_days = 3
        max_binlog_size = 100M
        sync_binlog = 0

        # Logs are getting deleted before backup can get past initial stages.
        # Set value to double the size of max_binlog_size.
        # See: http://dev.mysql.com/doc/refman/5.1/...-relaylog.html

        # * InnoDB
        # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
        # Read the manual for more InnoDB related options. There are many!
        innodb_file_per_table = 1
        innodb_file_format = barracuda


        # * Security Features
        # Read the manual, too, if you want chroot!
        # chroot = /var/lib/mysql/
        # For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
        # ssl-ca=/etc/mysql/cacert.pem
        # ssl-cert=/etc/mysql/server-cert.pem
        # ssl-key=/etc/mysql/server-key.pem

        max_allowed_packet = 64M

        #no-auto-rehash # faster start of mysql but no tab completition

        key_buffer = 16M

        # * IMPORTANT: Additional settings that can override those from this file!
        # The files must end with '.cnf', otherwise they'll be ignored.
        !includedir /etc/mysql/conf.d/