After a couple days of running one node in a 2 node cluster (with arbitrator) will error out saying "Could not read field" Error 1610 and then "Could not execute Update_rows event" Error 1030.
The other node continued. The field exists in the table. Nodes were initialized using xtrabackup method.
130417 13:54:15 [ERROR] Slave SQL: Could not read field 'UPDT_DB_DTTM' of table 'GLOBAL.CLIENT_LAST_OPEN_PLATFORM', Error_code: 1610
|
130417 13:54:15 [ERROR] Slave SQL: Could not execute Update_rows event on table GLOBAL.CLIENT_LAST_OPEN_PLATFORM; Got error 1610 from storage engine, Error_code: 1030; handler error No Error!; the event's master log FIRST,
|
end_log_pos 272, Error_code: 1030
|
130417 13:54:15 [Warning] WSREP: RBR event 2 Update_rows apply warning: 1610, 222480722
|
130417 13:54:15 [ERROR] WSREP: Failed to apply trx: source: 6b76b9ab-a623-11e2-0800-0bcb2fe17662 version: 2 local: 0 state: APPLYING flags: 1 conn_id: 7818 trx_id: 8694488863 seqnos (l: 60013290, g: 222480722, s: 222480721
|
, d: 222480682, ts: 1366221255138895484)
|
130417 13:54:15 [ERROR] WSREP: Failed to apply app buffer: seqno: 222480722, status: WSREP_FATAL
|
at galera/src/replicator_smm.cpp:apply_wscoll():53
|
at galera/src/replicator_smm.cpp:apply_trx_ws():120
|
130417 13:54:15 [ERROR] WSREP: Node consistency compromized, aborting...
|
130417 13:54:15 [Note] WSREP: Closing send monitor...
|
130417 13:54:15 [Note] WSREP: Closed send monitor.
|
130417 13:54:15 [Note] WSREP: gcomm: terminating thread
|
130417 13:54:15 [Note] WSREP: gcomm: joining thread
|
130417 13:54:15 [Note] WSREP: gcomm: closing backend
|
130417 13:54:15 [Note] WSREP: view(view_id(NON_PRIM,382a82e8-a3d4-11e2-0800-2753044f506b,25) memb {
|
382a82e8-a3d4-11e2-0800-2753044f506b,
|
} joined {
|
} left {
|
} partitioned {
|
6b76b9ab-a623-11e2-0800-0bcb2fe17662,
|
ebe9df82-9e2c-11e2-0800-65ae0b1b9cdc,
|
})
|
130417 13:54:15 [Note] WSREP: view((empty))
|
130417 13:54:15 [Note] WSREP: New COMPONENT: primary = no, bootstrap = no, my_idx = 0, memb_num = 1
|
130417 13:54:15 [Note] WSREP: gcomm: closed
|
130417 13:54:15 [Note] WSREP: Flow-control interval: [16, 16]
|
130417 13:54:15 [Note] WSREP: Received NON-PRIMARY.
|
130417 13:54:15 [Note] WSREP: Shifting SYNCED -> OPEN (TO: 222480778)
|
130417 13:54:15 [Note] WSREP: Received self-leave message.
|
130417 13:54:15 [Note] WSREP: Flow-control interval: [0, 0]
|
130417 13:54:15 [Note] WSREP: Received SELF-LEAVE. Closing connection.
|
130417 13:54:15 [Note] WSREP: Shifting OPEN -> CLOSED (TO: 222480778)
|
130417 13:54:15 [Note] WSREP: RECV thread exiting 0: Success
|
130417 13:54:15 [Note] WSREP: recv_thread() joined.
|
130417 13:54:15 [Note] WSREP: Closing slave action queue.
|
130417 13:54:15 [Note] WSREP: /usr/sbin/mysqld: Terminated.
|
130417 13:54:16 mysqld_safe Number of processes running now: 0
|
130417 13:54:16 mysqld_safe WSREP: not restarting wsrep node automatically
|
130417 13:54:16 mysqld_safe mysqld from pid file /database/data/4c-maria-02.pid ended
|
From our settings:
# logs and replication
|
log-bin=mysql-bin
|
binlog-format=ROW
|
max_binlog_cache_size=1024G
|
|
# Galera Settings
|
wsrep_provider=/usr/lib64/galera/libgalera_smm.so
|
wsrep_cluster_address=gcomm://10.200.0.10
|
wsrep_cluster_name='client01_cluster'
|
wsrep_node_name='4c-maria-02'
|
wsrep_slave_threads=24
|
wsrep_retry_autocommit=10
|
wsrep_sst_method=xtrabackup
|
wsrep_sst_auth=galera:AhwFVAahpZfh8BVG
|
|
|
# innodb (xtradb) settings
|
default_storage_engine=InnoDB
|
innodb_file_per_table
|
innodb_file_format=barracuda
|
innodb_log_file_size=2000M
|
innodb_log_files_in_group=2
|
innodb_flush_log_at_trx_commit=2
|
innodb_autoinc_lock_mode=2
|
innodb_locks_unsafe_for_binlog=1
|
MariaDB-galera was installed from repos.
Transition |
Time In Source Status |
Execution Times |
Open |
|
In Progress |
|
124d 12h 1m
|
1
|
In Progress |
|
Closed |
|
454d 15h 9m
|
1
|
{"report":{"fcp":843.5999999046326,"ttfb":229.40000009536743,"pageVisibility":"visible","entityId":23710,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"72ed1203-9bb7-404b-813b-1b839bc8009b","navigationType":0,"readyForUser":939.9000000953674,"redirectCount":0,"resourceLoadedEnd":586.0999999046326,"resourceLoadedStart":234.80000019073486,"resourceTiming":[{"duration":28.699999809265137,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bsh/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":234.80000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":234.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":263.5,"responseStart":0,"secureConnectionStart":0},{"duration":29.09999990463257,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bsh/820016/12ta74/eb142f92e4bd16bd1ef8b08c1b9d5d56/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":235.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":235.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":264.19999980926514,"responseStart":0,"secureConnectionStart":0},{"duration":204.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/c54b129276d75dc2a3460e1d78f37913-CDN/lu2bsh/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":235.19999980926514,"connectEnd":235.19999980926514,"connectStart":235.19999980926514,"domainLookupEnd":235.19999980926514,"domainLookupStart":235.19999980926514,"fetchStart":235.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":269.59999990463257,"responseEnd":439.59999990463257,"responseStart":284.40000009536743,"secureConnectionStart":235.19999980926514},{"duration":59.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/ffdb17665775c5b4d6f097f3974ee359-CDN/lu2bsh/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":235.40000009536743,"connectEnd":235.40000009536743,"connectStart":235.40000009536743,"domainLookupEnd":235.40000009536743,"domainLookupStart":235.40000009536743,"fetchStart":235.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":270.59999990463257,"responseEnd":294.59999990463257,"responseStart":292,"secureConnectionStart":235.40000009536743},{"duration":350.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/f867843cd2fdb209d4d1d4f760f86346-CDN/lu2bsh/820016/12ta74/eb142f92e4bd16bd1ef8b08c1b9d5d56/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":235.40000009536743,"connectEnd":235.40000009536743,"connectStart":235.40000009536743,"domainLookupEnd":235.40000009536743,"domainLookupStart":235.40000009536743,"fetchStart":235.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":270.40000009536743,"responseEnd":586.0999999046326,"responseStart":290,"secureConnectionStart":235.40000009536743},{"duration":64.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":235.59999990463257,"connectEnd":235.59999990463257,"connectStart":235.59999990463257,"domainLookupEnd":235.59999990463257,"domainLookupStart":235.59999990463257,"fetchStart":235.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":271.19999980926514,"responseEnd":299.90000009536743,"responseStart":295.59999990463257,"secureConnectionStart":235.59999990463257},{"duration":64.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":235.59999990463257,"connectEnd":235.59999990463257,"connectStart":235.59999990463257,"domainLookupEnd":235.59999990463257,"domainLookupStart":235.59999990463257,"fetchStart":235.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":272.30000019073486,"responseEnd":300.09999990463257,"responseStart":296.19999980926514,"secureConnectionStart":235.59999990463257},{"duration":33.09999990463257,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bsh/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":235.80000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":235.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":268.90000009536743,"responseStart":0,"secureConnectionStart":0},{"duration":64.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":235.90000009536743,"connectEnd":235.90000009536743,"connectStart":235.90000009536743,"domainLookupEnd":235.90000009536743,"domainLookupStart":235.90000009536743,"fetchStart":235.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":273.69999980926514,"responseEnd":300.30000019073486,"responseStart":296.90000009536743,"secureConnectionStart":235.90000009536743},{"duration":34.30000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bsh/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":236,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":236,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":270.30000019073486,"responseStart":0,"secureConnectionStart":0},{"duration":67.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/81b5d7c27af3ebc078cc4a36383678ba-CDN/lu2bsh/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":236.09999990463257,"connectEnd":236.09999990463257,"connectStart":236.09999990463257,"domainLookupEnd":236.09999990463257,"domainLookupStart":236.09999990463257,"fetchStart":236.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":274.19999980926514,"responseEnd":304,"responseStart":297.5,"secureConnectionStart":236.09999990463257},{"duration":321.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":241.40000009536743,"connectEnd":241.40000009536743,"connectStart":241.40000009536743,"domainLookupEnd":241.40000009536743,"domainLookupStart":241.40000009536743,"fetchStart":241.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":295.19999980926514,"responseEnd":562.9000000953674,"responseStart":555,"secureConnectionStart":241.40000009536743},{"duration":321.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":241.5,"connectEnd":241.5,"connectStart":241.5,"domainLookupEnd":241.5,"domainLookupStart":241.5,"fetchStart":241.5,"redirectEnd":0,"redirectStart":0,"requestStart":298.80000019073486,"responseEnd":563.3000001907349,"responseStart":556.6999998092651,"secureConnectionStart":241.5},{"duration":205.90000009536743,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":565.9000000953674,"connectEnd":565.9000000953674,"connectStart":565.9000000953674,"domainLookupEnd":565.9000000953674,"domainLookupStart":565.9000000953674,"fetchStart":565.9000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":741.1999998092651,"responseEnd":771.8000001907349,"responseStart":770.6999998092651,"secureConnectionStart":565.9000000953674},{"duration":165,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":799.3000001907349,"connectEnd":799.3000001907349,"connectStart":799.3000001907349,"domainLookupEnd":799.3000001907349,"domainLookupStart":799.3000001907349,"fetchStart":799.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":934.3000001907349,"responseEnd":964.3000001907349,"responseStart":963.3000001907349,"secureConnectionStart":799.3000001907349},{"duration":138.7999997138977,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":836.3000001907349,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":836.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":975.0999999046326,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":60,"responseStart":229,"responseEnd":234,"domLoading":232,"domInteractive":1043,"domContentLoadedEventStart":1043,"domContentLoadedEventEnd":1084,"domComplete":1757,"loadEventStart":1757,"loadEventEnd":1757,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1019.4000000953674},{"name":"bigPipe.sidebar-id.end","time":1020.0999999046326},{"name":"bigPipe.activity-panel-pipe-id.start","time":1020.1999998092651},{"name":"bigPipe.activity-panel-pipe-id.end","time":1021.0999999046326},{"name":"activityTabFullyLoaded","time":1096.0999999046326}],"measures":[],"correlationId":"cba6ee59fca886","effectiveType":"4g","downlink":9.6,"rtt":0,"serverDuration":89,"dbReadsTimeInMs":29,"dbConnsTimeInMs":38,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}