[rabbitmq-discuss] Rabbit Cluster Failure
Farshad Meshkinpour
fmeshkinpour at fonality.com
Wed Apr 10 17:51:29 BST 2013
Hello,
We are running Rabbit 2.8.6 on Erlang R14B04. We got this error on one of
3 rabbit servers in our cluster. After that we observed the followings:
1. all 3 rabbit servers went on high cpu usage
2. management interface was not functional
3. workers were not able to consume from queues
Trying to stop the rabbit server failed (/etc/init.d/rabbitmq-server stop)
and end up hard kill (kill -9) rabbit process on all nodes of the cluster
to get the system back. All mirrored queues contents were lost.
Can someone points us to what may have caused this? Any help is greatly
appreciated.
Thanks,
ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.9085.0> terminating
** Last message in was {'$gen_cast',
{delete_and_terminate,
{function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,
rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,
249,92,182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,
223,15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,
#Fun<rabbit_mirror_queue_slave.1.52898136>,0,
0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}}}
** When Server state == {state,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
<0.4404.0>,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86480>]],
[[<0.17781.499>|#Ref<0.0.14563.86174>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86397>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86263>],
[<7526.18741.575>|#Ref<0.0.14563.86686>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|
#Ref<0.0.14563.86622>]]}}},
#Fun<rabbit_mirror_queue_master.1.37767645>,
#Fun<rabbit_mirror_queue_master.2.24464450>}
** Reason for termination ==
** {function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,0,
infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}
=ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.4403.0> terminating
** Last message in was {add_on_right,{74,<0.7181.510>}}
** When Server state == {q,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,
0,0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}
** Reason for termination ==
** {function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,0,
infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}
=ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.7181.510> terminating
** Last message in was {'$gen_cast',join}
** When Server state == {state,
{74,<0.7181.510>},
{{74,<0.7181.510>},undefined},
{{74,<0.7181.510>},undefined},
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.18335.fonsyncd">>},
rabbit_mirror_queue_slave,undefined,-1,undefined,
[<0.7180.510>],
{[],[]},
[],undefined}
** Reason for termination ==
** {{function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,
0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]},
{gen_server2,call,[<0.4403.0>,{add_on_right,{74,<0.7181.510>}},infinity]}}
ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.9085.0> terminating
** Last message in was {'$gen_cast',
{delete_and_terminate,
{function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,
rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,
249,92,182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,
223,15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,
#Fun<rabbit_mirror_queue_slave.1.52898136>,0,
0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}}}
** When Server state == {state,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
<0.4404.0>,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86480>]],
[[<0.17781.499>|#Ref<0.0.14563.86174>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86397>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86263>],
[<7526.18741.575>|#Ref<0.0.14563.86686>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|
#Ref<0.0.14563.86622>]]}}},
#Fun<rabbit_mirror_queue_master.1.37767645>,
#Fun<rabbit_mirror_queue_master.2.24464450>}
** Reason for termination ==
** {function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,0,
infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}
=ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.4403.0> terminating
** Last message in was {add_on_right,{74,<0.7181.510>}}
** When Server state == {q,
{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,
0,0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}
** Reason for termination ==
** {function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,0,
infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]}
=ERROR REPORT==== 9-Apr-2013::10:56:49 ===
** Generic server <0.7181.510> terminating
** Last message in was {'$gen_cast',join}
** When Server state == {state,
{74,<0.7181.510>},
{{74,<0.7181.510>},undefined},
{{74,<0.7181.510>},undefined},
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.18335.fonsyncd">>},
rabbit_mirror_queue_slave,undefined,-1,undefined,
[<0.7180.510>],
{[],[]},
[],undefined}
** Reason for termination ==
** {{function_clause,
[{rabbit_amqqueue_process,handle_call,
[{add_on_right,{74,<0.7181.510>}},
{<0.7181.510>,#Ref<0.0.14625.121725>},
{q,{amqqueue,
{resource,<<"/fonsyncd/prod/">>,queue,
<<"Q-SID.16970.fonsyncd">>},
true,false,none,
[{<<"x-message-ttl">>,signedint,100000},
{<<"x-dead-letter-exchange">>,longstr,
<<"EX-fonsyncd-Exception">>},
{<<"x-ha-policy">>,longstr,<<"all">>}],
<0.4403.0>,[],all},
none,false,rabbit_mirror_queue_master,
{state,<0.4404.0>,<0.9085.0>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{0,{[],[]}},
{0,{[],[]}},
2114,
{0,nil},
undefined,
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/queues/EYHEI3SQW41F6NFSH8WB2CD91",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
[]},
#Ref<0.0.25.88612>,0,262144,
#Fun<rabbit_variable_queue.2.36908642>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<72,22,217,216,236,0,110,187,235,185,163,249,92,
182,156,56>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,696393,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_persistent",
<0.269.0>,700490,692296,704587,708684},
{client_msstate,msg_store_transient,
<<51,15,67,134,52,182,142,196,131,185,177,5,223,
15,45,61>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,675908,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit at rmq1
/msg_store_transient",
<0.264.0>,680005,671795,684102,688199}},
true,0,#Fun<rabbit_mirror_queue_slave.1.52898136>,0,
0,infinity,0,0,0,0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1365,529204,602738},0},
{{1365,529204,602738},0},
0.0,0.0,
{1365,529286,74299}}},
0,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
[],
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{set,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[<0.17350.499>],
[<0.17781.499>],
[],
[<7525.9758.575>],
[],
[<7526.18741.575>,<7526.16733.575>],
[],[],[],[],[],[],[],[],
[<7525.13515.575>]}}}},
{[],[]},
undefined,undefined,undefined,undefined,
{state,fine,5000,undefined},
{0,nil},
100000,undefined,1365529282744274,
{dict,6,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],
[[<0.17350.499>|#Ref<0.0.14563.86478>]],
[[<0.17781.499>|#Ref<0.0.14563.86169>]],
[],
[[<7525.9758.575>|#Ref<0.0.14563.86395>]],
[],
[[<7526.16733.575>|#Ref<0.0.14563.86260>],
[<7526.18741.575>|#Ref<0.0.14563.86683>]],
[],[],[],[],[],[],[],[],
[[<7525.13515.575>|#Ref<0.0.14563.86620>]]}}},
1,
{{0,nil},{0,nil}},
undefined,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},
{resource,<<"/fonsyncd/prod/">>,exchange,
<<"EX-fonsyncd-Exception">>},
undefined}]},
{gen_server2,handle_msg,2},
{proc_lib,wake_up,3}]},
{gen_server2,call,[<0.4403.0>,{add_on_right,{74,<0.7181.510>}},infinity]}}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20130410/6c5c73d3/attachment.htm>
More information about the rabbitmq-discuss
mailing list