[rabbitmq-discuss] RabbitMQ 3.2.2 cluster shutting down

[Andy Martin]

=ERROR REPORT==== 6-Mar-2014::09:59:33 ===
** Generic server <0.298.0> terminating
** Last message in was pre_hibernate
** When Server state == {q,
                         {amqqueue,
                          {resource,<<"/">>,queue,
                          
 <<"SUBSCRIBER_PACE_PORTFOLIO_BOSAPPDEVSMC001_7E6F5793-2230-4704-9B63-CCEAF4FC5F38">>},
                          false,false,none,
                          [{<<"x-expires">>,signedint,600000}],
                          <0.298.0>,[],[],
                          [{vhost,<<"/">>},
                           {name,<<"HA">>},
                           {pattern,<<"^SUBSCRIBER_">>},
                           {'apply-to',<<"queues">>},
                           {definition,
                            [{<<"ha-mode">>,<<"all">>},
                             {<<"ha-sync-mode">>,<<"automatic">>}]},
                           {priority,0}],
                          [{<0.299.0>,<0.298.0>}],
                          []},
                         none,true,rabbit_mirror_queue_master,
                         {state,
                          {resource,<<"/">>,queue,
                          
 <<"SUBSCRIBER_PACE_PORTFOLIO_BOSAPPDEVSMC001_7E6F5793-2230-4704-9B63-CCEAF4FC5F38">>},
                          <0.299.0>,<0.19972.36>,rabbit_variable_queue,
                          {vqstate,
                           {0,{[],[]}},
                           {0,{[],[]}},
                           {delta,undefined,0,undefined},
                           {0,{[],[]}},
                           {0,{[],[]}},
                           381,
                           {0,nil},
                           {0,nil},
                           {qistate,
                            
"c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit at MARRISDEVFAB201-mnesia/queues/6WOGOU8HI3AF43KE7Y8YAHG2G",
                            {{dict,0,16,16,8,80,48,
                              {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                               []},
                              
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                                []}}},
                             []},
                            undefined,0,65536,
                            #Fun<rabbit_variable_queue.2.81334491>,
                            {0,nil}},
                           {undefined,
                            {client_msstate,msg_store_transient,
                             <<215,212,149,239,187,145,76,128,128,7,227,249,
                               174,103,162,200>>,
                             {dict,0,16,16,8,80,48,
                              {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                               []},
                              
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                                []}}},
                             {state,294982,
                              
"c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit at MARRISDEVFAB201-mnesia/msg_store_transient"},
                             rabbit_msg_store_ets_index,
                            
 "c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit at MARRISDEVFAB201-mnesia/msg_store_transient",
                             <0.241.0>,299079,290885,303176,307273}},
                           false,0,0,0,infinity,0,0,0,0,0,
                           {rates,
                            {{1394,98997,711100},0},
                            {{1394,98997,711100},0},
                            0.0,0.0,
                            {1394,117373,529281}},
                           {0,nil},
                           {0,nil},
                           {0,nil},
                           {0,nil},
                           0,0,
                           {rates,
                            {{1394,98997,711100},0},
                            {{1394,98997,711100},0},
                            0.0,0.0,
                            {1394,117373,529281}}},
                          {dict,0,16,16,8,80,48,
                          
 {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                             []}}},
                          [],
                          {set,0,16,16,8,80,48,
                          
 {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                             []}}}},
                         {queue,
                          [{<0.19777.36>,
                            {consumer,<<"amq.ctag-k8a2E1BDwyCVG2ncSPsbnA">>,
                             true,[]}}],
                          [],1},
                         600000,undefined,#Ref<0.0.31.217038>,undefined,
                         {state,fine,5000,#Ref<0.0.31.217039>},
                         {0,nil},
                         undefined,undefined,undefined,
                         {state,
                          {dict,0,16,16,8,80,48,
                          
 {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                             []}}},
                          delegate},
                         undefined,undefined,undefined,3,running}
** Reason for termination == 
** {{badmatch,{error,not_found}},
    [{rabbit_mirror_queue_master,stop_all_slaves,2,[]},
     {rabbit_mirror_queue_master,delete_and_terminate,2,[]},
     {rabbit_amqqueue_process,'-terminate_delete/3-fun-1-',6,[]},
     {rabbit_amqqueue_process,terminate_shutdown,2,[]},
     {gen_server2,terminate,3,[]},
     {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}
** In 'terminate' callback with reason ==
** {{badmatch,{error,not_found}},
    [{rabbit_amqqueue_process,i,2,[]},
     {rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2,[]},
     {rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2,[]},
     {rabbit_amqqueue_process,emit_stats,2,[]},
     {rabbit_event,if_enabled,3,[]},
     {rabbit_amqqueue_process,handle_pre_hibernate,1,[]},
     {gen_server2,pre_hibernate,1,[]},
     {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]


On Thursday, March 6, 2014 9:55:28 AM UTC-5, Andy Martin wrote:
>
> I have a two-node autoheal cluster. Yesterday I came in to find out that 
> neither server in the cluster was seeing the other, but there was no 
> message about a partition having occurred. Today I came in to find one of 
> the servers down. I can't find anything in the logs that would indicate why 
> this is happening. What should I be looking for in the logs?
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20140306/d795b773/attachment.html>