<div dir="ltr"><div>=ERROR REPORT==== 6-Mar-2014::09:59:33 ===</div><div>** Generic server <0.298.0> terminating</div><div>** Last message in was pre_hibernate</div><div>** When Server state == {q,</div><div>                         {amqqueue,</div><div>                          {resource,<<"/">>,queue,</div><div>                           <<"SUBSCRIBER_PACE_PORTFOLIO_BOSAPPDEVSMC001_7E6F5793-2230-4704-9B63-CCEAF4FC5F38">>},</div><div>                          false,false,none,</div><div>                          [{<<"x-expires">>,signedint,600000}],</div><div>                          <0.298.0>,[],[],</div><div>                          [{vhost,<<"/">>},</div><div>                           {name,<<"HA">>},</div><div>                           {pattern,<<"^SUBSCRIBER_">>},</div><div>                           {'apply-to',<<"queues">>},</div><div>                           {definition,</div><div>                            [{<<"ha-mode">>,<<"all">>},</div><div>                             {<<"ha-sync-mode">>,<<"automatic">>}]},</div><div>                           {priority,0}],</div><div>                          [{<0.299.0>,<0.298.0>}],</div><div>                          []},</div><div>                         none,true,rabbit_mirror_queue_master,</div><div>                         {state,</div><div>                          {resource,<<"/">>,queue,</div><div>                           <<"SUBSCRIBER_PACE_PORTFOLIO_BOSAPPDEVSMC001_7E6F5793-2230-4704-9B63-CCEAF4FC5F38">>},</div><div>                          <0.299.0>,<0.19972.36>,rabbit_variable_queue,</div><div>                          {vqstate,</div><div>                           {0,{[],[]}},</div><div>                           {0,{[],[]}},</div><div>                           {delta,undefined,0,undefined},</div><div>                           {0,{[],[]}},</div><div>                           {0,{[],[]}},</div><div>                           381,</div><div>                           {0,nil},</div><div>                           {0,nil},</div><div>                           {qistate,</div><div>                            "c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit@MARRISDEVFAB201-mnesia/queues/6WOGOU8HI3AF43KE7Y8YAHG2G",</div><div>                            {{dict,0,16,16,8,80,48,</div><div>                              {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                               []},</div><div>                              {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                                []}}},</div><div>                             []},</div><div>                            undefined,0,65536,</div><div>                            #Fun<rabbit_variable_queue.2.81334491>,</div><div>                            {0,nil}},</div><div>                           {undefined,</div><div>                            {client_msstate,msg_store_transient,</div><div>                             <<215,212,149,239,187,145,76,128,128,7,227,249,</div><div>                               174,103,162,200>>,</div><div>                             {dict,0,16,16,8,80,48,</div><div>                              {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                               []},</div><div>                              {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                                []}}},</div><div>                             {state,294982,</div><div>                              "c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit@MARRISDEVFAB201-mnesia/msg_store_transient"},</div><div>                             rabbit_msg_store_ets_index,</div><div>                             "c:/Users/admin-amartin/AppData/Roaming/RabbitMQ/db/rabbit@MARRISDEVFAB201-mnesia/msg_store_transient",</div><div>                             <0.241.0>,299079,290885,303176,307273}},</div><div>                           false,0,0,0,infinity,0,0,0,0,0,</div><div>                           {rates,</div><div>                            {{1394,98997,711100},0},</div><div>                            {{1394,98997,711100},0},</div><div>                            0.0,0.0,</div><div>                            {1394,117373,529281}},</div><div>                           {0,nil},</div><div>                           {0,nil},</div><div>                           {0,nil},</div><div>                           {0,nil},</div><div>                           0,0,</div><div>                           {rates,</div><div>                            {{1394,98997,711100},0},</div><div>                            {{1394,98997,711100},0},</div><div>                            0.0,0.0,</div><div>                            {1394,117373,529281}}},</div><div>                          {dict,0,16,16,8,80,48,</div><div>                           {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},</div><div>                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                             []}}},</div><div>                          [],</div><div>                          {set,0,16,16,8,80,48,</div><div>                           {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},</div><div>                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                             []}}}},</div><div>                         {queue,</div><div>                          [{<0.19777.36>,</div><div>                            {consumer,<<"amq.ctag-k8a2E1BDwyCVG2ncSPsbnA">>,</div><div>                             true,[]}}],</div><div>                          [],1},</div><div>                         600000,undefined,#Ref<0.0.31.217038>,undefined,</div><div>                         {state,fine,5000,#Ref<0.0.31.217039>},</div><div>                         {0,nil},</div><div>                         undefined,undefined,undefined,</div><div>                         {state,</div><div>                          {dict,0,16,16,8,80,48,</div><div>                           {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},</div><div>                           {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],</div><div>                             []}}},</div><div>                          delegate},</div><div>                         undefined,undefined,undefined,3,running}</div><div>** Reason for termination == </div><div>** {{badmatch,{error,not_found}},</div><div>    [{rabbit_mirror_queue_master,stop_all_slaves,2,[]},</div><div>     {rabbit_mirror_queue_master,delete_and_terminate,2,[]},</div><div>     {rabbit_amqqueue_process,'-terminate_delete/3-fun-1-',6,[]},</div><div>     {rabbit_amqqueue_process,terminate_shutdown,2,[]},</div><div>     {gen_server2,terminate,3,[]},</div><div>     {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}</div><div>** In 'terminate' callback with reason ==</div><div>** {{badmatch,{error,not_found}},</div><div>    [{rabbit_amqqueue_process,i,2,[]},</div><div>     {rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2,[]},</div><div>     {rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2,[]},</div><div>     {rabbit_amqqueue_process,emit_stats,2,[]},</div><div>     {rabbit_event,if_enabled,3,[]},</div><div>     {rabbit_amqqueue_process,handle_pre_hibernate,1,[]},</div><div>     {gen_server2,pre_hibernate,1,[]},</div><div>     {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]</div><div><br></div><br>On Thursday, March 6, 2014 9:55:28 AM UTC-5, Andy Martin wrote:<blockquote class="gmail_quote" style="margin: 0;margin-left: 0.8ex;border-left: 1px #ccc solid;padding-left: 1ex;"><div dir="ltr">I have a two-node autoheal cluster. Yesterday I came in to find out that neither server in the cluster was seeing the other, but there was no message about a partition having occurred. Today I came in to find one of the servers down. I can't find anything in the logs that would indicate why this is happening. What should I be looking for in the logs?</div></blockquote></div>