[rabbitmq-discuss] Why rabbitmq stop pushing message to consumer
qiujixiong
qiujixiong at 163.com
Fri Apr 25 10:09:49 BST 2014
Hi,
I have a problem with rabbitmq recently .
The message in queue accumulated and the queue become more and more bigger(about hundred thousands of messages in queue). Then I found rabbitmq did not push message to consumers and all consumers were blocked on tcp receive function.
While the publishers were continue publishing message at that time. (When I found the channel was blocked, I would try to re-create a connection to publish message. Am I wrong to publish message in such a way?)
Is stopping pushing message to consumers the reason of message accumulated or the result of message accumulated? Another thing, the message is durable, and disk io at that time was almost full. May this be the reason of message accumulated? Then, still, why consumers could not receive message from rabbitmq?
Can anyone help? Thanks very much in advanced!
Here is the status of rabbitmq I dump when problem occur:
Reporting server status on {{2014,4,23},{14,50,1}}
...
Status of node rabbit4 at rabbitmq153 ...
[{pid,8934},
{running_applications,
[{rabbitmq_federation_management,"RabbitMQ Federation Management",
"3.1.5"},
{rabbitmq_management,"RabbitMQ Management Console","3.1.5"},
{rabbitmq_federation,"RabbitMQ Federation","3.1.5"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},
{rabbit,"RabbitMQ","3.1.5"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},
{webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.1.5"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] [async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,2384958696},
{connection_procs,243278664},
{queue_procs,1295680360},
{plugins,2061384},
{other_proc,189406552},
{mnesia,138736},
{mgmt_db,54466288},
{msg_index,8958408},
{other_ets,37974792},
{binary,367954632},
{code,17812261},
{atom,2285609},
{other_system,164941010}]},
{vm_memory_high_watermark,0.9},
{vm_memory_limit,15137044070},
{disk_free_limit,1000000000},
{disk_free,219615227904},
{file_descriptors,
[{total_limit,262040},
{total_used,4426},
{sockets_limit,235834},
{sockets_used,4422}]},
{processes,[{limit,1048576},{used,39049}]},
{run_queue,12},
{uptime,3590787}]
Cluster status of node rabbit4 at rabbitmq153 ...
[{nodes,[{disc,[rabbit3 at rabbitmq152,rabbit4 at rabbitmq153]}]},
{running_nodes,[rabbit3 at rabbitmq152,rabbit4 at rabbitmq153]},
{partitions,[]}]
Application environment of node rabbit4 at rabbitmq153 ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{[],disc}},
{cluster_partition_handling,ignore},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/data/yy/log/rabbitmq/rabbit4 at rabbitmq153.log"}},
{frame_max,131072},
{heartbeat,600},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},
{plugins_expand_dir,"/data/rabbitmq/rabbit4 at rabbitmq153-plugins-expand"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,false},
{sasl_error_logger,{file,"/data/yy/log/rabbitmq/rabbit4 at rabbitmq153-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.9},
{vm_memory_high_watermark_paging_ratio,0.9}]
Status of node rabbit3 at rabbitmq152 ...
[{pid,21118},
{running_applications,
[{rabbitmq_federation_management,"RabbitMQ Federation Management",
"3.1.5"},
{rabbitmq_management,"RabbitMQ Management Console","3.1.5"},
{rabbitmq_federation,"RabbitMQ Federation","3.1.5"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},
{rabbit,"RabbitMQ","3.1.5"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},
{webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.1.5"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] [async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,1948612952},
{connection_procs,273728912},
{queue_procs,1262564272},
{plugins,577536},
{other_proc,15237064},
{mnesia,138720},
{mgmt_db,9904},
{msg_index,3223064},
{other_ets,27246728},
{binary,198736656},
{code,17812261},
{atom,1624553},
{other_system,147713282}]},
{vm_memory_high_watermark,0.9},
{vm_memory_limit,15136985088},
{disk_free_limit,1000000000},
{disk_free,221611429888},
{file_descriptors,
[{total_limit,262040},
{total_used,1010},
{sockets_limit,235834},
{sockets_used,1006}]},
{processes,[{limit,1048576},{used,9175}]},
{run_queue,0},
{uptime,47734}]
Cluster status of node rabbit3 at rabbitmq152 ...
[{nodes,[{disc,[rabbit3 at rabbitmq152,rabbit4 at rabbitmq153]}]},
{running_nodes,[rabbit4 at rabbitmq153,rabbit3 at rabbitmq152]},
{partitions,[]}]
Application environment of node rabbit3 at rabbitmq152 ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{[],disc}},
{cluster_partition_handling,ignore},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/data/yy/log/rabbitmq/rabbit3 at rabbitmq152.log"}},
{frame_max,131072},
{heartbeat,600},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},
{plugins_expand_dir,"/data/rabbitmq/rabbit3 at rabbitmq152-plugins-expand"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,false},
{sasl_error_logger,{file,"/data/yy/log/rabbitmq/rabbit3 at rabbitmq152-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.9},
{vm_memory_high_watermark_paging_ratio,0.9}]
Connections:
(the report stopped here and no further info show)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20140425/2d359c9d/attachment.html>
More information about the rabbitmq-discuss
mailing list