[rabbitmq-discuss] Rabbitmq 3.2.4 running out of memory
srikanth tns
srikanthtns at gmail.com
Tue Aug 26 19:47:28 BST 2014
The UI is not accessible , The 3-nodes are clustered with HA two policy
rabbitmqctl reports shows
Reporting server status on {{2014,8,26},{18,39,31}}
...
Status of node rabbit at adsmv0020 ...
[{pid,22153},
{running_applications,
[{rabbitmq_stomp,"Embedded Rabbit Stomp Adapter","3.2.4"},
{rabbitmq_management,"RabbitMQ Management Console","3.2.4"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.2.4"},
{rabbit,"RabbitMQ","3.2.4"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.2.4"},
{webmachine,"webmachine","1.10.3-rmq3.2.4-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.2.4-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.2.4"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:4:4] [rq:4]
[async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,2285543720},
{connection_procs,207275088},
{queue_procs,599282104},
{plugins,234739776},
{other_proc,309911344},
{mnesia,289075584},
{mgmt_db,9096},
{msg_index,14688048},
{other_ets,28236624},
{binary,290512400},
{code,17776116},
{atom,1621321},
{other_system,292416219}]},
{vm_memory_high_watermark,0.7},
{vm_memory_limit,11753753804},
{disk_free_limit,1000000000},
{disk_free,19386892288},
{file_descriptors,
[{total_limit,327579},
{total_used,3137},
{sockets_limit,294819},
{sockets_used,3135}]},
{processes,[{limit,1048576},{used,144321}]},
{run_queue,0},
{uptime,6267}]
Cluster status of node rabbit at adsmv0020 ...
[{nodes,[{disc,[rabbit at adsmv0019,rabbit at adsmv0018]},{ram,[rabbit at adsmv0020
]}]},
{running_nodes,[rabbit at adsmv0018,rabbit at adsmv0019,rabbit at adsmv0020]},
{partitions,[]}]
Application environment of node rabbit at adsmv0020 ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{[],disc}},
{cluster_partition_handling,autoheal},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0020.log"}},
{frame_max,131072},
{halt_on_upgrade_failure,true},
{heartbeat,580},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.2.4/sbin/../plugins"},
{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at adsmv0020-plugins-expand
"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,true},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0020-sasl.log"}},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0020-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.7},
{vm_memory_high_watermark_paging_ratio,0.7}]
Status of node rabbit at adsmv0019 ...
[{pid,27139},
{running_applications,
[{rabbitmq_stomp,"Embedded Rabbit Stomp Adapter","3.2.4"},
{rabbitmq_management,"RabbitMQ Management Console","3.2.4"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.2.4"},
{rabbit,"RabbitMQ","3.2.4"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.2.4"},
{webmachine,"webmachine","1.10.3-rmq3.2.4-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.2.4-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.2.4"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:4:4] [rq:4]
[async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,2375638736},
{connection_procs,228104952},
{queue_procs,643238504},
{plugins,259036872},
{other_proc,276878920},
{mnesia,288547136},
{mgmt_db,9096},
{msg_index,14503600},
{other_ets,30000664},
{binary,321298960},
{binary,321298960},
{code,17776116},
{atom,1621321},
{other_system,294622595}]},
{vm_memory_high_watermark,0.7},
{vm_memory_limit,11753753804},
{disk_free_limit,1000000000},
{disk_free,19439706112},
{file_descriptors,
[{total_limit,327579},
{total_used,3466},
{sockets_limit,294819},
{sockets_used,3464}]},
{processes,[{limit,1048576},{used,159459}]},
{run_queue,0},
{uptime,6302}]
Cluster status of node rabbit at adsmv0019 ...
[{nodes,[{disc,[rabbit at adsmv0018,rabbit at adsmv0019]},{ram,[rabbit at adsmv0020
]}]},
{running_nodes,[rabbit at adsmv0020,rabbit at adsmv0018,rabbit at adsmv0019]},
{partitions,[]}]
Application environment of node rabbit at adsmv0019 ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{[],disc}},
{cluster_partition_handling,autoheal},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0019.log"}},
{frame_max,131072},
{halt_on_upgrade_failure,true},
{heartbeat,580},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.2.4/sbin/../plugins"},
{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at adsmv0019-plugins-expand
"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,true},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0019-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.7},
{vm_memory_high_watermark_paging_ratio,0.7}]
Status of node rabbit at adsmv0018 ...
[{pid,20727},
{running_applications,
[{rabbitmq_stomp,"Embedded Rabbit Stomp Adapter","3.2.4"},
{rabbitmq_management,"RabbitMQ Management Console","3.2.4"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.2.4"},
{rabbit,"RabbitMQ","3.2.4"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.2.4"},
{webmachine,"webmachine","1.10.3-rmq3.2.4-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.2.4-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.2.4"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:4:4] [rq:4]
[async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,6095435856},
{connection_procs,185360136},
{queue_procs,526782896},
{plugins,-1872186488},
{other_proc,2447546120},
{mnesia,289695904},
{mgmt_db,3870226464},
{msg_index,23253920},
{other_ets,51366128},
{binary,249989160},
{code,17776116},
{atom,1622129},
{other_system,304003371}]},
{vm_memory_high_watermark,0.7},
{vm_memory_limit,11753753804},
{disk_free_limit,1000000000},
{disk_free,19338166272},
{file_descriptors,
[{total_limit,327579},
{total_used,2748},
{sockets_limit,294819},
{sockets_used,2746}]},
{processes,[{limit,1048576},{used,126430}]},
{run_queue,0},
{uptime,6789}]
Cluster status of node rabbit at adsmv0018 ...
[{nodes,[{disc,[rabbit at adsmv0018,rabbit at adsmv0019]},{ram,[rabbit at adsmv0020
]}]},
{running_nodes,[rabbit at adsmv0020,rabbit at adsmv0019,rabbit at adsmv0018]},
{partitions,[]}]
Application environment of node rabbit at adsmv0018 ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{[],disc}},
{cluster_partition_handling,autoheal},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0018.log"}},
{frame_max,131072},
{halt_on_upgrade_failure,true},
{heartbeat,580},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.2.4/sbin/../plugins"},
{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at adsmv0018-plugins-expand
"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,true},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at adsmv0018-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.7},
{vm_memory_high_watermark_paging_ratio,0.7}]
On Tue, Aug 26, 2014 at 11:42 AM, Michael Klishin <mklishin at pivotal.io>
wrote:
> On 26 August 2014 at 20:02:09, srikanth tns (srikanthtns at gmail.com) wrote:
> > > netstat -tan | grep ':15672' | awk '{print $6 }' | sort | uniq -c
> > 3 CLOSE_WAIT
> > 1882 ESTABLISHED
> > 1 LISTEN
> > 10 TIME_WAIT
>
> That's about 1900 concurrent connections to a single node. Without knowing
> how many channels can be open on the open ones, that's still a fairly
> large number for a single node.
>
> Consider clustering this node with a couple more and spread the connections
> between them, e.g. using an HAproxy instance in front of the cluster.
> --
> MK
>
> Staff Software Engineer, Pivotal/RabbitMQ
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20140826/43a33528/attachment.html>
More information about the rabbitmq-discuss
mailing list