[rabbitmq-discuss] Help High CPU Usage

Brent Williams brentrwill at gmail.com
Sat Jun 7 00:14:11 BST 2014


All,

 

I have a clustered RabbitMQ service running 3.1.5. Both servers are Ubuntu 
12.04 VMs running on ESX Vmware. Both have 16 CPUs and 32 GBs of RAM and 
100 GBs of storage.

 

I have 10 Queues and multiple clients connecting around 1,000. My CPU is 
running at 300 % on the beam.swp process. Also my RabbitMq management 
console is reporting 5800 erlang processes. Any ideas what I might be 
missing? I have attached my rabbitmqctl report output below.

 

Reporting server status on {{2014,6,6},{22,48,23}}

 

...

Status of node 'rabbit at video-rabbitmq02' ...

[{pid,24741},

{running_applications,

     [{rabbitmq_management_visualiser,"RabbitMQ Visualiser","3.1.5"},

      {rabbitmq_management,"RabbitMQ Management Console","3.1.5"},

      {rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},

      {rabbit,"RabbitMQ","3.1.5"},

      {os_mon,"CPO  CXC 138 46","2.2.7"},

      {rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},

      {webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},

      {mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},

      {xmerl,"XML parser","1.2.10"},

      {inets,"INETS  CXC 138 49","5.7.1"},

      {mnesia,"MNESIA  CXC 138 12","4.5"},

      {amqp_client,"RabbitMQ AMQP Client","3.1.5"},

      {sasl,"SASL  CXC 138 11","2.1.10"},

      {stdlib,"ERTS  CXC 138 10","1.17.5"},

      {kernel,"ERTS  CXC 138 10","2.14.5"}]},

{os,{unix,linux}},

{erlang_version,

     "Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] 
[async-threads:30] [kernel-poll:true]\n"},

{memory,

     [{total,170703648},

      {connection_procs,883728},

      {queue_procs,186440},

      {plugins,499872},

      {other_proc,9559536},

      {mnesia,111640},

      {mgmt_db,9904},

      {msg_index,43920},

      {other_ets,1167880},

      {binary,1848960},

      {code,17693226},

      {atom,1569041},

      {other_system,137129501}]},

{vm_memory_high_watermark,0.4},

{vm_memory_limit,12345879756},

{disk_free_limit,1000000000},

{disk_free,31037026304},

{file_descriptors,

     [{total_limit,202300},

      {total_used,14},

      {sockets_limit,182068},

      {sockets_used,12}]},

{processes,[{limit,1048576},{used,344}]},

{run_queue,0},

{uptime,2426}]

 

Cluster status of node 'rabbit at video-rabbitmq02' ...

[{nodes,[{disc,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']}]},

{running_nodes,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']},

{partitions,[]}]

 

Application environment of node 'rabbit at video-rabbitmq02' ...

[{auth_backends,[rabbit_auth_backend_internal]},

{auth_mechanisms,['PLAIN','AMQPLAIN']},

{backing_queue_module,rabbit_variable_queue},

{cluster_nodes,{['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02'],disc}},

{cluster_partition_handling,ignore},

{collect_statistics,fine},

{collect_statistics_interval,5000},

{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},

{default_user,<<"guest">>},

{default_user_tags,[administrator]},

{default_vhost,<<"/">>},

{delegate_count,16},

{disk_free_limit,1000000000},

{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},

{error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq02.log"}},

{frame_max,131072},

{heartbeat,30},

{hipe_compile,false},

{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,

                rabbit_command_assembler,rabbit_framing_amqp_0_9_1,

                rabbit_basic,rabbit_event,lists,queue,priority_queue,

                rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,

                rabbit_exchange_type_direct,rabbit_guid,rabbit_net,

                rabbit_amqqueue_process,rabbit_variable_queue,

                
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,

                sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,

                
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,

                file_handle_cache,rabbit_msg_store,array,

                rabbit_msg_store_ets_index,rabbit_msg_file,

                
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,

                
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,

                ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,

                ssl]},

{included_applications,[]},

{log_levels,[{connection,info}]},

{msg_store_file_size_limit,16777216},

{msg_store_index_module,rabbit_msg_store_ets_index},

{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},

{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at video-rabbitmq02-plugins-expand"},

{queue_index_max_journal_entries,65536},

{reverse_dns_lookups,false},

{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq02-sasl.log
"}},

{server_properties,[]},

{ssl_apps,[asn1,crypto,public_key,ssl]},

{ssl_cert_login_from,distinguished_name},

{ssl_listeners,[]},

{ssl_options,[]},

{tcp_listen_options,[binary,

                      {packet,raw},

                      {reuseaddr,true},

                      {backlog,128},

                      {nodelay,true},

                      {linger,{true,0}},

                      {exit_on_close,false}]},

{tcp_listeners,[5672]},

{trace_vhosts,[]},

{vm_memory_high_watermark,0.4}]

 

Status of node 'rabbit at video-rabbitmq01' ...

[{pid,635},

{running_applications,

     [{rabbitmq_management_visualiser,"RabbitMQ Visualiser","3.1.5"},

      {rabbitmq_management,"RabbitMQ Management Console","3.1.5"},

      {rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},

      {rabbit,"RabbitMQ","3.1.5"},

      {os_mon,"CPO  CXC 138 46","2.2.7"},

      {rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},

      {webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},

      {mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},

      {xmerl,"XML parser","1.2.10"},

      {inets,"INETS  CXC 138 49","5.7.1"},

      {mnesia,"MNESIA  CXC 138 12","4.5"},

      {amqp_client,"RabbitMQ AMQP Client","3.1.5"},

      {sasl,"SASL  CXC 138 11","2.1.10"},

      {stdlib,"ERTS  CXC 138 10","1.17.5"},

      {kernel,"ERTS  CXC 138 10","2.14.5"}]},

{os,{unix,linux}},

{erlang_version,

     "Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] 
[async-threads:30] [kernel-poll:true]\n"},

{memory,

     [{total,192724600},

      {connection_procs,64347776},

      {queue_procs,684680},

      {plugins,656872},

      {other_proc,13256928},

      {mnesia,111608},

      {mgmt_db,16321672},

      {msg_index,43920},

      {other_ets,2051136},

      {binary,4663672},

      {code,17271483},

      {atom,1551265},

      {other_system,71763588}]},

{vm_memory_high_watermark,0.4},

{vm_memory_limit,13402839449},

{disk_free_limit,1000000000},

{disk_free,34438590464},

{file_descriptors,

     [{total_limit,102300},

      {total_used,759},

      {sockets_limit,92068},

      {sockets_used,757}]},

{processes,[{limit,1048576},{used,8547}]},

{run_queue,125},

{uptime,3741}]

 

Cluster status of node 'rabbit at video-rabbitmq01' ...

[{nodes,[{disc,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']}]},

{running_nodes,['rabbit at video-rabbitmq02','rabbit at video-rabbitmq01']},

{partitions,[]}]

 

Application environment of node 'rabbit at video-rabbitmq01' ...

[{auth_backends,[rabbit_auth_backend_internal]},

{auth_mechanisms,['PLAIN','AMQPLAIN']},

{backing_queue_module,rabbit_variable_queue},

{cluster_nodes,{['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02'],disc}},

{cluster_partition_handling,ignore},

{collect_statistics,fine},

{collect_statistics_interval,5000},

{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},

{default_user,<<"guest">>},

{default_user_tags,[administrator]},

{default_vhost,<<"/">>},

{delegate_count,16},

{disk_free_limit,1000000000},

{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},

{error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq01.log"}},

{frame_max,131072},

{heartbeat,30},

{hipe_compile,false},

{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,

                rabbit_command_assembler,rabbit_framing_amqp_0_9_1,

                rabbit_basic,rabbit_event,lists,queue,priority_queue,

                rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,

                rabbit_exchange_type_direct,rabbit_guid,rabbit_net,

                rabbit_amqqueue_process,rabbit_variable_queue,

                
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,

                sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,

                
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,

                file_handle_cache,rabbit_msg_store,array,

                rabbit_msg_store_ets_index,rabbit_msg_file,

                
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,

                
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,

                ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,

                ssl]},

{included_applications,[]},

{log_levels,[{connection,info}]},

{msg_store_file_size_limit,16777216},

{msg_store_index_module,rabbit_msg_store_ets_index},

{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},

{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at video-rabbitmq01-plugins-expand"},

{queue_index_max_journal_entries,65536},

{reverse_dns_lookups,false},

{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq01-sasl.log
"}},

{server_properties,[]},

{ssl_apps,[asn1,crypto,public_key,ssl]},

{ssl_cert_login_from,distinguished_name},

{ssl_listeners,[]},

{ssl_options,[]},

{tcp_listen_options,[binary,

                      {packet,raw},

                      {reuseaddr,true},

                      {backlog,128},

                      {nodelay,true},

                      {linger,{true,0}},

                      {exit_on_close,false}]},

{tcp_listeners,[5672]},

{trace_vhosts,[]},

{vm_memory_high_watermark,0.4}]

Consumers on /:

 Permissions on /:

user       configure             write     read

guest     .*            .*            .*

 

Policies on /:

vhost     name    pattern definition            priority

/              ha-all     ^.            {"ha-mode":"all"}            0

 

Parameters on /:

 

...done.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20140606/2afc8dc3/attachment.html>


More information about the rabbitmq-discuss mailing list