[rabbitmq-discuss] High CPU Usage
Williams, Brent (Snapfish Web Platform)
brent.williams at hp.com
Fri Jun 6 23:59:58 BST 2014
All,
I have a clustered RabbitMQ service running 3.1.5. Both servers are Ubuntu 12.04 VMs running on ESX Vmware. Both have 16 CPUS and 32 GBs. Of RAM and 100 GBs of storage.
I have 10 Queues and multiple clients connecting around 1,000. My CPU is running at 300 % on the beam.swp process. Also my RabbitMq management console is reporting 5800 erlang processes. Any ideas what I might be missing? I have attached my rabbitmqctl report output below.
Reporting server status on {{2014,6,6},{22,48,23}}
...
Status of node 'rabbit at video-rabbitmq02' ...
[{pid,24741},
{running_applications,
[{rabbitmq_management_visualiser,"RabbitMQ Visualiser","3.1.5"},
{rabbitmq_management,"RabbitMQ Management Console","3.1.5"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},
{rabbit,"RabbitMQ","3.1.5"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},
{webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.1.5"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] [async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,170703648},
{connection_procs,883728},
{queue_procs,186440},
{plugins,499872},
{other_proc,9559536},
{mnesia,111640},
{mgmt_db,9904},
{msg_index,43920},
{other_ets,1167880},
{binary,1848960},
{code,17693226},
{atom,1569041},
{other_system,137129501}]},
{vm_memory_high_watermark,0.4},
{vm_memory_limit,12345879756},
{disk_free_limit,1000000000},
{disk_free,31037026304},
{file_descriptors,
[{total_limit,202300},
{total_used,14},
{sockets_limit,182068},
{sockets_used,12}]},
{processes,[{limit,1048576},{used,344}]},
{run_queue,0},
{uptime,2426}]
Cluster status of node 'rabbit at video-rabbitmq02' ...
[{nodes,[{disc,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']}]},
{running_nodes,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']},
{partitions,[]}]
Application environment of node 'rabbit at video-rabbitmq02' ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02'],disc}},
{cluster_partition_handling,ignore},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq02.log"}},
{frame_max,131072},
{heartbeat,30},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},
{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at video-rabbitmq02-plugins-expand"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,false},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq02-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.4}]
Status of node 'rabbit at video-rabbitmq01' ...
[{pid,635},
{running_applications,
[{rabbitmq_management_visualiser,"RabbitMQ Visualiser","3.1.5"},
{rabbitmq_management,"RabbitMQ Management Console","3.1.5"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","3.1.5"},
{rabbit,"RabbitMQ","3.1.5"},
{os_mon,"CPO CXC 138 46","2.2.7"},
{rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.1.5"},
{webmachine,"webmachine","1.10.3-rmq3.1.5-gite9359c7"},
{mochiweb,"MochiMedia Web Server","2.7.0-rmq3.1.5-git680dba8"},
{xmerl,"XML parser","1.2.10"},
{inets,"INETS CXC 138 49","5.7.1"},
{mnesia,"MNESIA CXC 138 12","4.5"},
{amqp_client,"RabbitMQ AMQP Client","3.1.5"},
{sasl,"SASL CXC 138 11","2.1.10"},
{stdlib,"ERTS CXC 138 10","1.17.5"},
{kernel,"ERTS CXC 138 10","2.14.5"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R14B04 (erts-5.8.5) [source] [64-bit] [smp:16:16] [rq:16] [async-threads:30] [kernel-poll:true]\n"},
{memory,
[{total,192724600},
{connection_procs,64347776},
{queue_procs,684680},
{plugins,656872},
{other_proc,13256928},
{mnesia,111608},
{mgmt_db,16321672},
{msg_index,43920},
{other_ets,2051136},
{binary,4663672},
{code,17271483},
{atom,1551265},
{other_system,71763588}]},
{vm_memory_high_watermark,0.4},
{vm_memory_limit,13402839449},
{disk_free_limit,1000000000},
{disk_free,34438590464},
{file_descriptors,
[{total_limit,102300},
{total_used,759},
{sockets_limit,92068},
{sockets_used,757}]},
{processes,[{limit,1048576},{used,8547}]},
{run_queue,125},
{uptime,3741}]
Cluster status of node 'rabbit at video-rabbitmq01' ...
[{nodes,[{disc,['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02']}]},
{running_nodes,['rabbit at video-rabbitmq02','rabbit at video-rabbitmq01']},
{partitions,[]}]
Application environment of node 'rabbit at video-rabbitmq01' ...
[{auth_backends,[rabbit_auth_backend_internal]},
{auth_mechanisms,['PLAIN','AMQPLAIN']},
{backing_queue_module,rabbit_variable_queue},
{cluster_nodes,{['rabbit at video-rabbitmq01','rabbit at video-rabbitmq02'],disc}},
{cluster_partition_handling,ignore},
{collect_statistics,fine},
{collect_statistics_interval,5000},
{default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
{default_user,<<"guest">>},
{default_user_tags,[administrator]},
{default_vhost,<<"/">>},
{delegate_count,16},
{disk_free_limit,1000000000},
{enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
{error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq01.log"}},
{frame_max,131072},
{heartbeat,30},
{hipe_compile,false},
{hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
rabbit_basic,rabbit_event,lists,queue,priority_queue,
rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
rabbit_amqqueue_process,rabbit_variable_queue,
rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
file_handle_cache,rabbit_msg_store,array,
rabbit_msg_store_ets_index,rabbit_msg_file,
rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
ssl]},
{included_applications,[]},
{log_levels,[{connection,info}]},
{msg_store_file_size_limit,16777216},
{msg_store_index_module,rabbit_msg_store_ets_index},
{plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.1.5/sbin/../plugins"},
{plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit at video-rabbitmq01-plugins-expand"},
{queue_index_max_journal_entries,65536},
{reverse_dns_lookups,false},
{sasl_error_logger,{file,"/var/log/rabbitmq/rabbit at video-rabbitmq01-sasl.log"}},
{server_properties,[]},
{ssl_apps,[asn1,crypto,public_key,ssl]},
{ssl_cert_login_from,distinguished_name},
{ssl_listeners,[]},
{ssl_options,[]},
{tcp_listen_options,[binary,
{packet,raw},
{reuseaddr,true},
{backlog,128},
{nodelay,true},
{linger,{true,0}},
{exit_on_close,false}]},
{tcp_listeners,[5672]},
{trace_vhosts,[]},
{vm_memory_high_watermark,0.4}]
name type durable auto_delete internal arguments policy
direct true false false []
amq.direct direct true false false [] ha-all
amq.fanout fanout true false false [] ha-all
amq.headers headers true false false [] ha-all
amq.match headers true false false [] ha-all
amq.rabbitmq.log topic true false false [] ha-all
amq.rabbitmq.trace topic true false false [] ha-all
amq.topic topic true false false [] ha-all
default direct true false false [] ha-all
snapfish.video.transcode.completedjob direct true false false [] ha-all
snapfish.video.transcode.startjob direct true false false [] ha-all
Bindings on /:
source_name source_kind destination_name destination_kind routing_key arguments
exchange POD1_SFPRD_FINISHED_JOBS queue POD1_SFPRD_FINISHED_JOBS []
exchange POD1_SFQA_FINISHED_JOBS queue POD1_SFQA_FINISHED_JOBS []
exchange POD2_SFPRD_FINISHED_JOBS queue POD2_SFPRD_FINISHED_JOBS []
exchange POD2_SFQA_FINISHED_JOBS queue POD2_SFQA_FINISHED_JOBS []
exchange POD3_SFPRD_FINISHED_JOBS queue POD3_SFPRD_FINISHED_JOBS []
exchange POD4_SFPRD_FINISHED_JOBS queue POD4_SFPRD_FINISHED_JOBS []
exchange POD5_SFPRD_FINISHED_JOBS queue POD5_SFPRD_FINISHED_JOBS []
exchange snapfish.video.transcode.startjob queue snapfish.video.transcode.startjob []
default exchange POD1_SFPRD_FINISHED_JOBS queue POD1_SFPRD_FINISHED_JOBS []
default exchange POD1_SFPRD_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD1_SFQA_FINISHED_JOBS queue POD1_SFQA_FINISHED_JOBS []
default exchange POD1_SFQA_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD2_SFPRD_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD2_SFQA_FINISHED_JOBS queue POD2_SFQA_FINISHED_JOBS []
default exchange POD2_SFQA_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD3_SFPRD_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD4_SFPRD_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange POD5_SFPRD_FINISHED_JOBS queue snapfish.transcode.cj []
default exchange snapfish.video.transcode.startjob queue snapfish.video.transcode.startjob []
Consumers on /:
Permissions on /:
user configure write read
guest .* .* .*
Policies on /:
vhost name pattern definition priority
/ ha-all ^. {"ha-mode":"all"} 0
Parameters on /:
...done.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/attachments/20140606/86cf0fd1/attachment.html>
More information about the rabbitmq-discuss
mailing list