[rabbitmq-discuss] Per-Connection Flow Control - RMQ 2.8.1
DawgTool
dawgtool at aol.com
Wed Mar 28 13:43:51 BST 2012
Hi Simon,
Thanks for looking into this. This is a really large project which we
will be scaling to handle ~300k msg/sec, right now 2.7.1 is running at
90k msg/sec.
Here is my Configuration (UAT): (2 servers)
x4: Intel(R) Xeon(R) CPU X5650 @ 2.67GHz
x1: MemTotal: 16777216 kB
Number of publishers: 20
Testing with TTL only, No consumers (testing for durability during
maintenance windows on consumers)
Publishing Rate: 100 - 350 messages/sec per publisher (ie: 100 x 20 =
2000) to app.publish exchange
x-message-ttl at 10000 should cause flow control to block every second
to 10 seconds
x-message-ttl at 60000 should cause flow control to block every second
to 10 seconds, as queue reaches ~1gB expiring will lock the queue and
everything will block for several minutes.
==> *rabbitmq status* <==
Linux 2.6.18-238.el5xen #1 SMP Thu Jan 13 16:41:45 EST 2011 x86_64
x86_64 x86_64 GNU/Linux (CentOS 5)
Cluster status of node 'dc001 at dc001' ...
[{nodes,[{disc,['dc001 at dc002','dc001 at dc001']}]},
{running_nodes,['dc001 at dc002','dc001 at dc001']}]
...done.
Status of node 'dc001 at dc001' ...
[{pid,26253},
{running_applications,
[{rabbitmq_management,"RabbitMQ Management Console","2.8.1"},
{rabbitmq_management_agent,"RabbitMQ Management Agent","2.8.1"},
{amqp_client,"RabbitMQ AMQP Client","2.8.1"},
{rabbit,"RabbitMQ","2.8.1"},
{mnesia,"MNESIA CXC 138 12","4.6"},
{os_mon,"CPO CXC 138 46","2.2.8"},
{sasl,"SASL CXC 138 11","2.2"},
{rabbitmq_mochiweb,"RabbitMQ Mochiweb Embedding","2.8.1"},
{webmachine,"webmachine","1.7.0-rmq2.8.1-hg"},
{mochiweb,"MochiMedia Web Server","1.3-rmq2.8.1-git"},
{inets,"INETS CXC 138 49","5.8"},
{stdlib,"ERTS CXC 138 10","1.18"},
{kernel,"ERTS CXC 138 10","2.15"}]},
{os,{unix,linux}},
{erlang_version,
"Erlang R15B (erts-5.9) [source] [64-bit] [smp:4:4]
[async-threads:30] [hipe] [kernel-poll:true]\n"},
{memory,
[{total,485192568},
{processes,303500420},
{processes_used,303400224},
{system,181692148},
{atom,711569},
{atom_used,701087},
{binary,158312632},
{code,17241165},
{ets,1651976}]},
{vm_memory_high_watermark,0.3999999999650754},
{vm_memory_limit,6871947673},
{file_descriptors,
[{total_limit,924},
{total_used,23},
{sockets_limit,829},
{sockets_used,21}]},
{processes,[{limit,1048576},{used,378}]},
{run_queue,26},
{uptime,32423}]
...done.
==> rabbitmq-cfg.dc001.config <==
[
{rabbit, [{tcp_listeners, [60001]},
{vm_memory_high_watermark, 0.4 },
{collect_statistics, fine },
{collect_statistics_interval, 5000 },
{hipe_compile, true }
]
},
{mnesia, [{dc_dump_limit, 40 },
{dump_log_write_threshold, 50000},
{no_table_loaders, 1 },
{send_compressed, 9 },
{snmp, true }
]
},
{rabbitmq_management, [{http_log_dir,
"/data/rabbitmq/dc001/rabbit-mgmt" },
{load_definitions,
"/etc/rabbitmq/rabbitmq-mgt.dc001.json"}
]
},
{rabbitmq_management_agent, [ {force_fine_statistics, true} ] },
{rabbitmq_mochiweb,
[{listeners,
[{'*', [{port, 61001}]},
{mgmt, [{port, 62001}]}
]
}]
}
].
==> rabbitmq-env.dc001.conf <==
NODENAME=dc001
BASE=/data/rabbitmq/dc001
LOG_BASE=/data/rabbitmq/dc001/log
MNESIA_BASE=/data/rabbitmq/dc001/mnesia
CONFIG_FILE=/etc/rabbitmq/rabbitmq-cfg.dc001
SERVER_START_ARGS="+K true -smp enable"
ENABLED_PLUGINS_FILE=/etc/rabbitmq/rabbitmq-plg.dc001.conf
==> rabbitmq-mgt.dc001.json <==
{
"rabbit_version":"2.8.1",
"users":
[
{"name":"guest",
"password_hash":"**secret**","tags":"administrator" },
{"name":"appadmin", "password_hash":"**secret**","tags":"app
administrator"},
{"name":"appwriter","password_hash":"**secret**","tags":"app
write-only" },
{"name":"appreader","password_hash":"**secret**","tags":"app
read-only" }
],
"vhosts":
[
{"name":"/"},
{"name":"app"}
],
"permissions":
[
{"user":"guest",
"vhost":"app","configure":".*","write":".*","read":".*"},
{"user":"guest", "vhost":"/",
"configure":".*","write":".*","read":".*"},
{"user":"appadmin",
"vhost":"app","configure":".*","write":".*","read":".*"},
{"user":"appwriter","vhost":"app","configure":"",
"write":".*","read":"" },
{"user":"appreader","vhost":"app","configure":"", "write":"",
"read":".*"}
],
"queues":
[
{"name":"app.noroute.all","vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":360000,"x-ha-policy":"all"}},
{"name":"app.q01.all",
"vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":10000,
"x-ha-policy":"all"}},
{"name":"app.q02.all",
"vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":10000,
"x-ha-policy":"all"}}
],
"exchanges":
[
{"name":"app.publish","vhost":"app","type":"topic",
"durable":true,"auto_delete":false,"internal":false,"arguments":{"alternate-exchange":"app.fanout"}},
{"name":"app.fanout",
"vhost":"app","type":"fanout","durable":true,"auto_delete":false,"internal":true,
"arguments":{} }
],
"bindings":
[
{"source":"app.fanout",
"vhost":"app","destination":"app.noroute.all","destination_type":"queue","routing_key":"",
"arguments":{}},
{"source":"app.publish","vhost":"app","destination":"app.q01.all",
"destination_type":"queue","routing_key":"q01.#","arguments":{}},
{"source":"app.publish","vhost":"app","destination":"app.q02.all",
"destination_type":"queue","routing_key":"q02.#","arguments":{}}
]
}
==> rabbitmq-plg.dc001.conf <==
[rabbitmq_management,rabbitmq_management_agent].
==> *record example* <==
Timestamp, Message_id, Delivery Mode, etc. set by the publisher.
Routing Key: q01.abc.123.1025819054.1
Timestamp: 1332936969243454
Message_id: 648926607241636355
Delivery_mode: 1
Content_encoding: en_US.UTF-8
content_type: application/octet-stream
Payload: 348 bytes
6Np6696Kw66666/To666R66S66W86pol3W66w6t66S646266666666Q666P4666Q6+66t6t6666n66/n669w666666/66rQt6766rTL06O/6626m662T7T4Qq6016W66QL6666OL66S56P6Orp3s666+6/6666666l666L0K0nQ64Q6n6s+6m66M86kT666M666321T6S6PR946Q66KO66666+s6o664K63T6W665066666n24p66+pMT66663t666n/4o606q67N6K6T7116mLS66676K66T666666O+6q8666r6666m68rQ666k668mm66p66W6l/QlmKW66666966666=
On 3/28/12 6:32 AM, Simon MacMullen wrote:
> On 28/03/12 04:20, DawgTool wrote:
>> The Flow-Control in 2.8.1 has been a huge failure for my UAT systems.
>
> Hi. Sorry to hear that. Obviously our testing hasn't shown any of the
> issues you raise. I'll try to reproduce your issues as best I can but
> the more information we have, the better. So just to be clear, you have:
>
> * Clustering
> * Non-persistent messages
> * Queues with x-message-ttl set (and a large number of messages
> actually expiring, by the sound of it?)
> * Large queues
> * Some HA queues
>
> I will try to reproduce based on the above, but anything else you can
> tell me would be a great help.
>
> Cheers, Simon
>
More information about the rabbitmq-discuss
mailing list