[rabbitmq-discuss] Per-Connection Flow Control - RMQ 2.8.1

DawgTool dawgtool at aol.com
Wed Mar 28 13:43:51 BST 2012


Hi Simon,

Thanks for looking into this.  This is a really large project which we 
will be scaling to handle ~300k msg/sec, right now 2.7.1 is running at 
90k msg/sec.

Here is my Configuration (UAT): (2 servers)
x4: Intel(R) Xeon(R) CPU X5650  @ 2.67GHz
x1: MemTotal:  16777216 kB

Number of publishers: 20
Testing with TTL only, No consumers (testing for durability during 
maintenance windows on consumers)
Publishing Rate: 100 - 350 messages/sec per publisher (ie: 100 x 20 = 
2000) to app.publish exchange
x-message-ttl at 10000 should cause flow control to block every second 
to 10 seconds
x-message-ttl at 60000 should cause flow control to block every second 
to 10 seconds, as queue reaches ~1gB expiring will lock the queue and 
everything will block for several minutes.


==> *rabbitmq status* <==
Linux 2.6.18-238.el5xen #1 SMP Thu Jan 13 16:41:45 EST 2011 x86_64 
x86_64 x86_64 GNU/Linux (CentOS 5)

Cluster status of node 'dc001 at dc001' ...
[{nodes,[{disc,['dc001 at dc002','dc001 at dc001']}]},
  {running_nodes,['dc001 at dc002','dc001 at dc001']}]
...done.

Status of node 'dc001 at dc001' ...
[{pid,26253},
  {running_applications,
      [{rabbitmq_management,"RabbitMQ Management Console","2.8.1"},
       {rabbitmq_management_agent,"RabbitMQ Management Agent","2.8.1"},
       {amqp_client,"RabbitMQ AMQP Client","2.8.1"},
       {rabbit,"RabbitMQ","2.8.1"},
       {mnesia,"MNESIA  CXC 138 12","4.6"},
       {os_mon,"CPO  CXC 138 46","2.2.8"},
       {sasl,"SASL  CXC 138 11","2.2"},
       {rabbitmq_mochiweb,"RabbitMQ Mochiweb Embedding","2.8.1"},
       {webmachine,"webmachine","1.7.0-rmq2.8.1-hg"},
       {mochiweb,"MochiMedia Web Server","1.3-rmq2.8.1-git"},
       {inets,"INETS  CXC 138 49","5.8"},
       {stdlib,"ERTS  CXC 138 10","1.18"},
       {kernel,"ERTS  CXC 138 10","2.15"}]},
  {os,{unix,linux}},
  {erlang_version,
      "Erlang R15B (erts-5.9) [source] [64-bit] [smp:4:4] 
[async-threads:30] [hipe] [kernel-poll:true]\n"},
  {memory,
      [{total,485192568},
       {processes,303500420},
       {processes_used,303400224},
       {system,181692148},
       {atom,711569},
       {atom_used,701087},
       {binary,158312632},
       {code,17241165},
       {ets,1651976}]},
  {vm_memory_high_watermark,0.3999999999650754},
  {vm_memory_limit,6871947673},
  {file_descriptors,
      [{total_limit,924},
       {total_used,23},
       {sockets_limit,829},
       {sockets_used,21}]},
  {processes,[{limit,1048576},{used,378}]},
  {run_queue,26},
  {uptime,32423}]
...done.


==> rabbitmq-cfg.dc001.config <==
[
   {rabbit,                    [{tcp_listeners,               [60001]},
                                {vm_memory_high_watermark,    0.4    },
                                {collect_statistics,          fine   },
                                {collect_statistics_interval, 5000   },
                                {hipe_compile,                true   }
                               ]
   },
   {mnesia,                    [{dc_dump_limit,            40   },
                                {dump_log_write_threshold, 50000},
                                {no_table_loaders,         1    },
                                {send_compressed,          9    },
                                {snmp,                     true }
                               ]
   },
   {rabbitmq_management,       [{http_log_dir,     
"/data/rabbitmq/dc001/rabbit-mgmt" },
                                {load_definitions, 
"/etc/rabbitmq/rabbitmq-mgt.dc001.json"}
                               ]
   },
   {rabbitmq_management_agent, [ {force_fine_statistics, true} ] },
   {rabbitmq_mochiweb,
                               [{listeners,
                                            [{'*',  [{port, 61001}]},
                                             {mgmt, [{port, 62001}]}
                                            ]
                               }]
   }
].

==> rabbitmq-env.dc001.conf <==
NODENAME=dc001
BASE=/data/rabbitmq/dc001
LOG_BASE=/data/rabbitmq/dc001/log
MNESIA_BASE=/data/rabbitmq/dc001/mnesia
CONFIG_FILE=/etc/rabbitmq/rabbitmq-cfg.dc001
SERVER_START_ARGS="+K true -smp enable"
ENABLED_PLUGINS_FILE=/etc/rabbitmq/rabbitmq-plg.dc001.conf

==> rabbitmq-mgt.dc001.json <==
{
"rabbit_version":"2.8.1",
"users":
   [
     {"name":"guest",     
"password_hash":"**secret**","tags":"administrator"     },
     {"name":"appadmin", "password_hash":"**secret**","tags":"app 
administrator"},
     {"name":"appwriter","password_hash":"**secret**","tags":"app 
write-only"   },
     {"name":"appreader","password_hash":"**secret**","tags":"app 
read-only"    }
    ],
"vhosts":
   [
     {"name":"/"},
     {"name":"app"}
   ],
"permissions":
   [
     {"user":"guest",     
"vhost":"app","configure":".*","write":".*","read":".*"},
     {"user":"guest",     "vhost":"/",   
"configure":".*","write":".*","read":".*"},
     {"user":"appadmin", 
"vhost":"app","configure":".*","write":".*","read":".*"},
     {"user":"appwriter","vhost":"app","configure":"",  
"write":".*","read":""  },
     {"user":"appreader","vhost":"app","configure":"",  "write":"",  
"read":".*"}
   ],
"queues":
   [
     
{"name":"app.noroute.all","vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":360000,"x-ha-policy":"all"}},
     {"name":"app.q01.all",    
"vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":10000, 
"x-ha-policy":"all"}},
     {"name":"app.q02.all",    
"vhost":"app","durable":true,"auto_delete":false,"arguments":{"x-message-ttl":10000, 
"x-ha-policy":"all"}}
   ],
"exchanges":
   [
     {"name":"app.publish","vhost":"app","type":"topic", 
"durable":true,"auto_delete":false,"internal":false,"arguments":{"alternate-exchange":"app.fanout"}},
     {"name":"app.fanout", 
"vhost":"app","type":"fanout","durable":true,"auto_delete":false,"internal":true, 
"arguments":{}                                  }
   ],
"bindings":
   [
     {"source":"app.fanout", 
"vhost":"app","destination":"app.noroute.all","destination_type":"queue","routing_key":"",      
"arguments":{}},
     
{"source":"app.publish","vhost":"app","destination":"app.q01.all",    
"destination_type":"queue","routing_key":"q01.#","arguments":{}},
     
{"source":"app.publish","vhost":"app","destination":"app.q02.all",    
"destination_type":"queue","routing_key":"q02.#","arguments":{}}
   ]
}


==> rabbitmq-plg.dc001.conf <==
[rabbitmq_management,rabbitmq_management_agent].


==> *record example* <==
Timestamp, Message_id, Delivery Mode, etc. set by the publisher.

Routing Key:      q01.abc.123.1025819054.1
Timestamp:        1332936969243454
Message_id:       648926607241636355
Delivery_mode:    1
Content_encoding: en_US.UTF-8
content_type:     application/octet-stream
Payload:          348 bytes
6Np6696Kw66666/To666R66S66W86pol3W66w6t66S646266666666Q666P4666Q6+66t6t6666n66/n669w666666/66rQt6766rTL06O/6626m662T7T4Qq6016W66QL6666OL66S56P6Orp3s666+6/6666666l666L0K0nQ64Q6n6s+6m66M86kT666M666321T6S6PR946Q66KO66666+s6o664K63T6W665066666n24p66+pMT66663t666n/4o606q67N6K6T7116mLS66676K66T666666O+6q8666r6666m68rQ666k668mm66p66W6l/QlmKW66666966666=




On 3/28/12 6:32 AM, Simon MacMullen wrote:
> On 28/03/12 04:20, DawgTool wrote:
>> The Flow-Control in 2.8.1 has been a huge failure for my UAT systems.
>
> Hi. Sorry to hear that. Obviously our testing hasn't shown any of the 
> issues you raise. I'll try to reproduce your issues as best I can but 
> the more information we have, the better. So just to be clear, you have:
>
> * Clustering
> * Non-persistent messages
> * Queues with x-message-ttl set (and a large number of messages 
> actually expiring, by the sound of it?)
> * Large queues
> * Some HA queues
>
> I will try to reproduce based on the above, but anything else you can 
> tell me would be a great help.
>
> Cheers, Simon
>



More information about the rabbitmq-discuss mailing list