作者：容易 2013-08-06 13:38:34
测试场景一、时间 1 小时，前提数据基本初始到内存中。
查询：查询每秒8000
[root@mdb-166 test]# cat mselectmongodb.py
import time,pymongo,multiprocessing,random,string
import sys,time
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
d=random.randint(start,end)
val=col.find({"user_id":d})
a=list(val)
time.sleep(0.005)
# print a
def gen_load(x,taskid):
task=SqlToMongo()
print "task %s start!" % taskid
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
插入:每秒2000
[root@mdb-166 test]# cat minsertmongodb.py
import time,pymongo,multiprocessing,random,string
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
username="user"+str(i)
dic={"user_id":i,"name":username,"title":"system","mail":"yj@chinapnr.com","telphone":15718441234,"company":"chinapnr","g_roup":"kiwi","address":"hongmei1801num","ip":"192.168.168.254","mac":"of:fd:67:8c:2f:8f","os":"win7","database":"mongodb"}
col.insert(dic)
time.sleep(0.02)
def gen_load(x,taskid):
task=SqlToMongo()
print "task start!"
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
print "insert into 0K!,sleep 60 seconds!"
time.sleep(5)
print "Sub-process(es) done."
更新:每秒800个并发
[root@mdb-166 test]# cat mupdatemongodb.py
import time,pymongo,multiprocessing,random,string
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
# username="user"+str(i)
id=random.randint(start,end)
# dic={"user_id":i,"name":username,"title":"system","mail":"yj@chinapnr.com","telphone":15718441234,"company":"chinapnr","g_roup":"kiwi","address":"hongmei1801num","ip":"192.168.168.254","mac":"of:fd:67:8c:2f:8f","os":"win7","database":"mongodb"}
col.update({"user_id":id},{"$set":{"database":"oracle","os":"linux"}})
# col.insert(dic)
time.sleep(0.05)
def gen_load(x,taskid):
task=SqlToMongo()
print "task start!"
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
删除：每秒400个
[root@mdb-166 test]# cat mdeletemongod.py2
import time,pymongo,multiprocessing,random,string
import sys,time
class SqlToMongo:
def m_sql(self,x,y,z):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(z):
d=random.randint(start,end)
col.remove({"user_id":d})
time.sleep(0.1)
def gen_load(x,taskid,run_count):
task=SqlToMongo()
print "task %s start!" % taskid
task.m_sql(x,taskid,run_count)
if __name__ == "__main__":
inser_number=2500000
run_count=300000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid,run_count))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
mongostat 监控结果，从结果显示，已经产生了严重的阻塞，虽然依然能够处理，但是已经无法及时响应用户请求了。
locked db qr|qw ar|aw
db的锁以及读写队列都比较大了。
544 683 302 101 0 3|0 0 58.1g 117g 52.9g 1 syslog:168.6% 0 364|0 1|0 242k 279k 402 09:04:18
2029 2846 1224 386 0 1|0 0 58.1g 117g 52.9g 3 syslog:65.8% 0 369|0 0|0 942k 1m 402 09:04:19
2058 2895 1267 433 0 3|0 0 58.1g 117g 52.9g 2 syslog:61.1% 0 368|0 1|0 958k 1m 402 09:04:20
2029 2993 1268 426 0 1|0 0 58.1g 117g 52.9g 3 syslog:61.3% 0 397|1 0|1 953k 1m 402 09:04:21
2206 2965 1355 441 0 3|0 0 58.1g 117g 53g 2 syslog:62.6% 0 365|0 0|1 1m 1m 402 09:04:22
2016 2769 1205 426 0 1|0 0 58.1g 117g 53g 2 syslog:64.1% 0 367|0 2|0 931k 1m 402 09:04:23
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
739 1025 355 112 0 2|0 0 58.1g 117g 53g 1 syslog:70.0% 0 235|0 0|159 300k 407k 402 09:04:25
1706 2508 1098 337 0 2|0 0 58.1g 117g 52.9g 2 syslog:179.2% 0 367|0 0|1 834k 1m 402 09:04:26
2150 2882 1272 436 0 3|0 0 58.1g 117g 52.9g 3 syslog:63.6% 0 366|0 0|1 986k 1m 402 09:04:27
1985 2728 1150 377 0 1|0 0 58.1g 117g 52.9g 3 syslog:61.9% 0 249|0 0|117 887k 1m 402 09:04:28
1565 2107 965 312 0 2|0 0 58.1g 117g 53g 5 syslog:69.9% 0 265|0 0|13 718k 896k 402 09:04:30
812 1122 534 166 0 2|0 0 58.1g 117g 53g 3 syslog:195.3% 0 359|0 0|1 407k 482k 402 09:04:31
227 314 157 30 0 2|0 0 58.1g 117g 52.9g 0 syslog:4.9% 0 399|1 0|1 103k 113k 402 09:04:32
1186 2261 689 225 0 2|0 0 58.1g 117g 52.9g 3 syslog:147.3% 0 277|0 0|105 559k 999k 402 09:04:34
787 1255 513 148 0 2|0 0 58.1g 117g 53g 2 syslog:83.8% 0 117|0 0|11 350k 531k 402 09:04:35
1170 1803 855 269 0 2|0 0 58.1g 117g 52.9g 1 syslog:263.1% 0 378|0 0|21 615k 752k 402 09:04:36
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
2290 3130 1445 438 0 5|0 1 58.1g 117g 52.9g 3 syslog:84.6% 0 369|0 0|0 1m 1m 402 09:04:37
2085 2965 1258 424 0 1|0 0 58.1g 117g 53g 2 syslog:64.8% 0 360|0 7|0 969k 1m 402 09:04:38
2175 2943 1279 424 0 3|0 0 58.1g 117g 53g 2 syslog:62.1% 0 359|0 0|1 998k 1m 402 09:04:39
2102 2974 1305 436 0 1|0 0 58.1g 117g 53g 2 syslog:61.6% 0 363|0 0|1 980k 1m 402 09:04:40
2195 2859 1288 416 0 3|0 0 58.1g 117g 52.9g 4 syslog:64.3% 0 360|0 2|0 999k 1m 402 09:04:41
2069 2856 1269 401 0 1|0 0 58.1g 117g 52.9g 2 syslog:64.4% 0 367|0 0|1 957k 1m 402 09:04:42
1965 2621 1180 380 0 3|0 0 58.1g 117g 53g 2 syslog:72.8% 0 393|0 0|7 896k 1m 402 09:04:44
2590 3456 1511 509 0 1|0 0 58.1g 117g 53g 3 syslog:78.8% 0 369|0 7|0 1m 1m 402 09:04:45
2009 2857 1178 402 0 3|0 0 58.1g 117g 53g 3 syslog:60.2% 0 359|0 1|0 930k 1m 402 09:04:46
2092 2871 1231 408 0 1|0 0 58.1g 117g 53g 3 syslog:59.4% 0 374|0 0|10 958k 1m 402 09:04:47
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
2125 2882 1253 408 0 3|0 0 58.1g 117g 52.9g 2 syslog:65.7% 0 374|0 2|0 978k 1m 402 09:04:48
1788 2419 1054 359 0 1|0 0 58.1g 117g 52.9g 4 syslog:49.8% 0 399|1 0|1 819k 999k 402 09:04:50
2552 3541 1531 507 0 3|0 0 58.1g 117g 53g 2 syslog:87.1% 0 360|0 0|1 1m 1m 402 09:04:51
2114 2994 1273 404 0 3|0 0 58.1g 117g 53g 2 syslog:60.0% 0 376|0 4|0 979k 1m 402 09:04:52
1888 2733 1210 376 0 1|0 0 58.1g 117g 53g 3 syslog:68.3% 0 362|0 0|1 888k 1m 402 09:04:53
2119 2921 1251 432 0 3|0 0 58.1g 117g 53g 3 syslog:67.3% 0 366|0 0|1 976k 1m 402 09:04:54
2021 2818 1220 442 0 1|0 0 58.1g 117g 52.9g 2 syslog:66.8% 0 363|0 0|0 938k 1m 402 09:04:55
2144 3030 1292 432 0 3|0 0 58.1g 117g 52.9g 3 syslog:58.1% 0 399|1 0|1 993k 1m 402 09:04:56
2242 3000 1332 421 0 1|0 0 58.1g 117g 53g 3 syslog:60.6% 0 369|0 0|1 1m 1m 402 09:04:57
2024 2794 1149 405 0 3|0 0 58.1g 117g 53g 3 syslog:59.7% 0 277|0 0|123 907k 1m 402 09:04:58
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
1849 2699 1157 366 0 1|0 0 58.1g 117g 52.9g 6 syslog:103.7% 0 375|0 0|25 875k 1m 402 09:04:59
520 705 334 106 0 3|0 0 58.1g 117g 53g 1 syslog:46.4% 0 334|0 0|48 242k 301k 402 09:05:01
902 1201 551 178 0 3|0 0 58.1g 117g 53g 2 syslog:306.5% 0 360|0 0|1 419k 484k 402 09:05:02
530 759 311 107 0 1|0 0 58.1g 117g 52.9g 2 syslog:127.9% 0 359|0 0|0 253k 305k 402 09:05:03
658 1031 364 127 0 2|0 0 58.1g 117g 52.9g 2 syslog:108.3% 0 305|0 0|77 285k 420k 402 09:05:05
1328 1920 833 264 0 2|0 0 58.1g 117g 53g 1 syslog:182.6% 0 367|0 0|0 637k 800k 402 09:05:06
975 1472 592 192 0 3|0 0 58.1g 117g 52.9g 1 syslog:119.0% 0 271|0 0|123 440k 571k 402 09:05:07
733 1114 472 149 0 1|0 0 58.1g 117g 53g 3 syslog:87.8% 0 317|0 0|82 355k 458k 402 09:05:09
936 1565 644 180 0 5|0 0 58.1g 117g 53g 1 syslog:211.1% 0 350|0 0|49 462k 666k 402 09:05:10
1504 2264 941 294 0 3|0 1 58.1g 117g 53g 4 syslog:122.1% 0 363|0 4|0 722k 961k 402 09:05:11
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
2043 2888 1232 383 0 1|0 0 58.1g 117g 53g 2 syslog:62.2% 0 360|0 0|1 947k 1m 402 09:05:12
2060 2886 1264 400 0 3|0 0 58.1g 117g 52.9g 2 syslog:66.6% 0 370|0 1|0 956k 1m 402 09:05:13
2224 3089 1338 430 0 1|0 0 58.1g 117g 53g 2 syslog:58.4% 0 370|0 1|0 1m 1m 402 09:05:14
2052 2874 1206 390 0 3|0 0 58.1g 117g 53g 3 syslog:66.6% 0 366|0 0|1 947k 1m 402 09:05:15
1849 2535 1105 383 0 1|0 0 58.1g 117g 53g 3 syslog:74.2% 0 361|0 0|1 838k 1m 402 09:05:16
1723 2630 1069 381 0 3|0 0 58.1g 117g 52.9g 3 syslog:78.8% 0 361|0 0|1 822k 1m 402 09:05:17
1932 2906 1205 403 0 1|0 0 58.1g 117g 52.9g 0 syslog:61.3% 0 368|0 0|1 920k 1m 402 09:05:18
2093 2872 1269 420 0 3|0 0 58.1g 117g 53g 3 syslog:61.5% 0 364|0 0|1 967k 1m 402 09:05:19
2167 2972 1300 415 0 1|0 0 58.1g 117g 53g 2 syslog:61.5% 0 360|0 4|1 998k 1m 402 09:05:20
2085 2861 1170 401 0 3|0 0 58.1g 117g 53g 2 syslog:67.3% 0 332|0 0|44 944k 1m 402 09:05:21
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
2008 2846 1215 401 0 1|0 0 58.1g 117g 53g 3 syslog:62.7% 0 361|0 0|0 941k 1m 402 09:05:22
2106 2934 1267 422 0 3|0 0 58.1g 117g 52.9g 3 syslog:60.4% 0 361|0 2|0 975k 1m 402 09:05:23
2290 2977 1356 454 0 1|0 0 58.1g 117g 53g 2 syslog:61.9% 0 369|0 5|0 1m 1m 402 09:05:24
2072 2899 1267 406 0 3|0 0 58.1g 117g 53g 2 syslog:62.1% 0 362|0 0|1 962k 1m 402 09:05:25
2029 2806 1211 419 0 1|0 0 58.1g 117g 53g 2 syslog:60.3% 0 365|0 0|22 931k 1m 402 09:05:26
2035 2811 1222 423 0 3|0 0 58.1g 117g 53g 4 syslog:67.4% 0 326|0 0|41 940k 1m 402 09:05:27
1612 2206 994 335 0 1|0 0 58.1g 117g 52.9g 2 syslog:82.9% 0 399|1 0|1 749k 920k 402 09:05:28
755 1147 448 145 0 3|0 0 58.1g 117g 53g 2 syslog:62.6% 0 396|0 0|4 345k 499k 402 09:05:30
223 384 139 42 0 3|0 0 58.1g 117g 53g 2 syslog:141.2% 0 347|0 0|52 109k 169k 402 09:05:32
1234 1828 775 247 0 3|0 0 58.1g 117g 53g 3 syslog:245.8% 0 280|0 0|116 571k 752k 402 09:05:35
insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time
iostat 监控，由于有大量的插入，更新与删除操作，当mongodb执行数据同步的时候还是会造成磁盘IO的繁忙，从监控显示系统没有什么负载。
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 59.00 0.00 53.00 0.00 7172.00 270.64 0.01 0.25 0.11 0.60
avg-cpu: %user %nice %system %iowait %steal %idle
9.02 0.00 8.08 2.58 0.00 80.33
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1902.00 0.00 4688.00 0.00 31428.00 13.41 3.72 0.76 0.09 40.60
avg-cpu: %user %nice %system %iowait %steal %idle
8.48 0.00 6.70 3.39 0.00 81.42
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 37.00 0.00 32.00 0.00 3840.00 240.00 0.41 17.66 12.62 40.40
avg-cpu: %user %nice %system %iowait %steal %idle
13.14 0.00 10.29 0.09 0.00 76.49
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 61.00 0.00 53.00 0.00 7220.00 272.45 0.02 0.32 0.15 0.80
avg-cpu: %user %nice %system %iowait %steal %idle
13.24 0.00 10.37 0.00 0.00 76.39
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 58.00 0.00 54.00 0.00 7164.00 265.33 0.01 0.19 0.09 0.50
avg-cpu: %user %nice %system %iowait %steal %idle
14.32 0.00 10.81 0.00 0.00 74.87
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 59.00 0.00 54.00 0.00 7440.00 275.56 0.01 0.17 0.09 0.50
avg-cpu: %user %nice %system %iowait %steal %idle
13.55 0.00 11.66 2.23 0.00 72.56
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 4317.00 0.00 5906.00 0.00 46568.00 15.77 42.68 6.75 0.06 37.00
avg-cpu: %user %nice %system %iowait %steal %idle
13.72 0.00 10.12 2.74 0.00 73.41
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1128.00 1.00 1194.00 8.00 17236.00 28.86 42.72 38.13 0.28 33.50
avg-cpu: %user %nice %system %iowait %steal %idle
14.33 0.00 11.35 0.00 0.00 74.32
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 58.00 0.00 55.00 0.00 7400.00 269.09 0.01 0.25 0.11 0.60
avg-cpu: %user %nice %system %iowait %steal %idle
14.62 0.00 10.49 0.00 0.00 74.89
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 59.00 0.00 54.00 0.00 7116.00 263.56 0.02 0.31 0.11 0.60
avg-cpu: %user %nice %system %iowait %steal %idle
14.02 0.00 10.85 0.09 0.00 75.04
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 58.00 0.00 52.00 0.00 7080.00 272.31 0.01 0.23 0.13 0.70
avg-cpu: %user %nice %system %iowait %steal %idle
13.21 0.00 10.28 0.00 0.00 76.51
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 59.00 0.00 52.00 0.00 6860.00 263.85 0.01 0.27 0.10 0.50
avg-cpu: %user %nice %system %iowait %steal %idle
11.91 0.00 9.75 5.61 0.00 72.74
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1065.00 0.00 6800.00 0.00 37088.00 10.91 98.49 14.15 0.11 73.10
avg-cpu: %user %nice %system %iowait %steal %idle
2.79 0.00 2.54 10.14 0.00 84.53
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 145.00 0.00 1260.00 0.00 7024.00 11.15 68.28 55.57 0.79 100.10
One Response