作者：容易 2013-08-06 14:08:34
根据第一次的测试结果，稍微做调整
测试场景二、
查询：查询每秒2000
[root@mdb-166 test]# cat mselectmongodb.py
import time,pymongo,multiprocessing,random,string
import sys,time
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
d=random.randint(start,end)
val=col.find({"user_id":d})
a=list(val)
time.sleep(0.02)
# print a
def gen_load(x,taskid):
task=SqlToMongo()
print "task %s start!" % taskid
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
插入:每秒800
[root@mdb-166 test]# cat minsertmongodb.py
import time,pymongo,multiprocessing,random,string
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
username="user"+str(i)
dic={"user_id":i,"name":username,"title":"system","mail":"yj@chinapnr.com","telphone":15718441234,"company":"chinapnr","g_roup":"kiwi","address":"hongmei1801num","ip":"192.168.168.254","mac":"of:fd:67:8c:2f:8f","os":"win7","database":"mongodb"}
col.insert(dic)
time.sleep(0.05)
def gen_load(x,taskid):
task=SqlToMongo()
print "task start!"
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
print "insert into 0K!,sleep 60 seconds!"
time.sleep(5)
print "Sub-process(es) done."
更新:每秒400个并发
[root@mdb-166 test]# cat mupdatemongodb.py
import time,pymongo,multiprocessing,random,string
class SqlToMongo:
def m_sql(self,x,y):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(start,end):
# username="user"+str(i)
id=random.randint(start,end)
# dic={"user_id":i,"name":username,"title":"system","mail":"yj@chinapnr.com","telphone":15718441234,"company":"chinapnr","g_roup":"kiwi","address":"hongmei1801num","ip":"192.168.168.254","mac":"of:fd:67:8c:2f:8f","os":"win7","database":"mongodb"}
col.update({"user_id":id},{"$set":{"database":"oracle","os":"linux"}})
# col.insert(dic)
time.sleep(0.1)
def gen_load(x,taskid):
task=SqlToMongo()
print "task start!"
task.m_sql(x,taskid)
if __name__ == "__main__":
inser_number=2500000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
删除：每秒400个
[root@mdb-166 test]# cat mdeletemongod.py2
import time,pymongo,multiprocessing,random,string
import sys,time
class SqlToMongo:
def m_sql(self,x,y,z):
server="mongodb://python:oracle@192.168.4.167:27017/syslog"
conn=pymongo.Connection(server)
db=conn.syslog
col=db.thing
start=x*y
end=start+x
for i in xrange(z):
d=random.randint(start,end)
col.remove({"user_id":d})
time.sleep(0.1)
def gen_load(x,taskid,run_count):
task=SqlToMongo()
print "task %s start!" % taskid
task.m_sql(x,taskid,run_count)
if __name__ == "__main__":
inser_number=2500000
run_count=300000
pro_pool = multiprocessing.Pool(processes=101)
print time.strftime('%Y-%m-%d:%H-%M-%S',time.localtime(time.time()))
start_time=time.time()
manager = multiprocessing.Manager()
for i in xrange(40):
taskid=i
pro_pool.apply_async(gen_load,args=(inser_number,taskid,run_count))
pro_pool.close()
pro_pool.join()
elapsed = time.time()-start_time
print elapsed
time.sleep(1)
print "Sub-process(es) done."
mongostat监控
398 1859 398 399 0 3|0 0 62.1g 125g 43.4g 0 syslog:26.8% 0 0|0 2|0 295k 789k 162 14:03:47
310 1054 348 346 0 1|0 0 62.1g 125g 43.3g 0 syslog:96.9% 0 120|0 0|0 206k 439k 162 14:03:48
470 1145 452 453 0 5|0 0 62.1g 125g 43.4g 2 syslog:98.2% 0 109|0 0|21 280k 463k 162 14:03:49
406 1248 413 410 0 1|0 0 62.1g 125g 43.3g 0 syslog:81.4% 0 121|0 0|39 260k 549k 162 14:03:52
1194 1695 1138 1180 0 5|0 1 62.1g 125g 43.4g 1 syslog:172.1% 0 45|0 0|1 663k 743k 162 14:03:53
448 1500 479 436 0 1|0 0 62.1g 125g 43.3g 1 syslog:70.9% 0 0|0 0|0 302k 633k 162 14:03:54
399 1863 399 400 0 3|0 0 62.1g 125g 43.3g 0 syslog:27.7% 0 0|0 1|0 296k 810k 162 14:03:55
400 1840 400 398 0 1|0 0 62.1g 125g 43.3g 1 syslog:29.1% 0 0|0 0|0 295k 780k 162 14:03:56
399 1824 398 400 0 3|0 0 62.1g 125g 43.3g 0 syslog:32.3% 0 1|0 3|0 293k 763k 162 14:03:57
386 1892 400 400 0 1|0 0 62.1g 125g 43.3g 2 syslog:26.1% 0 77|1 0|1 290k 780k 162 14:03:58
414 1867 402 400 0 3|0 0 62.1g 125g 43.4g 0 syslog:28.6% 0 1|0 0|1 305k 777k 162 14:03:59
157 749 188 189 0 1|0 0 62.1g 125g 43.3g 0 syslog:13.4% 0 136|0 0|24 111k 303k 162 14:04:01
628 971 664 639 0 3|0 0 62.1g 125g 43.4g 2 syslog:222.8% 0 152|0 0|3 357k 420k 162 14:04:02
901 1442 833 865 0 3|0 1 62.1g 125g 43.3g 0 syslog:94.2% 0 0|0 0|9 504k 605k 162 14:04:03
399 1538 398 399 0 1|0 0 62.1g 125g 43.3g 0 syslog:62.7% 0 2|0 0|0 277k 649k 162 14:04:04
401 1894 400 399 0 3|0 0 62.1g 125g 43.4g 1 syslog:26.5% 0 2|0 6|0 298k 795k 162 14:04:05
399 1851 399 397 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.7% 0 0|0 0|0 295k 773k 162 14:04:06
400 1846 400 400 0 3|0 0 62.1g 125g 43.4g 0 syslog:29.0% 0 0|0 0|0 295k 790k 162 14:04:07
399 1867 400 400 0 1|0 0 62.1g 125g 43.4g 1 syslog:28.2% 0 1|0 0|1 296k 819k 162 14:04:08
400 1853 400 399 0 3|0 0 62.1g 125g 43.4g 1 syslog:29.0% 0 1|0 1|0 296k 773k 162 14:04:09
400 1879 400 399 0 1|0 0 62.1g 125g 43.4g 0 syslog:27.5% 0 0|0 0|0 297k 823k 162 14:04:10
400 1848 399 396 0 3|0 0 62.1g 125g 43.3g 0 syslog:29.2% 0 0|0 0|0 295k 801k 162 14:04:11
396 1112 400 400 0 1|0 0 62.1g 125g 43.4g 1 syslog:109.0% 0 0|0 0|0 251k 495k 162 14:04:12
386 1010 400 399 0 3|0 0 62.1g 125g 43.3g 0 syslog:100.1% 0 76|0 0|6 237k 380k 162 14:04:13
152 480 174 179 0 1|0 0 62.1g 125g 43.3g 0 syslog:35.8% 0 145|0 0|15 97k 197k 162 14:04:16
1222 1961 1186 1182 0 5|0 1 62.1g 125g 43.3g 2 syslog:194.4% 0 0|0 0|0 690k 871k 162 14:04:17
398 1878 400 400 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.5% 0 0|0 0|7 295k 778k 162 14:04:18
400 1849 400 399 0 5|0 0 62.1g 125g 43.3g 1 syslog:28.7% 0 0|0 1|0 297k 763k 162 14:04:19
398 1847 398 400 0 1|0 0 62.1g 125g 43.4g 0 syslog:29.4% 0 0|0 1|0 294k 810k 162 14:04:20
400 1850 400 397 0 3|0 0 62.1g 125g 43.4g 1 syslog:28.3% 0 3|0 0|3 295k 800k 162 14:04:21
400 1854 398 399 0 1|0 0 62.1g 125g 43.4g 0 syslog:29.2% 0 0|0 0|0 296k 797k 162 14:04:22
397 1869 400 399 0 3|0 0 62.1g 125g 43.4g 1 syslog:27.3% 0 3|0 1|0 296k 735k 162 14:04:23
354 1692 392 394 0 1|0 0 62.1g 125g 43.4g 0 syslog:23.3% 0 156|0 0|4 262k 709k 162 14:04:24
285 810 272 275 0 2|0 0 62.1g 125g 43.4g 1 syslog:67.6% 0 118|0 0|42 174k 361k 162 14:04:26
467 588 480 493 0 2|0 0 62.1g 125g 43.4g 0 syslog:210.5% 0 151|0 0|9 259k 279k 162 14:04:27
1057 1283 1018 1000 0 3|0 1 62.1g 125g 43.4g 3 syslog:144.4% 0 0|0 4|0 571k 544k 162 14:04:28
396 1869 399 399 0 1|0 0 62.1g 125g 43.4g 0 syslog:29.3% 0 1|0 0|1 295k 763k 162 14:04:29
400 1827 399 399 0 3|0 0 62.1g 125g 43.4g 0 syslog:30.0% 0 0|0 0|1 294k 750k 162 14:04:30
399 1846 399 400 0 1|0 0 62.1g 125g 43.4g 1 syslog:29.3% 0 0|0 0|0 295k 758k 162 14:04:31
398 1868 399 399 0 3|0 0 62.1g 125g 43.4g 1 syslog:28.2% 0 13|0 1|0 295k 797k 162 14:04:32
399 1864 400 400 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.1% 0 8|0 1|0 296k 793k 162 14:04:33
399 1856 400 397 0 3|0 0 62.1g 125g 43.4g 1 syslog:27.3% 0 0|0 0|0 296k 805k 162 14:04:34
400 1879 400 399 0 1|0 0 62.1g 125g 43.4g 1 syslog:26.9% 0 1|0 0|0 297k 783k 162 14:04:35
380 755 397 400 0 3|0 0 62.1g 125g 43.3g 0 syslog:130.3% 0 91|1 0|1 220k 294k 162 14:04:36
501 1509 483 484 0 1|0 0 62.1g 125g 43.4g 1 syslog:77.5% 0 4|0 1|0 324k 638k 162 14:04:37
361 1016 392 394 0 3|0 0 62.1g 125g 43.4g 0 syslog:86.4% 0 137|0 0|23 223k 410k 162 14:04:38
203 410 211 208 0 1|0 0 62.1g 125g 43.4g 1 syslog:64.8% 0 157|0 0|3 121k 175k 162 14:04:40
1052 1453 1041 1036 0 3|0 1 62.1g 125g 43.3g 1 syslog:212.9% 0 50|0 0|1 581k 632k 162 14:04:41
448 1888 420 428 0 3|0 0 62.1g 125g 43.4g 0 syslog:29.5% 0 0|0 0|0 319k 814k 162 14:04:42
398 1870 400 398 0 1|0 0 62.1g 125g 43.4g 1 syslog:30.1% 0 0|0 0|1 296k 771k 162 14:04:43
397 1823 400 399 0 3|0 0 62.1g 125g 43.4g 1 syslog:29.7% 0 0|0 0|1 293k 757k 162 14:04:44
400 1881 399 399 0 1|0 0 62.1g 125g 43.4g 0 syslog:27.7% 0 0|0 0|0 297k 826k 162 14:04:45
399 1827 400 400 0 3|0 0 62.1g 125g 43.4g 0 syslog:31.2% 0 9|0 2|0 293k 751k 162 14:04:46
400 1707 400 400 0 1|0 0 62.1g 125g 43.4g 1 syslog:41.8% 0 0|0 1|0 288k 731k 162 14:04:47
360 897 380 382 0 3|0 0 62.1g 125g 43.4g 1 syslog:110.4% 0 112|0 0|48 210k 349k 162 14:04:48
465 969 441 441 0 1|0 0 62.1g 125g 43.5g 0 syslog:146.8% 0 13|0 0|45 276k 416k 162 14:04:51
640 853 700 684 0 5|0 1 62.1g 125g 43.4g 0 syslog:165.7% 0 134|0 0|1 358k 386k 162 14:04:52
756 1705 696 711 0 3|0 0 62.1g 125g 43.4g 2 syslog:59.1% 0 0|0 0|0 452k 717k 162 14:04:53
400 1860 400 400 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.0% 0 3|0 0|1 296k 784k 162 14:04:54
398 1880 399 396 0 3|0 0 62.1g 125g 43.4g 0 syslog:29.1% 0 0|0 0|1 297k 783k 162 14:04:55
399 1858 398 400 0 1|0 0 62.1g 125g 43.4g 2 syslog:27.8% 0 1|0 3|0 295k 806k 162 14:04:56
399 1866 400 400 0 3|0 0 62.1g 125g 43.4g 0 syslog:27.9% 0 0|0 0|0 297k 858k 162 14:04:57
399 1799 400 399 0 1|0 0 62.1g 125g 43.4g 0 syslog:36.9% 0 0|0 4|0 292k 760k 162 14:04:58
399 1026 399 400 0 3|0 0 62.1g 125g 43.5g 1 syslog:112.0% 0 0|0 0|0 247k 438k 162 14:04:59
145 388 162 165 0 1|0 0 62.1g 125g 43.4g 0 .:18.6% 0 94|0 0|66 77k 150k 162 14:05:01
927 1230 910 903 0 3|0 0 62.1g 125g 43.5g 2 syslog:222.3% 0 63|0 0|1 514k 554k 162 14:05:02
443 1632 442 449 0 3|0 0 62.1g 125g 43.4g 0 syslog:54.3% 0 0|0 0|0 307k 694k 162 14:05:03
107 339 140 142 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.7% 0 139|0 0|21 63k 131k 162 14:05:04
787 1721 754 754 0 3|0 1 62.1g 125g 43.4g 1 syslog:197.6% 0 0|0 0|0 476k 724k 162 14:05:05
400 1888 400 398 0 1|0 0 62.1g 125g 43.4g 0 syslog:28.0% 0 0|0 0|0 298k 780k 162 14:05:06
400 1849 400 398 0 3|0 0 62.1g 125g 43.4g 1 syslog:29.5% 0 0|0 0|0 296k 787k 162 14:05:07
iostat监控
从监控结果看，系统基本没有压力，当mongodb进行数据同步到硬盘的时候会瞬间的IO繁忙。
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1457.00 0.00 4323.25 0.00 25193.00 11.65 68.53 16.02 0.21 90.97
avg-cpu: %user %nice %system %iowait %steal %idle
5.92 0.00 3.00 0.00 0.00 91.08
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 98.75 0.00 94.75 0.00 5658.00 119.43 0.02 0.19 0.05 0.50
avg-cpu: %user %nice %system %iowait %steal %idle
4.71 0.00 3.29 4.80 0.00 87.20
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1610.25 0.00 3532.25 0.00 23960.00 13.57 70.84 20.05 0.16 57.05
avg-cpu: %user %nice %system %iowait %steal %idle
3.98 0.00 2.83 7.36 0.00 85.83
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1027.75 0.00 3220.50 0.00 20407.00 12.67 83.80 26.03 0.24 78.65
avg-cpu: %user %nice %system %iowait %steal %idle
4.09 0.00 2.57 3.24 0.00 90.10
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 728.25 0.00 2058.00 0.00 14145.00 13.75 36.15 17.45 0.24 50.40
avg-cpu: %user %nice %system %iowait %steal %idle
5.48 0.00 3.18 4.42 0.00 86.92
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 1654.25 0.00 3290.25 0.00 23816.00 14.48 21.52 6.59 0.18 60.52
avg-cpu: %user %nice %system %iowait %steal %idle
5.06 0.00 2.93 3.76 0.00 88.25
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 235.75 0.00 900.75 0.00 8116.00 18.02 9.19 10.29 0.48 42.80
avg-cpu: %user %nice %system %iowait %steal %idle
5.42 0.00 2.84 0.00 0.00 91.74
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 67.50 0.00 75.00 0.00 4113.00 109.68 0.01 0.16 0.07 0.53
avg-cpu: %user %nice %system %iowait %steal %idle
3.90 0.00 2.60 8.14 0.00 85.35
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 2100.00 0.00 3208.75 0.00 24159.00 15.06 124.59 38.49 0.18 57.25
avg-cpu: %user %nice %system %iowait %steal %idle
3.58 0.00 2.70 7.48 0.00 86.24
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 844.75 0.00 3593.50 0.00 21423.00 11.92 77.17 21.78 0.25 88.05
[mdb@mdb-167 ~]$ tail -f ioidus_new2.txt
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 579.50 0.00 1533.75 0.00 12305.00 16.05 35.22 23.16 0.23 35.38
avg-cpu: %user %nice %system %iowait %steal %idle
5.48 0.00 2.76 0.00 0.00 91.76
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 71.00 0.00 43.75 0.00 4121.00 188.39 0.01 0.14 0.09 0.40
One Response