Revision 4906
Added by daigle over 15 years ago
| test/stress-test/insert-load-test.py | ||
|---|---|---|
| 11 | 11 |
interval = int(sys.argv[3]) |
| 12 | 12 |
host = sys.argv[4] |
| 13 | 13 |
|
| 14 |
# set debug=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug. |
|
| 15 |
debug=0
|
|
| 14 |
# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
|
|
| 15 |
debug_level=2
|
|
| 16 | 16 |
|
| 17 | 17 |
run_time = str(time()).replace('.','')
|
| 18 | 18 |
|
| 19 |
output_file = "./insert-" + letter + ".out" |
|
| 20 |
output_file = open(output_file, 'w') |
|
| 19 |
output_file_name = "./insert-" + letter + ".out" |
|
| 20 |
output_file = open(output_file_name, 'w') |
|
| 21 |
output_file.close() |
|
| 21 | 22 |
|
| 22 | 23 |
def debug (message): |
| 24 |
output_file = open(output_file_name, 'a') |
|
| 23 | 25 |
output_file.write(message + '\n') |
| 26 |
output_file.close() |
|
| 24 | 27 |
|
| 25 | 28 |
def debug1 (message): |
| 26 |
if debug > 0: |
|
| 29 |
if debug_level > 0: |
|
| 30 |
output_file = open(output_file_name, 'a') |
|
| 27 | 31 |
output_file.write(message + '\n') |
| 32 |
output_file.close() |
|
| 28 | 33 |
|
| 29 | 34 |
def debug2 (message): |
| 30 |
if debug > 1: |
|
| 35 |
if debug_level > 1: |
|
| 36 |
output_file = open(output_file_name, 'a') |
|
| 31 | 37 |
output_file.write(message + '\n') |
| 38 |
output_file.close() |
|
| 32 | 39 |
|
| 33 | 40 |
def debug3 (message): |
| 34 |
if debug > 2: |
|
| 41 |
if debug_level > 2: |
|
| 42 |
output_file = open(output_file_name, 'a') |
|
| 35 | 43 |
output_file.write(message + '\n') |
| 44 |
output_file.close() |
|
| 36 | 45 |
|
| 37 |
dictionary_file = open('./dictionary.txt', 'r')
|
|
| 46 |
dictionary_file_name = './dictionary-' + letter + '.txt' |
|
| 47 |
dictionary_file = open(dictionary_file_name, 'r') |
|
| 38 | 48 |
|
| 39 | 49 |
iter_count = 0 |
| 40 | 50 |
|
| 41 | 51 |
debug("************************************************** ")
|
| 42 |
debug("Starting squery-load-test for ")
|
|
| 52 |
debug("Starting insert-load-test for ")
|
|
| 43 | 53 |
debug(" letter: $letter ")
|
| 44 | 54 |
debug(" iterations: $iterations ")
|
| 45 | 55 |
debug(" interval: $interval ")
|
| 56 |
debug(" host: $host ")
|
|
| 46 | 57 |
debug("************************************************** ")
|
| 47 | 58 |
|
| 48 | 59 |
# Initialize a metacat client connection and log in as test user |
| ... | ... | |
| 62 | 73 |
insert_template_file.close |
| 63 | 74 |
|
| 64 | 75 |
for line in dictionary_file: |
| 65 |
if line.lower().startswith(letter.lower()): |
|
| 66 |
word = line.strip() |
|
| 76 |
word = line.strip() |
|
| 67 | 77 |
|
| 68 |
debug3("word: " + word)
|
|
| 69 |
debug3("run time: " + run_time)
|
|
| 70 |
debug3("iter count: " + str(iter_count))
|
|
| 71 |
docid = word + run_time + '.' + str(iter_count) + '.1'
|
|
| 72 |
doc = insert_template.replace("@!docid!@", docid)
|
|
| 73 |
t1 = time()
|
|
| 78 |
debug3("word: " + word)
|
|
| 79 |
debug3("run time: " + run_time)
|
|
| 80 |
debug3("iter count: " + str(iter_count))
|
|
| 81 |
docid = word + run_time + '.' + str(iter_count) + '.1' |
|
| 82 |
doc = insert_template.replace("@!docid!@", docid)
|
|
| 83 |
t1 = time() |
|
| 74 | 84 |
|
| 75 |
debug("[%s] -- Processing Insert" % (docid))
|
|
| 76 |
response = mc.insert(docid, doc)
|
|
| 77 |
t2 = time()
|
|
| 85 |
debug("[%s] -- Processing Insert" % (docid))
|
|
| 86 |
response = mc.insert(docid, doc) |
|
| 87 |
t2 = time() |
|
| 78 | 88 |
|
| 79 |
if (response.lower().find('<error>') == -1):
|
|
| 80 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
|
|
| 81 |
else:
|
|
| 82 |
debug("[%s] -- ERROR: %s" % (docid, response))
|
|
| 89 |
if (response.lower().find('<error>') == -1):
|
|
| 90 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
|
|
| 91 |
else: |
|
| 92 |
debug("[%s] -- ERROR: %s" % (docid, response))
|
|
| 83 | 93 |
|
| 84 |
iter_count = iter_count + 1
|
|
| 85 |
debug3("iter_count: %d, iterations: %d" % (iter_count, iterations))
|
|
| 86 |
if iter_count >= iterations:
|
|
| 87 |
sys.exit()
|
|
| 94 |
iter_count = iter_count + 1 |
|
| 95 |
debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
|
|
| 96 |
if iter_count >= iterations: |
|
| 97 |
sys.exit() |
|
| 88 | 98 |
|
| 89 |
sleep(interval)
|
|
| 99 |
sleep(interval) |
|
| 90 | 100 |
|
| 91 | 101 |
dictionary_file.close() |
| 92 | 102 |
output_file.close() |
| test/stress-test/metacat.py | ||
|---|---|---|
| 71 | 71 |
return False |
| 72 | 72 |
|
| 73 | 73 |
def read(self, docid, qformat="xml"): |
| 74 |
postdata = { 'action' : 'read',
|
|
| 74 |
postdata = { 'action' : 'read',
|
|
| 75 | 75 |
'qformat' : qformat, |
| 76 | 76 |
'docid' : docid } |
| 77 |
response = self.postRequest(postdata) |
|
| 78 |
# if error node returned |
|
| 79 |
if response.find("<error>") != -1:
|
|
| 80 |
return False |
|
| 81 |
else: |
|
| 82 |
return response |
|
| 77 |
response = self.postRequest(postdata) |
|
| 78 |
# if error node returned |
|
| 79 |
return response |
|
| 83 | 80 |
|
| 84 | 81 |
|
| 85 | 82 |
def insert(self, docid, doctext): |
| test/stress-test/squery-load-test.py | ||
|---|---|---|
| 11 | 11 |
interval = int(sys.argv[3]) |
| 12 | 12 |
host = sys.argv[4] |
| 13 | 13 |
|
| 14 |
# set debug=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug. |
|
| 15 |
debug=0
|
|
| 14 |
# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
|
|
| 15 |
debug_level=2
|
|
| 16 | 16 |
|
| 17 |
output_file = "./squery-" + letter + ".out" |
|
| 18 |
output_file = open(output_file, 'w') |
|
| 17 |
output_file_name = "./squery-" + letter + ".out" |
|
| 18 |
output_file = open(output_file_name, 'w') |
|
| 19 |
output_file.close() |
|
| 19 | 20 |
|
| 20 | 21 |
def debug (message): |
| 22 |
output_file = open(output_file_name, 'a') |
|
| 21 | 23 |
output_file.write(message + '\n') |
| 24 |
output_file.close() |
|
| 22 | 25 |
|
| 23 | 26 |
def debug1 (message): |
| 24 |
if debug > 0: |
|
| 27 |
if debug_level > 0: |
|
| 28 |
output_file = open(output_file_name, 'a') |
|
| 25 | 29 |
output_file.write(message + '\n') |
| 30 |
output_file.close() |
|
| 26 | 31 |
|
| 27 | 32 |
def debug2 (message): |
| 28 |
if debug > 1: |
|
| 33 |
if debug_level > 1: |
|
| 34 |
output_file = open(output_file_name, 'a') |
|
| 29 | 35 |
output_file.write(message + '\n') |
| 36 |
output_file.close() |
|
| 30 | 37 |
|
| 31 | 38 |
def debug3 (message): |
| 32 |
if debug > 2: |
|
| 39 |
if debug_level > 2: |
|
| 40 |
output_file = open(output_file_name, 'a') |
|
| 33 | 41 |
output_file.write(message + '\n') |
| 42 |
output_file.close() |
|
| 34 | 43 |
|
| 35 |
dictionary_file = open('./dictionary.txt', 'r')
|
|
| 44 |
dictionary_file_name = './dictionary-' + letter + '.txt' |
|
| 45 |
dictionary_file = open(dictionary_file_name, 'r') |
|
| 36 | 46 |
|
| 37 | 47 |
iter_count = 0 |
| 38 | 48 |
|
| ... | ... | |
| 41 | 51 |
debug(" letter: $letter ")
|
| 42 | 52 |
debug(" iterations: $iterations ")
|
| 43 | 53 |
debug(" interval: $interval ")
|
| 54 |
debug(" host: $host ")
|
|
| 44 | 55 |
debug("************************************************** ")
|
| 45 | 56 |
|
| 46 | 57 |
mc = metacat.MetacatClient(host,'/knb/metacat') |
| ... | ... | |
| 50 | 61 |
query_template_file.close |
| 51 | 62 |
|
| 52 | 63 |
for line in dictionary_file: |
| 53 |
if line.lower().startswith(letter.lower()): |
|
| 54 |
word = line.strip() |
|
| 55 |
debug("[%s] -- Processing Search" % (word))
|
|
| 56 |
query = query_template.replace("@!search-word!@", word)
|
|
| 57 |
t1 = time() |
|
| 58 |
response = mc.squery(query) |
|
| 59 |
t2 = time() |
|
| 64 |
word = line.strip() |
|
| 65 |
debug("[%s] -- Processing Search" % (word))
|
|
| 66 |
query = query_template.replace("@!search-word!@", word)
|
|
| 67 |
t1 = time() |
|
| 68 |
response = mc.squery(query) |
|
| 69 |
t2 = time() |
|
| 60 | 70 |
|
| 61 |
if response.find("<resultset>") != -1:
|
|
| 62 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (word, t2-t1))
|
|
| 63 |
else:
|
|
| 64 |
debug("[%s] -- ERROR: " % (word), response)
|
|
| 71 |
if response.find("<resultset>") != -1:
|
|
| 72 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (word, t2-t1))
|
|
| 73 |
else: |
|
| 74 |
debug("[%s] -- ERROR: " % (word), response)
|
|
| 65 | 75 |
|
| 66 |
iter_count = iter_count + 1
|
|
| 67 |
debug3("iter_count: %d, iterations: %d" % (iter_count, iterations))
|
|
| 68 |
if iter_count >= iterations:
|
|
| 69 |
sys.exit()
|
|
| 76 |
iter_count = iter_count + 1 |
|
| 77 |
debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
|
|
| 78 |
if iter_count >= iterations: |
|
| 79 |
sys.exit() |
|
| 70 | 80 |
|
| 71 |
sleep(interval)
|
|
| 81 |
sleep(interval) |
|
| 72 | 82 |
|
| 73 | 83 |
dictionary_file.close() |
| 74 | 84 |
output_file.close() |
| test/stress-test/read-load-test.py | ||
|---|---|---|
| 1 |
#!/usr/bin/env python |
|
| 2 |
|
|
| 3 |
import metacat |
|
| 4 |
import sys |
|
| 5 |
|
|
| 6 |
from time import time |
|
| 7 |
from time import sleep |
|
| 8 |
|
|
| 9 |
prefix = sys.argv[1] |
|
| 10 |
iterations = int(sys.argv[2]) |
|
| 11 |
interval = int(sys.argv[3]) |
|
| 12 |
host = sys.argv[4] |
|
| 13 |
|
|
| 14 |
# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug. |
|
| 15 |
debug_level=2 |
|
| 16 |
|
|
| 17 |
run_time = str(time()).replace('.','')
|
|
| 18 |
|
|
| 19 |
output_file_name = "./read-" + prefix + ".out" |
|
| 20 |
output_file = open(output_file_name, 'w') |
|
| 21 |
output_file.close() |
|
| 22 |
|
|
| 23 |
def debug (message): |
|
| 24 |
output_file = open(output_file_name, 'a') |
|
| 25 |
output_file.write(message + '\n') |
|
| 26 |
output_file.close() |
|
| 27 |
|
|
| 28 |
def debug1 (message): |
|
| 29 |
if debug_level > 0: |
|
| 30 |
output_file = open(output_file_name, 'a') |
|
| 31 |
output_file.write(message + '\n') |
|
| 32 |
output_file.close() |
|
| 33 |
|
|
| 34 |
def debug2 (message): |
|
| 35 |
if debug_level > 1: |
|
| 36 |
output_file = open(output_file_name, 'a') |
|
| 37 |
output_file.write(message + '\n') |
|
| 38 |
output_file.close() |
|
| 39 |
|
|
| 40 |
def debug3 (message): |
|
| 41 |
if debug_level > 2: |
|
| 42 |
output_file = open(output_file_name, 'a') |
|
| 43 |
output_file.write(message + '\n') |
|
| 44 |
output_file.close() |
|
| 45 |
|
|
| 46 |
iter_count = 0 |
|
| 47 |
|
|
| 48 |
debug("************************************************** ")
|
|
| 49 |
debug("Starting read-load-test for ")
|
|
| 50 |
debug(" letter: $letter ")
|
|
| 51 |
debug(" iterations: $iterations ")
|
|
| 52 |
debug(" interval: $interval ")
|
|
| 53 |
debug(" host: $host ")
|
|
| 54 |
debug("************************************************** ")
|
|
| 55 |
|
|
| 56 |
# Initialize a metacat client connection and log in as test user |
|
| 57 |
t1 = time() |
|
| 58 |
mc = metacat.MetacatClient(host,'/knb/metacat') |
|
| 59 |
debug("[test] -- Processing Login")
|
|
| 60 |
response = mc.login('test', 'test', 'NCEAS')
|
|
| 61 |
t2 = time() |
|
| 62 |
|
|
| 63 |
if (response): |
|
| 64 |
debug("[test] -- SUCCESS: elapsed time: %.5f seconds" % (t2-t1))
|
|
| 65 |
else: |
|
| 66 |
debug("[test] -- ERROR: could not log in")
|
|
| 67 |
|
|
| 68 |
# Insert a document |
|
| 69 |
insert_template_file = open('insert.xml.tmpl', 'r')
|
|
| 70 |
insert_template = insert_template_file.read() |
|
| 71 |
insert_template_file.close |
|
| 72 |
|
|
| 73 |
t1 = time() |
|
| 74 |
docid = 'readtest-' + prefix + run_time + '.1.1' |
|
| 75 |
doc = insert_template.replace("@!docid!@", docid)
|
|
| 76 |
debug("[%s] -- Processing Insert" % (docid))
|
|
| 77 |
response = mc.insert(docid, doc) |
|
| 78 |
t2 = time() |
|
| 79 |
|
|
| 80 |
if (response.lower().find('<error>') == -1):
|
|
| 81 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
|
|
| 82 |
else: |
|
| 83 |
debug("[%s] -- ERROR: %s" % (docid, response))
|
|
| 84 |
t2 = time() |
|
| 85 |
|
|
| 86 |
for i in range(1,iterations): |
|
| 87 |
t1 = time() |
|
| 88 |
|
|
| 89 |
debug("[%s] -- Processing Read" % (docid))
|
|
| 90 |
response = mc.read(docid) |
|
| 91 |
t2 = time() |
|
| 92 |
|
|
| 93 |
if (response.lower().find('<error>') == -1):
|
|
| 94 |
debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
|
|
| 95 |
else: |
|
| 96 |
debug("[%s] -- ERROR: %s" % (docid, response))
|
|
| 97 |
|
|
| 98 |
iter_count = iter_count + 1 |
|
| 99 |
debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
|
|
| 100 |
|
|
| 101 |
sleep(interval) |
|
| 102 |
|
|
| 103 |
output_file.close() |
|
| test/stress-test/load-test-driver.sh | ||
|---|---|---|
| 6 | 6 |
# 3 April 2009 Michael Daigle (daigle@nceas.ucsb.edu) |
| 7 | 7 |
############################################################################# |
| 8 | 8 |
|
| 9 |
###################### |
|
| 9 | 10 |
# squery-load-test.py dictionary-letter iterations delay test-server |
| 11 |
###################### |
|
| 10 | 12 |
|
| 11 | 13 |
##### run squeries ##### |
| 12 |
./squery-load-test.py a 100 3 dev.nceas.ucsb.edu 2>&1 &
|
|
| 14 |
#./squery-load-test.py m 50 2 dev.nceas.ucsb.edu 2>&1 &
|
|
| 13 | 15 |
|
| 14 | 16 |
##### run squeries #####" |
| 15 |
./squery-load-test.py b 100 3 dev.nceas.ucsb.edu 2>&1 &
|
|
| 17 |
#./squery-load-test.py n 50 2 dev.nceas.ucsb.edu 2>&1 &
|
|
| 16 | 18 |
|
| 17 | 19 |
##### run squeries ##### |
| 18 |
./squery-load-test.py c 100 3 dev.nceas.ucsb.edu 2>&1 &
|
|
| 20 |
#./squery-load-test.py o 50 2 dev.nceas.ucsb.edu 2>&1 &
|
|
| 19 | 21 |
|
| 20 | 22 |
##### run squeries ##### |
| 21 |
./squery-load-test.py d 100 3 dev.nceas.ucsb.edu 2>&1 &
|
|
| 23 |
#./squery-load-test.py p 50 2 dev.nceas.ucsb.edu 2>&1 &
|
|
| 22 | 24 |
|
| 25 |
###################### |
|
| 26 |
# insert-load-test.py dictionary-letter iterations delay test-server |
|
| 27 |
###################### |
|
| 23 | 28 |
|
| 29 |
##### run squeries ##### |
|
| 30 |
./insert-load-test.py a 50 2 dev.nceas.ucsb.edu 2>&1 & |
|
| 31 |
|
|
| 32 |
##### run squeries #####" |
|
| 33 |
./insert-load-test.py b 50 2 dev.nceas.ucsb.edu 2>&1 & |
|
| 34 |
|
|
| 35 |
##### run squeries ##### |
|
| 36 |
./insert-load-test.py c 50 2 dev.nceas.ucsb.edu 2>&1 & |
|
| 37 |
|
|
| 38 |
##### run squeries ##### |
|
| 39 |
./insert-load-test.py d 50 2 dev.nceas.ucsb.edu 2>&1 & |
|
| 40 |
|
|
| 41 |
###################### |
|
| 42 |
# read-load-test.py prefix iterations delay test-server |
|
| 43 |
###################### |
|
| 44 |
|
|
| 45 |
##### run squeries ##### |
|
| 46 |
#./read-load-test.py a 300 3 dev.nceas.ucsb.edu 2>&1 & |
|
| 47 |
|
|
| 48 |
##### run squeries #####" |
|
| 49 |
#./read-load-test.py b 300 3 dev.nceas.ucsb.edu 2>&1 & |
|
| 50 |
|
|
| 51 |
##### run squeries ##### |
|
| 52 |
#./read-load-test.py c 300 3 dev.nceas.ucsb.edu 2>&1 & |
|
| 53 |
|
|
| 54 |
##### run squeries ##### |
|
| 55 |
#./read-load-test.py d 300 3 dev.nceas.ucsb.edu 2>&1 & |
|
| 56 |
|
|
| 57 |
echo |
|
| 58 |
echo "Running the following load tests:" |
|
| 59 |
ps auxwww |grep load-test.py |grep -v grep |
|
| 60 |
|
|
| test/stress-test/generate-dictionaries.sh | ||
|---|---|---|
| 1 |
#!/bin/bash |
|
| 2 |
|
|
| 3 |
############################################################################# |
|
| 4 |
# Create dictionary files of five letter words with every possible combination |
|
| 5 |
# of letters. Each file is named dictionary-X.txt where X is each letter of |
|
| 6 |
# the alphabet and the file holds all possible words that start with that |
|
| 7 |
# letter. |
|
| 8 |
# |
|
| 9 |
# 6 April 2009 Michael Daigle (daigle@nceas.ucsb.edu) |
|
| 10 |
############################################################################# |
|
| 11 |
|
|
| 12 |
ltr=(a b c d e f g h i j k l m n o p q r s t u v w x y z) |
|
| 13 |
|
|
| 14 |
> ./dictionary2.txt |
|
| 15 |
|
|
| 16 |
for i in {0..25}
|
|
| 17 |
do |
|
| 18 |
for j in {0..25}
|
|
| 19 |
do |
|
| 20 |
for k in {0..25}
|
|
| 21 |
do |
|
| 22 |
for l in {0..25}
|
|
| 23 |
do |
|
| 24 |
for m in {0..25}
|
|
| 25 |
do |
|
| 26 |
echo ${ltr[$i]}${ltr[$j]}${ltr[$k]}${ltr[$l]}${ltr[$m]} >> ./dictionary-${ltr[$i]}.txt
|
|
| 27 |
done |
|
| 28 |
done |
|
| 29 |
done |
|
| 30 |
done |
|
| 31 |
done |
|
| 0 | 32 | |
Also available in: Unified diff
Update load test code