-
Notifications
You must be signed in to change notification settings - Fork 738
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Introduce automated performance testing. #1068
Changes from 1 commit
a8323cc
c8b1f9b
0cf8c5a
cf2b5d8
07b54f2
55220bd
ef4df24
5a40307
542382b
40173e0
615517a
dcf5d4a
c7c3701
3060990
42b79e9
485d598
2d8d01b
efd0151
3d3f14e
a5b2086
ee6a9ac
9d0aa34
0325ae5
b76d9b0
591deea
ebef0ae
5e01f11
711896a
1a5bbfc
e4d603f
7e2ad40
d43d4a6
c5c9564
449f88c
082901b
4811d74
f674a55
487ce50
439ea17
a2f8c27
0e7ef6e
56b0118
b46b3ea
18b10c5
9f2e891
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,209 @@ | ||
function check_for_timeout() { | ||
if [[ $((SECONDS - $1)) -gt 10000 ]]; then | ||
RUNNING_PERFORMANCE=false | ||
on_error | ||
fi | ||
} | ||
|
||
function run_performance_test_130_pods() { | ||
echo "Running performance tests against cluster" | ||
RUNNING_PERFORMANCE=true | ||
|
||
DEPLOY_START=$SECONDS | ||
|
||
SCALE_UP_DURATION_ARRAY=() | ||
SCALE_DOWN_DURATION_ARRAY=() | ||
while [ ${#SCALE_UP_DURATION_ARRAY[@]} -lt 3 ] | ||
do | ||
ITERATION_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-130-pods.yaml --replicas=130 | ||
sleep 20 | ||
while [[ ! $($KUBECTL_PATH get deploy | grep 130/130) ]] | ||
do | ||
sleep 1 | ||
echo "Scaling UP" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
|
||
SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) ) | ||
MIDPOINT_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-130-pods.yaml --replicas=0 | ||
while [[ $($KUBECTL_PATH get pods) ]] | ||
do | ||
sleep 1 | ||
echo "Scaling DOWN" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
SCALE_DOWN_DURATION_ARRAY+=($((SECONDS - MIDPOINT_START))) | ||
done | ||
|
||
echo "Times to scale up:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_UP_DURATION_ARRAY[@]} ] | ||
do | ||
echo ${SCALE_UP_DURATION_ARRAY[$INDEX]} | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
echo "Times to scale down:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_DOWN_DURATION_ARRAY[@]} ] | ||
do | ||
echo "${SCALE_DOWN_DURATION_ARRAY[$INDEX]} seconds" | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
DEPLOY_DURATION=$((SECONDS - DEPLOY_START)) | ||
|
||
now="pod-130-Test#${TEST_ID}-$(date +"%m-%d-%Y-%T").csv" | ||
echo $now | ||
|
||
echo $(date +"%m-%d-%Y-%T") >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[0])), $((SCALE_DOWN_DURATION_ARRAY[0])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[1])), $((SCALE_DOWN_DURATION_ARRAY[1])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[2])), $((SCALE_DOWN_DURATION_ARRAY[2])) >> $now | ||
|
||
cat $now | ||
aws s3 cp $now s3://cni-performance-test-data | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this bucket needs to be a configurable setting. And if it's not set, we should skip the upload. Something like if [[ -n "${S3_PERF_TEST_BUCKET:-}" ]]; then
aws s3 cp $filename "$S3_PERF_TEST_BUCKET"
else
echo "No S3 bucket name given, not uploading results"
fi There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we do the check for S3 bucket before the test? |
||
|
||
echo "TIMELINE: 130 Pod performance test took $DEPLOY_DURATION seconds." | ||
RUNNING_PERFORMANCE=false | ||
} | ||
|
||
function run_performance_test_730_pods() { | ||
echo "Running performance tests against cluster" | ||
RUNNING_PERFORMANCE=true | ||
|
||
DEPLOY_START=$SECONDS | ||
|
||
SCALE_UP_DURATION_ARRAY=() | ||
SCALE_DOWN_DURATION_ARRAY=() | ||
while [ ${#SCALE_UP_DURATION_ARRAY[@]} -lt 3 ] | ||
do | ||
ITERATION_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-730-pods.yaml --replicas=730 | ||
sleep 100 | ||
while [[ ! $($KUBECTL_PATH get deploy | grep 730/730) ]] | ||
do | ||
sleep 2 | ||
echo "Scaling UP" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
|
||
SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) ) | ||
MIDPOINT_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-730-pods.yaml --replicas=0 | ||
sleep 100 | ||
while [[ $($KUBECTL_PATH get pods) ]] | ||
do | ||
sleep 2 | ||
echo "Scaling DOWN" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
SCALE_DOWN_DURATION_ARRAY+=($((SECONDS - MIDPOINT_START))) | ||
done | ||
|
||
echo "Times to scale up:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_UP_DURATION_ARRAY[@]} ] | ||
do | ||
echo ${SCALE_UP_DURATION_ARRAY[$INDEX]} | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
echo "Times to scale down:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_DOWN_DURATION_ARRAY[@]} ] | ||
do | ||
echo "${SCALE_DOWN_DURATION_ARRAY[$INDEX]} seconds" | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
DEPLOY_DURATION=$((SECONDS - DEPLOY_START)) | ||
|
||
now="pod-730-Test#${TEST_ID}-$(date +"%m-%d-%Y-%T").csv" | ||
echo $now | ||
|
||
echo $(date +"%m-%d-%Y-%T") >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[0])), $((SCALE_DOWN_DURATION_ARRAY[0])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[1])), $((SCALE_DOWN_DURATION_ARRAY[1])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[2])), $((SCALE_DOWN_DURATION_ARRAY[2])) >> $now | ||
|
||
cat $now | ||
aws s3 cp $now s3://cni-performance-test-data | ||
|
||
echo "TIMELINE: 730 Pod performance test took $DEPLOY_DURATION seconds." | ||
RUNNING_PERFORMANCE=false | ||
} | ||
|
||
function run_performance_test_5000_pods() { | ||
echo "Running performance tests against cluster" | ||
RUNNING_PERFORMANCE=true | ||
|
||
DEPLOY_START=$SECONDS | ||
|
||
SCALE_UP_DURATION_ARRAY=() | ||
SCALE_DOWN_DURATION_ARRAY=() | ||
while [ ${#SCALE_UP_DURATION_ARRAY[@]} -lt 3 ] | ||
do | ||
ITERATION_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-5000-pods.yaml --replicas=5000 | ||
sleep 100 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is 100 seconds based on here? |
||
while [[ ! $($KUBECTL_PATH get deploy | grep 5000/5000) ]] | ||
do | ||
sleep 2 | ||
echo "Scaling UP" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
|
||
SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) ) | ||
MIDPOINT_START=$SECONDS | ||
$KUBECTL_PATH scale -f ./testdata/deploy-5000-pods.yaml --replicas=0 | ||
sleep 100 | ||
while [[ $($KUBECTL_PATH get pods) ]] | ||
do | ||
sleep 2 | ||
echo "Scaling DOWN" | ||
echo $($KUBECTL_PATH get deploy) | ||
check_for_timeout $DEPLOY_START | ||
done | ||
SCALE_DOWN_DURATION_ARRAY+=($((SECONDS - MIDPOINT_START))) | ||
done | ||
|
||
echo "Times to scale up:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_UP_DURATION_ARRAY[@]} ] | ||
do | ||
echo ${SCALE_UP_DURATION_ARRAY[$INDEX]} | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
echo "Times to scale down:" | ||
INDEX=0 | ||
while [ $INDEX -lt ${#SCALE_DOWN_DURATION_ARRAY[@]} ] | ||
do | ||
echo "${SCALE_DOWN_DURATION_ARRAY[$INDEX]} seconds" | ||
INDEX=$((INDEX + 1)) | ||
done | ||
echo "" | ||
DEPLOY_DURATION=$((SECONDS - DEPLOY_START)) | ||
|
||
now="pod-5000-Test#${TEST_ID}-$(date +"%m-%d-%Y-%T").csv" | ||
echo $now | ||
|
||
echo $(date +"%m-%d-%Y-%T") >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[0])), $((SCALE_DOWN_DURATION_ARRAY[0])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[1])), $((SCALE_DOWN_DURATION_ARRAY[1])) >> $now | ||
echo $((SCALE_UP_DURATION_ARRAY[2])), $((SCALE_DOWN_DURATION_ARRAY[2])) >> $now | ||
|
||
cat $now | ||
aws s3 cp $now s3://cni-performance-test-data | ||
|
||
echo "TIMELINE: 5000 Pod performance test took $DEPLOY_DURATION seconds." | ||
RUNNING_PERFORMANCE=false | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,6 +8,7 @@ DIR=$(cd "$(dirname "$0")"; pwd) | |
source "$DIR"/lib/common.sh | ||
source "$DIR"/lib/aws.sh | ||
source "$DIR"/lib/cluster.sh | ||
source "$DIR"/lib/performance_tests.sh | ||
|
||
# Variables used in /lib/aws.sh | ||
OS=$(go env GOOS) | ||
|
@@ -19,20 +20,24 @@ ARCH=$(go env GOARCH) | |
: "${DEPROVISION:=true}" | ||
: "${BUILD:=true}" | ||
: "${RUN_CONFORMANCE:=false}" | ||
: "${RUN_PERFORMANCE_TESTS:=false}" | ||
: "${RUNNING_PERFORMANCE:=false}" | ||
|
||
__cluster_created=0 | ||
__cluster_deprovisioned=0 | ||
|
||
on_error() { | ||
# Make sure we destroy any cluster that was created if we hit run into an | ||
# error when attempting to run tests against the cluster | ||
if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true ]]; then | ||
# prevent double-deprovisioning with ctrl-c during deprovisioning... | ||
__cluster_deprovisioned=1 | ||
echo "Cluster was provisioned already. Deprovisioning it..." | ||
down-test-cluster | ||
if [[ $RUNNING_PERFORMANCE == false ]]; then | ||
if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true ]]; then | ||
# prevent double-deprovisioning with ctrl-c during deprovisioning... | ||
__cluster_deprovisioned=1 | ||
echo "Cluster was provisioned already. Deprovisioning it..." | ||
down-test-cluster | ||
fi | ||
exit 1 | ||
fi | ||
exit 1 | ||
} | ||
|
||
# test specific config, results location | ||
|
@@ -213,6 +218,22 @@ if [[ $TEST_PASS -eq 0 && "$RUN_CONFORMANCE" == true ]]; then | |
echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds." | ||
fi | ||
|
||
if [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then | ||
START=$SECONDS | ||
$KUBECTL_PATH apply -f ./testdata/deploy-130-pods.yaml | ||
run_performance_test_130_pods | ||
$KUBECTL_PATH delete -f ./testdata/deploy-130-pods.yaml | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why are the testdata apply functions not done inside the test functions? |
||
|
||
$KUBECTL_PATH apply -f ./testdata/deploy-730-pods.yaml | ||
run_performance_test_730_pods | ||
$KUBECTL_PATH delete -f ./testdata/deploy-730-pods.yaml | ||
|
||
$KUBECTL_PATH apply -f ./testdata/deploy-5000-pods.yaml | ||
run_performance_test_5000_pods | ||
$KUBECTL_PATH delete -f ./testdata/deploy-5000-pods.yaml | ||
PERFORMANCE_DURATION=$((SECONDS - START)) | ||
bnapolitan marked this conversation as resolved.
Show resolved
Hide resolved
|
||
fi | ||
|
||
if [[ "$DEPROVISION" == true ]]; then | ||
START=$SECONDS | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we call this
filename
instead?