Skip to content

E
e2e-packet
Unverified Commit da1859a0 authored by Karthik Satchitanand's avatar Karthik Satchitanand Committed by GitHub
Browse files
Options

Merge pull request #6 from nsathyaseelan/modularization 

Create reusable bash functions & modularize the gitlab runner scripts
parents 19b7ea7e 54c0a57f
Hide whitespace changes
Inline Side-by-side
Showing with 968 additions and 2615 deletions
.gitlab-ci.yml
View file @ da1859a0
......@@ -10,6 +10,9 @@ stages:
- APP-CHAOS-TEST
- CLUSTER-CLEANUP
variables:
utils_path: "/openebs/e2e-packet/script/utils"
# Setup the kubernetes cluster
packet-cluster:
......@@ -134,6 +137,25 @@ mongo-cstor-run/load/check 1:2:
script:
- ./script/apps/mongo/workload/cstor/mongo-app-workload-cstor
# CASSANDRA_CSTOR
cassandra-cstor-run/load/check 0:2:
extends: .app_deploy_template
script:
- ./script/apps/cassandra/deployer/cassandra-app-deploy-cstor
cassandra-cstor-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/cassandra/workload/cstor/cassandra-app-workload-cstor
# JENKINS_CSTOR
jenkins-cstor-run/load/check:
extends: .app_deploy_template
script:
- ./script/apps/jenkins/deployer/jenkins-app-deploy-cstor
## Define job template for func test jobs
.func_test_template:
......@@ -155,20 +177,35 @@ app-upgrade-deployment-{jenkins-jiva}:
- ./script/apps/jenkins/functional/jiva/app-upgrade-deployment
volume-data-integrity-{fio-jiva}:
extends: .func_test_template #dependencies: jenkins-jiva
extends: .func_test_template
script:
- ./script/apps/fio/tests/jiva/fio-volume-data-integrity-check
volume-data-integrity-{fio-cstor}:
extends: .func_test_template
script:
- ./script/apps/fio/tests/cstor/fio-volume-data-integrity-check
volume-memory-consumption-{memcheck-jiva}:
extends: .func_test_template
script:
- ./script/apps/memleak/tests/jiva/memory-consumption
snapshot-clone-creation-{percona-jiva}:
k8s-snapshot-clone-creation-{percona-jiva}:
extends: .func_test_template
script:
- ./script/apps/percona/functional/jiva/k8s-snapshot-clone
app-replica-scale-{cassandra-cstor}:
extends: .func_test_template #dependencies: cassandra-cstor
script:
- ./script/apps/cassandra/functional/cstor/app-sts-replica-scale
app-upgrade-deployment-{jenkins-cstor}:
extends: .func_test_template #dependencies: jenkins-cstor
script:
- ./script/apps/jenkins/functional/cstor/app-upgrade-deployment
## Define job template for chaos jobs
.chaos_test_template:
......@@ -199,6 +236,11 @@ ctrl-disconnect-{postgres-jiva}:
script:
- ./script/apps/postgres/chaos/jiva/volume-controller-network-delay
app-kill-{jenkins-cstor}:
extends: .chaos_test_template #dependencies: jenkins-cstor
script:
- ./script/apps/jenkins/chaos/cstor/application-pod-failure
tgt-kill-{percona-cstor}:
extends: .chaos_test_template #dependencies: percona-cstor
script:
......
script/apps/cassandra/deployer/cassandra-app-deploy-cstor 0 → 100755
View file @ da1859a0
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## https://github.com/openebs/litmus/blob/master/apps/cassandra/deployers/test_vars.yml
run_id="cstor";test_name=$(${utils_path}/generate_test_name testcase=cassandra-deployment metadata=${run_id})
###################
## DEPENDENCIES ##
###################
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
############################
## LITMUS PRECONDITIONING ##
############################
: << EOF
-------------------------------------------------------------------------------------------
| specAttribute | kind | jiva | cStor-sparse |
-------------------------------------------------------------------------------------------
| litmusJobLabel | jobSpec | cassandra-deployment-litmus | casandra-deployment-cstor|
| providerStorageClass | env | openebs-standalone | openebs-cstor-sparse |
| appNamespace | env | app-cass-ns | cassandra-cstor |
| runID | env(add) | - | cstor |
-------------------------------------------------------------------------------------------
EOF
cp apps/cassandra/deployers/run_litmus_test.yml run_test.yml
sed -i -e 's/app: cassandra-deployment-litmus/app: cassandra-deployment-cstor/g' \
-e 's/value: openebs-standalone/value: openebs-cstor-sparse/g' \
-e 's/value: app-cass-ns/value: cassandra-cstor/g' run_test.yml
sed -i '/command:/i \
- name: RUN_ID\
value: '"$run_id"'\
' run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
${utils_path}/litmus_job_runner label='app:cassandra-deployment-cstor' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
${utils_path}/get_litmus_result ${test_name}
script/apps/cassandra/deployer/cassandra-app-deploy-jiva
View file @ da1859a0
......@@ -5,133 +5,40 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/cassandra/deployers/test_vars.yml
test_name="cassandra-deployment"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
test_name=$(${utils_path}/generate_test_name testcase=cassandra-deployment metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod.
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab.
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/deployers/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for cassandra app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-deployment-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/deployers/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='app:cassandra-deployment-litmus' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
${utils_path}/get_litmus_result ${test_name}
script/apps/cassandra/functional/cstor/app-sts-replica-scale 0 → 100755
View file @ da1859a0
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## https://github.com/openebs/litmus/blob/master/apps/cassandra/functional/scale_replicas/test_vars.yml
run_id="cstor";test_name=$(${utils_path}/generate_test_name testcase=scale-app-replicas metadata=${run_id})
###################
## DEPENDENCIES ##
###################
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
############################
## LITMUS PRECONDITIONING ##
############################
: << EOF
--------------------------------------------------------------------------------------
| specAttribute | kind | jiva | cStor-sparse |
--------------------------------------------------------------------------------------
| litmusJobLabel | jobSpec | cassandra-scaleup-litmus | cassandra-scaleup-cstor |
| appNamespace | env | app-cass-ns | cassandra-cstor |
| runID | env(add) | - | cstor |
--------------------------------------------------------------------------------------
EOF
cp apps/cassandra/functional/scale_replicas/run_litmus_test.yml run_test.yml
sed -i -e 's/app: cassandra-scaleup-litmus/app: cassandra-scaleup-cstor/g' \
-e 's/value: app-cass-ns/value: cassandra-cstor/g' run_test.yml
sed -i '/command:/i \
- name: RUN_ID\
value: '"$run_id"'\
' run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
${utils_path}/litmus_job_runner label='app:cassandra-scaleup-cstor' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
${utils_path}/get_litmus_result ${test_name}
script/apps/cassandra/functional/jiva/app-sts-replica-scale
View file @ da1859a0
......@@ -5,47 +5,16 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/chaos/openebs-replica-network-delay/test_vars.yml
test_name="scale-app-replicas"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
## https://github.com/openebs/litmus/blob/master/apps/cassandra/functional/scale_replicas/test_vars.yml
test_name=$(${utils_path}/generate_test_name testcase=scale-app-replicas metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
......@@ -55,14 +24,11 @@ cp -r openebs-packet/.kube/. ~/.kube/
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/functional/scale_replicas/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
......@@ -72,65 +38,17 @@ echo "Running the litmus test.."
## Create the litmus job for cassandra app func test: scale-app-replicas
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-scaleup-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/functional/scale_replicas/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='app:cassandra-scaleup-litmus' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
task_delimiter;
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
${utils_path}/get_litmus_result ${test_name}
script/apps/cassandra/workload/cstor/cassandra-app-workload-cstor 0 → 100755
View file @ da1859a0
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## https://github.com/openebs/litmus/blob/master/apps/cassandra/workload/test_vars.yml
run_id="cstor";test_name=$(${utils_path}/generate_test_name testcase=cassandra-loadgen metadata=${run_id})
###################
## DEPENDENCIES ##
###################
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
############################
## LITMUS PRECONDITIONING ##
############################
: << EOF
------------------------------------------------------------------------------------
| specAttribute | kind | jiva | cStor-sparse |
------------------------------------------------------------------------------------
| litmusJobLabel | jobSpec | cassandra-loadjob | casandra-loadjob-cstor |
| appNamespace | env | app-cass-ns | cassandra-cstor |
| runID | env(add) | - | cstor
------------------------------------------------------------------------------------
EOF
cp apps/cassandra/workload/run_litmus_test.yml run_test.yml
sed -i -e 's/loadgen: cassandra-loadjob/loadgen: cassandra-loadjob-cstor/g' \
-e 's/value: app-cass-ns/value: cassandra-cstor/g' run_test.yml
sed -i '/command:/i \
- name: RUN_ID\
value: '"$run_id"'\
' run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
${utils_path}/litmus_job_runner label='loadgen:cassandra-loadjob-cstor' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
${utils_path}/get_litmus_result ${test_name}
script/apps/cassandra/workload/jiva/cassandra-app-workload-jiva
View file @ da1859a0
......@@ -5,132 +5,40 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/deployers/test_vars.yml
test_name="cassandra-loadgen"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
## https://github.com/openebs/litmus/blob/master/apps/cassandra/workload/test_vars.yml
test_name=$(${utils_path}/generate_test_name testcase=cassandra-loadgen metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
cd litmus
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/workload/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for percona app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.loadgen==\"cassandra-loadjob\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/workload/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='loadgen:cassandra-loadjob' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
${utils_path}/get_litmus_result ${test_name}