Unverified Commit ea50e4e9 authored by Karthik Satchitanand's avatar Karthik Satchitanand Committed by GitHub

Merge pull request #3 from nsathyaseelan/restructure

Restructure e2e-packet scripts to categorize type of jobs & include cstor specific jobs
parents a05b702e 7299d9cf
## TODO: Modularize the gitlab yaml to reuse templates
## Define the stages & order of execution
stages:
- cluster-setup
- infra-setup
- stateful-apps-deploy
- Litmus-chaos-testing
- cleanup
- deploy
- CLUSTER-SETUP
- PROVIDER-INFRA-SETUP
- STATEFUL-APP-DEPLOY
- APP-FUNC-TEST
- APP-CHAOS-TEST
- CLUSTER-CLEANUP
# Setup the kubernetes cluster
packet-cluster:
image: chandankumar4/packet:v4
stage: cluster-setup
script:
stage: CLUSTER-SETUP
script:
- mkdir -p ~/.ssh && ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
- echo -e "Host *\nStrictHostKeyChecking no" > ~/.ssh/config
- chmod 755 ./script/packet
- ./script/packet
artifacts:
paths:
- cluster/
openebs-deploy:
## Setup OpenEBS control plane
openebs-packet-deploy:
image: atulabhi/kops:v8
stage: infra-setup
stage: PROVIDER-INFRA-SETUP
dependencies:
- packet-cluster
script:
- echo "deploying OpenEBS on Packet"
- chmod 775 ./script/infra-setup
- ./script/infra-setup
- ./script/provider/infra-setup
artifacts:
paths:
- openebs-packet/
percona-deploy:
## Define a job template for app deployers
.app_deploy_template:
image: atulabhi/kops:v8
stage: stateful-apps-deploy
stage: STATEFUL-APP-DEPLOY
dependencies:
- openebs-deploy
- openebs-packet-deploy
artifacts:
paths:
- openebs-packet/
## Define the app deployer jobs
## PERCONA-JIVA
percona-jiva-run/load/check 0:2:
extends: .app_deploy_template
script:
- echo "jobs-Packet-Started"
- chmod 775 ./script/stateful-app-deploy
- ./script/stateful-app-deploy
- ./script/apps/percona/deployer/percona-app-deploy-jiva
artifacts:
paths:
- packet-percona/
percona-jiva-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/percona/workload/jiva/percona-app-workload-jiva
apps-packet-deploy-cstor:
image: atulabhi/kops:v8
stage: stateful-apps-deploy
dependencies:
- openebs-deploy
## MONGO-JIVA
mongo-jiva-run/load/check 0:2:
extends: .app_deploy_template
script:
- ./script/apps/mongo/deployer/mongo-app-deploy-jiva
mongo-jiva-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/mongo/workload/jiva/mongo-app-workload-jiva
# POSTGRES-JIVA
postgres-jiva-run/load/check 0:2:
extends: .app_deploy_template
script:
- ./script/apps/postgres/deployer/postgres-app-deploy-jiva
postgres-jiva-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/postgres/workload/jiva/postgres-app-workload-jiva
# CASSANDRA-JIVA
cassandra-jiva-run/load/check 0:2:
extends: .app_deploy_template
script:
- ./script/apps/cassandra/deployer/cassandra-app-deploy-jiva
cassandra-jiva-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/cassandra/workload/jiva/cassandra-app-workload-jiva
# JENKINS-JIVA
jenkins-jiva-run/load/check:
extends: .app_deploy_template
script:
- ./script/apps/jenkins/deployer/jenkins-app-deploy-jiva
# PERCONA-CSTOR
percona-cstor-run/load/check 0:2:
extends: .app_deploy_template
script:
- echo "jobs-packet-Started"
- chmod 775 ./script/stateful-app-deploy-cstor
- ./script/stateful-app-deploy-cstor
- ./script/apps/percona/deployer/percona-app-deploy-cstor
percona-cstor-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/percona/workload/cstor/percona-app-workload-cstor
# MONGO-CSTOR
mongo-cstor-run/load/check 0:2:
extends: .app_deploy_template
script:
- ./script/apps/mongo/deployer/mongo-app-deploy-cstor
mongo-cstor-run/load/check 1:2:
extends: .app_deploy_template
before_script:
- sleep 180
script:
- ./script/apps/mongo/workload/cstor/mongo-app-workload-cstor
## Define job template for func test jobs
.func_test_template:
image: atulabhi/kops:v8
stage: APP-FUNC-TEST
when: always
artifacts:
paths:
- packet-cstor/
paths:
- openebs-packet/
app-replica-scale-{cassandra-jiva}:
extends: .func_test_template #dependencies: cassandra-jiva
script:
- ./script/apps/cassandra/functional/jiva/app-sts-replica-scale
app-upgrade-deployment-{jenkins-jiva}:
extends: .func_test_template #dependencies: jenkins-jiva
script:
- ./script/apps/jenkins/functional/jiva/app-upgrade-deployment
volume-data-integrity-{fio-jiva}:
extends: .func_test_template #dependencies: jenkins-jiva
script:
- ./script/apps/fio/tests/jiva/fio-volume-data-integrity-check
## Define job template for chaos jobs
Litmus-Chaos:
.chaos_test_template:
image: atulabhi/kops:v8
dependencies:
- openebs-deploy
stage: Litmus-chaos-testing
stage: APP-CHAOS-TEST
when: always
artifacts:
paths:
- openebs-packet/
repl-kill-{percona-jiva}:
extends: .chaos_test_template #dependencies: percona-jiva
script:
- ./script/apps/percona/chaos/jiva/volume-replica-failure
ctrl-kill-{mongo-jiva}:
extends: .chaos_test_template #dependencies: mongo-jiva
script:
- ./script/apps/mongo/chaos/jiva/volume-controller-failure
repl-disconnect-{jenkins-jiva}:
extends: .chaos_test_template #dependencies: jenkins-jiva
script:
- ./script/apps/jenkins/chaos/jiva/volume-replica-network-delay
ctrl-disconnect-{postgres-jiva}:
extends: .chaos_test_template #dependencies: postgres-jiva
script:
- ./script/apps/postgres/chaos/jiva/volume-controller-network-delay
tgt-kill-{percona-cstor}:
extends: .chaos_test_template #dependencies: percona-cstor
script:
- ./script/apps/percona/chaos/cstor/volume-target-failure
tgt-disconnect-{mongo-cstor}:
extends: .chaos_test_template #dependencies: mongo-cstor
script:
- echo "Introducing-Chaos"
- mkdir ~/.kube && cp -r openebs-packet/.kube/. ~/.kube/
- kubectl create -f https://raw.githubusercontent.com/atulabhi/litmus/v0.7-RC1/apps/percona/chaos/openebs_volume_replica_failure/run_litmus_test.yaml
- chmod 755 ./script/chaos
- ./script/chaos
- ./script/apps/mongo/chaos/cstor/volume-target-network-delay
cleanup-packet:
when: always
image: chandankumar4/packet:v4
dependencies:
- packet-cluster
stage: cleanup
stage: CLUSTER-CLEANUP
script:
- chmod 755 ./script/packet-cleanup
- ./script/packet-cleanup
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/cassandra/deployers/test_vars.yml
test_name="cassandra-deployment"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod.
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab.
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for cassandra app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-deployment-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/deployers/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
task_delimiter;
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/chaos/openebs-replica-network-delay/test_vars.yml
test_name="scale-app-replicas"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for cassandra app func test: scale-app-replicas
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-scaleup-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/functional/scale_replicas/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
task_delimiter;
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/deployers/test_vars.yml
test_name="cassandra-loadgen"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for percona app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.loadgen==\"cassandra-loadjob\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/workload/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
task_delimiter;
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
#!/bin/bash
set -x
#################
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/fio/tests/data-integrity/test_vars.yml
test_name="fio-data-integrity"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent