Commit 030c4d78 authored by sathyaseelan's avatar sathyaseelan

Create reusable bash functions & modularize the gitlab runner scripts

Signed-off-by: 's avatarsathyaseelan <sathyaseelan.n@cloudbyte.com>
parent 950aca2e
......@@ -10,6 +10,9 @@ stages:
- APP-CHAOS-TEST
- CLUSTER-CLEANUP
variables:
utils_path: "/openebs/e2e-packet/script/utils"
# Setup the kubernetes cluster
packet-cluster:
......
......@@ -5,133 +5,40 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/cassandra/deployers/test_vars.yml
test_name="cassandra-deployment"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
test_name=$(${utils_path}/generate_test_name testcase=cassandra-deployment metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod.
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab.
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/deployers/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for cassandra app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-deployment-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/deployers/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='app:cassandra-deployment-litmus' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
${utils_path}/get_litmus_result ${test_name}
......@@ -5,47 +5,16 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/chaos/openebs-replica-network-delay/test_vars.yml
test_name="scale-app-replicas"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
## https://github.com/openebs/litmus/blob/master/apps/cassandra/functional/scale_replicas/test_vars.yml
test_name=$(${utils_path}/generate_test_name testcase=scale-app-replicas metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
......@@ -55,14 +24,11 @@ cp -r openebs-packet/.kube/. ~/.kube/
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/functional/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
......@@ -72,65 +38,17 @@ echo "Running the litmus test.."
## Create the litmus job for cassandra app func test: scale-app-replicas
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-scaleup-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/functional/scale_replicas/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='app:cassandra-scaleup-litmus' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
task_delimiter;
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
${utils_path}/get_litmus_result ${test_name}
......@@ -5,132 +5,40 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/deployers/test_vars.yml
test_name="cassandra-loadgen"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
## https://github.com/openebs/litmus/blob/master/apps/cassandra/workload/test_vars.yml
test_name=$(${utils_path}/generate_test_name testcase=cassandra-loadgen metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
cd litmus
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/cassandra/workload/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for percona app deploy
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.loadgen==\"cassandra-loadjob\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/cassandra/workload/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='loadgen:cassandra-loadjob' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
${utils_path}/get_litmus_result ${test_name}
......@@ -5,132 +5,40 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/fio/tests/data-integrity/test_vars.yml
test_name="fio-data-integrity"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
test_name=$(${utils_path}/generate_test_name testcase=fio-data-integrity metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
cp apps/fio/tests/data-integrity/run_litmus_test.yml run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for fio data integrity test
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"fio-di-litmus\")].metadata.name}'"
job_deploy_out=$(kubectl create -f apps/fio/tests/data-integrity/run_litmus_test.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"
task_delimiter;
## Printing the test logs & cluster state for debug purposes
echo "Dumping Litmus test pod logs for debug"; echo ""
kubectl logs $litmus_pod -n litmus -c ansibletest
task_delimiter;
${utils_path}/litmus_job_runner label='app:fio-di-litmus' job=run_test.yml
${utils_path}/task_delimiter;
echo "Dumping state of cluster post job run"; echo ""
kubectl get pods --all-namespaces
kubectl get pvc --all-namespaces
kubectl get sc
${utils_path}/dump_cluster_state;
${utils_path}/task_delimiter;
task_delimiter;
#################
## GET RESULT ##
#################
## Check the test status & result from the litmus result custom resource
testStatus=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
testResult=$(kubectl get litmusresult $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
if [[ $testStatus == "completed" ]]; then
if [[ $testResult == "Pass" ]]; then
echo "TEST: PASS"
else
echo "TEST: FAILED"; exit 1
fi
else
echo "Test Execution was aborted"; exit 1
fi
${utils_path}/get_litmus_result ${test_name}
......@@ -5,66 +5,23 @@ set -x
## ENVIRONMENT ##
#################
## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
run_metadata=""
## https://github.com/openebs/litmus/blob/master/apps/percona/chaos/openebs-replica-network-delay/test_vars.yml
test_name="openebs-replica-network-delay"
if [[ -n "$run_metadata" ]]; then
test_name="$test_name-$run_metadata"
fi
################
## FUNCTIONS ##
################
error_handler()
{
rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
if [ $rc -ne 0 ]; then
echo "$message"
if [ "$act" == "exit" ]; then
exit 1
fi
fi
}
task_delimiter()
{
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
}
test_name=$(${utils_path}/generate_test_name testcase=openebs-replica-network-delay metadata="")
###################
## DEPENDENCIES ##
###################
## Derive the kubeconfig of the Packet cluster into the gitlab job runner pod
## "openebs-packet" is a shared path for the gitlab runner passed as artifact by gitlab
echo "Setting up test dependencies.."
mkdir ~/.kube
cp -r openebs-packet/.kube/. ~/.kube/
## Clone the litmus repo, checkout the e2e branch, navigate to litmus root
#git clone https://github.com/atulabhi/litmus.git
#cd litmus; git checkout v0.7-RC1
${utils_path}/setup_dependencies litmus-test
## Clone the litmus repo, navigate to litmus root
git clone https://github.com/openebs/litmus.git
cd litmus
task_delimiter;
############################
## LITMUS PRECONDITIONING ##
############################
## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent
## TODO: Add logic to add ENV for run_instance_metadata/RunID
: << EOF
----------------------------------------------------------------------------------
| specAttribute | kind | percona | jenkins |
......@@ -75,79 +32,27 @@ task_delimiter;
----------------------------------------------------------------------------------
EOF
cp apps/percona/chaos/openebs_replica_network_delay/run_litmus_test.yml run_jenkins_jiva_rep_network_delay.yml
cp apps/percona/chaos/openebs_replica_network_delay/run_litmus_test.yml run_test.yml
sed -i -e 's/''name=percona''/''app=jenkins-app''/g' \
-e 's/value: app-percona-ns/value: app-jenkins-ns/g' \
-e 's/value: percona-mysql-claim/value: jenkins-claim/g' run_jenkins_jiva_rep_network_delay.yml
-e 's/value: percona-mysql-claim/value: jenkins-claim/g' run_test.yml
#################
## RUNNER MAIN ##
#################
echo "Running the litmus test.."
## Create the litmus job for percona app chaos test: openebs_replica_network_delay
jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.name==\"openebs-replica-network\")].metadata.name}'"
job_deploy_out=$(kubectl create -f run_jenkins_jiva_rep_network_delay.yml) && \
job_name=$(eval $jobNameCmd); retcode=$?
error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
## Obtain the litmus pod name
litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
litmus_pod=$(eval $litmusPodCmd); retcode=$?
error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state
## satisfies the "necessary" condition for test job completion
containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
## TODO: Consider cases where litmus pod is evicted
while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
sleep 1
done
while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
sleep 1
done
echo "Litmus test run Job has completed"