Unverified Commit cccba681 authored by Erik Wilson's avatar Erik Wilson Committed by GitHub
Browse files

Test script cleanup backport from RKE2 (#2650)



* Test script cleanup backport from RKE2
Signed-off-by: default avatarErik Wilson <Erik.E.Wilson@gmail.com>

* Update scripts/test
Co-authored-by: default avatarBrian Downs <brian.downs@gmail.com>
Co-authored-by: default avatarBrian Downs <brian.downs@gmail.com>
parent fadc5a80
......@@ -10,7 +10,12 @@ mkdir -p $artifacts
# ---
docker ps
# ---
. ./scripts/test-run-basics
echo "Did test-run-basics $?"
# ---
......@@ -18,7 +23,7 @@ mkdir -p $artifacts
early-exit "Skipping remaining tests, images not available for $ARCH."
E2E_OUTPUT=$artifacts test-run-sonobuoy
echo "Did test-run-sonobuoy $?"
# ---
[ "$DRONE_BUILD_EVENT" = 'tag' ] && \
......@@ -28,3 +33,5 @@ E2E_OUTPUT=$artifacts test-run-sonobuoy
test-run-sonobuoy mysql
test-run-sonobuoy postgres
exit 0
......@@ -64,7 +64,7 @@ export -f wait-for-nodes
pod-ready() {
kubectl get pods -n kube-system -o json \
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
| jq ".items[].status | select(.containerStatuses != null) | .containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready
......@@ -72,7 +72,7 @@ export -f pod-ready
wait-for-services() {
for service in $@; do
while [[ "$(pod-ready $service)" != 'true' ]]; do
while [[ "$(pod-ready $service | sort -u)" != 'true' ]]; do
echo "Waiting for service $service to be ready..." >&2
sleep 5
done
......@@ -122,13 +122,27 @@ dump-logs() {
echo "#---------------------------------"
echo "#- Begin: logs for run ($testID)"
echo
local server=$(cat $TEST_DIR/servers/1/metadata/name)
docker exec $server kubectl get pods -A -o wide >$TEST_DIR/logs/kubectl-get-pods.txt
docker exec $server kubectl get nodes -o wide >$TEST_DIR/logs/kubectl-get-nodes.txt
docker exec $server kubectl describe pods -A >$TEST_DIR/logs/kubectl-describe-pods.txt
for node in $TEST_DIR/*/*; do
[ -d "$node" ] || continue
local name=$(cat $node/metadata/name 2>/dev/null)
[ "$name" ] || continue
mkdir -p $node/logs
local hostname=$(docker exec $name hostname)
docker exec $server kubectl describe node/$hostname >$node/logs/kubectl-describe-node.txt
docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null
docker logs $name >$node/logs/system.log 2>&1
docker exec $name crictl pods >$node/logs/crictl-pods.txt
docker exec $name crictl ps -a >$node/logs/crictl-ps.txt
docker exec $name crictl ps -a -o json >$node/metadata/crictl-ps.json
for container in $(jq -r '.containers[].id' <$node/metadata/crictl-ps.json); do
local cname=$(jq -r '.containers[] | select(.id == "'$container'") | .metadata.name' <$node/metadata/crictl-ps.json)
docker exec $name crictl logs $container >$node/logs/$cname-$container.log 2>&1
done
for log in $node/logs/*.log; do
echo
echo "#- Tail: $log"
......@@ -137,6 +151,13 @@ dump-logs() {
echo
done
done
for txt in $TEST_DIR/logs/*.txt $TEST_DIR/*/*/logs/*.txt; do
echo
echo "#- Cat: $txt"
cat $txt
echo "#- Done: $txt"
echo
done
echo
echo "#- Finish: logs for run ($testID)"
echo "#---------------------------------"
......@@ -176,6 +197,7 @@ retrieve-sonobuoy-logs() {
if [ -n "$LOG_OUTPUT" ]; then
cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT")
fi
tail -11 $e2eLog
awk '/^Summarizing .* Failures?:$/,0' $e2eLog
return $code
}
......@@ -183,6 +205,23 @@ export -f retrieve-sonobuoy-logs
# ---
test-wait() {
wait $1 || echo "test wait exit code $?"
local delay=15
sleep $delay
while sonobuoy status | grep -q -E ' +e2e +running +'; do
sleep $delay
done
sleep $delay
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
return 0
fi
return 1
}
export -f test-wait
# ---
sonobuoy-test() {
if [ "$ARCH" = 'arm' ]; then
echo "Aborting sonobuoy tests, images not available for $ARCH"
......@@ -200,8 +239,8 @@ sonobuoy-test() {
local sonobuoyPID=$!
local code=0
time wait $sonobuoyPID || code=$?
time timeout --foreground 30m bash -c test-wait $sonobuoyPID || code=$?
echo "Sonobuoy finished with code $code"
retrieve-sonobuoy-logs
return $code
}
......@@ -211,7 +250,7 @@ export -f sonobuoy-test
test-cleanup() {
local code=$?
set +e +x
set +e -x
echo 'Cleaning up...'
trap - EXIT INT TERM
if [[ $code -ne 0 ]]; then
......@@ -220,13 +259,14 @@ test-cleanup() {
for name in $TEST_DIR/*/*/metadata/name; do
[ -f "$name" ] || continue
local container=$(cat $name)
docker rm -f -v $container >/dev/null 2>&1 &
echo "Removing container $container"
docker rm -f -v $container
done
if [ "$TEST_CLEANUP" = true ]; then
echo "Removing $TEST_DIR"
rm -rf $TEST_DIR >/dev/null 2>&1 &
echo "Removing test directory $TEST_DIR"
rm -rf $TEST_DIR
fi
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 &
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK
echo
echo -n "Test $(basename $TEST_DIR) "
if [ $code -eq 0 ]; then
......@@ -247,8 +287,8 @@ test-setup() {
mkdir -p $TEST_DIR/metadata
if [ "$LABEL" ]; then
exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
exec > >(awk -W interactive "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
2> >(awk -W interactive "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
echo "$LABEL" >$TEST_DIR/metadata/label
fi
......@@ -417,7 +457,7 @@ provision-cluster() {
for i in $(seq 1 $NUM_SERVERS); do
provision-server
timeout --foreground 30s bash -c "wait-for-kubeconfig $i"
timeout --foreground 120s bash -c "wait-for-kubeconfig $i"
done
export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml
......@@ -427,8 +467,8 @@ provision-cluster() {
done
fi
timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES"
timeout --foreground 4m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
timeout --foreground 8m bash -c "wait-for-services $WAIT_SERVICES"
run-function cluster-post-hook
......@@ -480,10 +520,12 @@ run-e2e-tests() {
label=PARALLEL \
logName=e2e-STATUS-${ARCH}-parallel.log \
e2e-test ${sonobuoyParallelArgs[@]}
echo "Exit code $? for parallel start"
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
}
export -f run-e2e-tests
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment