Try to fix CI tests

Author: wvengen

.github/workflows/test.yml

@@ -54,11 +54,11 @@ jobs:
             cfg=`echo "$test" | sed 's/\.py$/.conf/'`
             python -m scrapyd_k8s -c scrapyd_k8s.sample-docker.conf -c "$cfg" &
             # wait for scrapyd-k8s to become ready
-            sleep 1; curl --retry 20 --retry-delay 2 --retry-all-errors http://localhost:6800/daemonstatus.json
+            curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
             # run test
             pytest -vv --color=yes "$test"
             # stop scrapyd-k8s again
-            kill %1
+            kill %1; wait %1 || true
           done
 
   test-manifest:
@@ -135,19 +135,28 @@ jobs:
             # run scrapyd-k8s with test-specific configuration file
             cfg=`echo "$test" | sed 's/\.py$/.conf/'`
             kubectl create cm scrapyd-k8s-testcfg --from-file=scrapyd_k8s.test.conf="$cfg"
+            echo "** before scaleup"
             kubectl scale --replicas=1 deploy/scrapyd-k8s
+            sleep 1; echo "** after scaleup 1s"; kubectl describe pod # debug
+            sleep 4; echo "** after scaleup 5s"; kubectl describe pod # debug
             # wait for scrapyd-k8s to become ready
             kubectl wait --for=condition=Available deploy/scrapyd-k8s --timeout=60s
-            curl --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
+            echo "** after scaleup wait"
+            curl -s --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
             # run test
             TEST_WITH_K8S=1 \
               TEST_BASE_URL=`minikube service scrapyd-k8s --url` \
               TEST_MAX_WAIT=60 \
               TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
               pytest -vv --color=yes "$test"
             # stop scrapyd-k8s and delete test-specific configmap
+            echo "** before scaledown"
             kubectl scale --replicas=0 deploy/scrapyd-k8s
-            kubectl delete cm scrapyd-k8s-testcfg
+            sleep 1; echo "** after scaledown 1s"; kubectl describe pod # debug
+            sleep 4; echo "** after scaledown 5s"; kubectl describe pod # debug
+            kubectl wait --for=delete pod -l app.kubernetes.io/name=scrapyd-k8s --timeout=90s
+            echo "** after scaledown wait"
+            kubectl delete cm scrapyd-k8s-testcfg --wait
           done
 
   test-k8s:
@@ -181,18 +190,18 @@ jobs:
       - name: Run tests
         run: |
           for test in scrapyd_k8s/tests/integration/test_*.py; do
-            echo; echo "# $test"
+            echo "# $test"
             # run scrapyd-k8s with test-specific configuration file
             cfg=`echo "$test" | sed 's/\.py$/.conf/'`
             [ -e "$cfg" ] || cfg=/dev/null
             python -m scrapyd_k8s -c scrapyd_k8s.sample-k8s.conf -c "$cfg" &
             # wait for scrapyd-k8s to become ready
-            sleep 1; curl --retry 20 --retry-delay 2 --retry-all-errors http://localhost:6800/daemonstatus.json
+            curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
             # run test
             TEST_WITH_K8S=1 \
               TEST_MAX_WAIT=60 \
               TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
               pytest -vv --color=yes "$test"
             # stop scrapyd-k8s again
-            kill %1
+            kill %1; wait %1 || true
           done