Skip to content
Scrapyd-k8s CI

6 scheduling #87

Sign in to view logs

6 scheduling

6 scheduling #87

Workflow file for this run

.github/workflows/test.yml at 85d4294
name: Scrapyd-k8s CI
on:
push:
branches:
- main
pull_request:
jobs:
test-unit:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Run tests
run: pytest -vv --color=yes scrapyd_k8s/tests/unit/
test-docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Pull example spider
run: docker pull ghcr.io/q-m/scrapyd-k8s-spider-example
- name: Run tests
run: |
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo; echo "# $test"
# run scrapyd-k8s with test-specific configuration file
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
python -m scrapyd_k8s -c scrapyd_k8s.sample-docker.conf -c "$cfg" &
# wait for scrapyd-k8s to become ready
curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
# run test
pytest -vv --color=yes "$test"
# stop scrapyd-k8s again
kill %1; wait %1 || true
done
test-manifest:
container:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build container
uses: docker/build-push-action@v5
with:
context: .
push: false
load: true
tags: test:latest
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Start minikube
uses: medyagh/setup-minikube@master
- name: Deploy to minikube
run: |
minikube image load test:latest
# already pull image so we don't have to wait for it later
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
# load manifest
sed -i 's/\(imagePullPolicy:\s*\)\w\+/\1Never/' kubernetes.yaml
sed -i 's/\(image:\s*\)ghcr\.io\/q-m\/scrapyd-k8s:/\1test:/' kubernetes.yaml
sed -i 's/\(type:\s*\)ClusterIP/\1NodePort/' kubernetes.yaml
kubectl create -f kubernetes.yaml
# don't start deployment just yet, as we want to run it with test-specific configuration
kubectl scale --replicas=0 deploy/scrapyd-k8s
# add second configuration file for test-specific configuration
kubectl patch deploy scrapyd-k8s --type=json -p='[
{
"op": "add",
"path": "/spec/template/spec/volumes/-",
"value": { "configMap": { "name": "scrapyd-k8s-testcfg" }, "name": "scrapyd-k8s-testcfg" }
},
{
"op": "add",
"path": "/spec/template/spec/containers/0/volumeMounts/-",
"value": { "name": "scrapyd-k8s-testcfg", "mountPath": "/opt/app/scrapyd_k8s.test.conf", "readOnly": true, "subPath": "scrapyd_k8s.test.conf" }
},
{
"op": "replace",
"path": "/spec/template/spec/containers/0/command",
"value": ["python3", "-m", "scrapyd_k8s", "-c", "scrapyd_k8s.conf", "-c", "scrapyd_k8s.test.conf"]
}
]'
- name: Run tests
run: |
# setup for in-cluster k8s
# for each integration test file
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo; echo "# $test"
# run scrapyd-k8s with test-specific configuration file
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
kubectl create cm scrapyd-k8s-testcfg --from-file=scrapyd_k8s.test.conf="$cfg"
kubectl scale --replicas=1 deploy/scrapyd-k8s
# wait for scrapyd-k8s to become ready
kubectl wait --for=condition=Available deploy/scrapyd-k8s --timeout=60s
curl -s --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
# run test
TEST_WITH_K8S=1 \
TEST_BASE_URL=`minikube service scrapyd-k8s --url` \
TEST_MAX_WAIT=60 \
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
pytest -vv --color=yes "$test"
# delete al jobs to start with a clean slate next time
kubectl delete job --all
# stop scrapyd-k8s and delete test-specific configmap
kubectl scale --replicas=0 deploy/scrapyd-k8s
kubectl wait --for=delete pod -l app.kubernetes.io/name=scrapyd-k8s --timeout=90s
kubectl delete cm scrapyd-k8s-testcfg --wait
done
test-k8s:
container:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Start minikube
uses: medyagh/setup-minikube@master
- name: Prepare Kubernetes environment
run: |
kubectl create secret generic example-env-secret --from-literal=FOO_1=bar
kubectl create configmap example-env-configmap --from-literal=FOO_2=baz
# already pull image so we don't have to wait for it later
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
- name: Run tests
run: |
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo "# $test"
# run scrapyd-k8s with test-specific configuration file
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
[ -e "$cfg" ] || cfg=/dev/null
python -m scrapyd_k8s -c scrapyd_k8s.sample-k8s.conf -c "$cfg" &
# wait for scrapyd-k8s to become ready
curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
# run test
TEST_WITH_K8S=1 \
TEST_MAX_WAIT=60 \
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
pytest -vv --color=yes "$test"
# stop scrapyd-k8s again
kill %1; wait %1 || true
done