Skip to content

Commit 33bae3e

Browse files
leewesleyvleewesleyv
authored
Add job start and end time (#11, PR #30)
also restructures the tests job end time is still missing for Docker (as noted in the README) --- Co-authored-by: Wesley van Lee <[email protected]>
1 parent 5f91165 commit 33bae3e

File tree

14 files changed

+209
-155
lines changed

14 files changed

+209
-155
lines changed

‎.github/workflows/test-docker.yml

-37
This file was deleted.

‎.github/workflows/test-k8s.yml

-47
This file was deleted.

‎.github/workflows/test-manifest.yml

-61
This file was deleted.

‎.github/workflows/test.yml

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
name: Scapyd-k8s CI
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
8+
jobs:
9+
test-unit:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout
13+
uses: actions/checkout@v4
14+
15+
- name: Set up Python
16+
uses: actions/setup-python@v4
17+
with:
18+
python-version: 3.11
19+
cache: 'pip'
20+
21+
- name: Install dependencies
22+
run: |
23+
pip install -r requirements.txt
24+
pip install -r requirements-test.txt
25+
26+
- name: Run tests
27+
run: pytest -vv --color=yes scrapyd_k8s/tests/unit/
28+
29+
test-docker:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: Checkout
33+
uses: actions/checkout@v4
34+
35+
- name: Set up Python
36+
uses: actions/setup-python@v4
37+
with:
38+
python-version: 3.11
39+
cache: 'pip'
40+
41+
- name: Install dependencies
42+
run: |
43+
pip install -r requirements.txt
44+
pip install -r requirements-test.txt
45+
46+
- name: Pull example spider
47+
run: docker pull ghcr.io/q-m/scrapyd-k8s-spider-example
48+
49+
- name: Run scrapyd-k8s
50+
run: |
51+
cp scrapyd_k8s.sample-docker.conf scrapyd_k8s.conf
52+
python -m scrapyd_k8s &
53+
while ! nc -q 1 localhost 6800 </dev/null; do sleep 1; done
54+
curl http://localhost:6800/daemonstatus.json
55+
56+
- name: Run tests
57+
run: pytest -vv --color=yes scrapyd_k8s/tests/integration/
58+
59+
test-manifest:
60+
container:
61+
runs-on: ubuntu-latest
62+
steps:
63+
- name: Checkout
64+
uses: actions/checkout@v4
65+
66+
- name: Set up Python
67+
uses: actions/setup-python@v4
68+
with:
69+
python-version: 3.11
70+
cache: 'pip'
71+
72+
- name: Install dependencies
73+
run: |
74+
pip install -r requirements.txt
75+
pip install -r requirements-test.txt
76+
77+
- name: Set up Docker Buildx
78+
uses: docker/setup-buildx-action@v3
79+
80+
- name: Build container
81+
uses: docker/build-push-action@v5
82+
with:
83+
context: .
84+
push: false
85+
load: true
86+
tags: test:latest
87+
cache-from: type=gha
88+
cache-to: type=gha,mode=max
89+
90+
- name: Start minikube
91+
uses: medyagh/setup-minikube@master
92+
93+
- name: Deploy to minikube
94+
run: |
95+
minikube image load test:latest
96+
# already pull image so we don't have to wait for it later
97+
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
98+
# load manifest
99+
sed -i 's/\(imagePullPolicy:\s*\)\w\+/\1Never/' kubernetes.yaml
100+
sed -i 's/\(image:\s*\)ghcr\.io\/q-m\/scrapyd-k8s:/\1test:/' kubernetes.yaml
101+
sed -i 's/\(type:\s*\)ClusterIP/\1NodePort/' kubernetes.yaml
102+
kubectl create -f kubernetes.yaml
103+
# and wait for scrapyd-k8s to become ready
104+
kubectl wait --for=condition=Available deploy/scrapyd-k8s --timeout=60s
105+
curl --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
106+
107+
- name: Run tests
108+
run: |
109+
TEST_WITH_K8S=1 \
110+
TEST_BASE_URL=`minikube service scrapyd-k8s --url` \
111+
TEST_MAX_WAIT=60 \
112+
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
113+
pytest -vv --color=yes scrapyd_k8s/tests/integration/
114+
test-k8s:
115+
container:
116+
runs-on: ubuntu-latest
117+
steps:
118+
- name: Checkout
119+
uses: actions/checkout@v4
120+
121+
- name: Set up Python
122+
uses: actions/setup-python@v4
123+
with:
124+
python-version: 3.11
125+
cache: 'pip'
126+
127+
- name: Install dependencies
128+
run: |
129+
pip install -r requirements.txt
130+
pip install -r requirements-test.txt
131+
132+
- name: Start minikube
133+
uses: medyagh/setup-minikube@master
134+
135+
- name: Prepare Kubernetes environment
136+
run: |
137+
kubectl create secret generic example-env-secret --from-literal=FOO_1=bar
138+
kubectl create configmap example-env-configmap --from-literal=FOO_2=baz
139+
# already pull image so we don't have to wait for it later
140+
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
141+
142+
- name: Run scrapyd-k8s
143+
run: |
144+
cp scrapyd_k8s.sample-k8s.conf scrapyd_k8s.conf
145+
python -m scrapyd_k8s &
146+
while ! nc -q 1 localhost 6800 </dev/null; do sleep 1; done
147+
curl http://localhost:6800/daemonstatus.json
148+
149+
- name: Run tests
150+
run: |
151+
TEST_WITH_K8S=1 \
152+
TEST_MAX_WAIT=60 \
153+
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
154+
pytest -vv --color=yes scrapyd_k8s/tests/integration/

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ curl http://localhost:6800/listjobs.json
151151
{
152152
"finished":[],
153153
"pending":[],
154-
"running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running"}],
154+
"running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running", "start_time":"2012-09-12 10:14:03.594664", "end_time":null}],
155155
"status":"ok"
156156
}
157157
```
@@ -228,6 +228,7 @@ Lists spiders from the spider image's `org.scrapy.spiders` label.
228228
### `listjobs.json` ([➽](https://scrapyd.readthedocs.io/en/latest/api.html#listjobs-json))
229229
230230
Lists current jobs by looking at Docker containers or Kubernetes jobs.
231+
Note that `end_time` is not yet supported for Docker.
231232
232233
### ~~`delversion.json`~~ ([➽](https://scrapyd.readthedocs.io/en/latest/api.html#delversion-json))
233234

‎scrapyd_k8s/__init__.py

Whitespace-only changes.

‎scrapyd_k8s/launcher/docker.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import logging
44

55
import docker
6-
from ..utils import native_stringify_dict
6+
from ..utils import format_iso_date_string, native_stringify_dict
77

88
logger = logging.getLogger(__name__)
99

@@ -72,11 +72,14 @@ def enable_joblogs(self, config):
7272
logger.warning("Job logs are not supported when using the Docker launcher.")
7373

7474
def _parse_job(self, c):
75+
state = self._docker_to_scrapyd_status(c.status)
7576
return {
7677
'id': c.labels.get(self.LABEL_JOB_ID),
77-
'state': self._docker_to_scrapyd_status(c.status),
78+
'state': state,
7879
'project': c.labels.get(self.LABEL_PROJECT),
79-
'spider': c.labels.get(self.LABEL_SPIDER)
80+
'spider': c.labels.get(self.LABEL_SPIDER),
81+
'start_time': format_iso_date_string(c.attrs['State']['StartedAt']) if state in ['running', 'finished'] else None,
82+
'end_time': None, # Not available using Docker's API. Add to the job representation to keep it the same as K8s jobs listing.
8083
}
8184

8285
def _get_container(self, project_id, job_id):

‎scrapyd_k8s/launcher/k8s.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import kubernetes.stream
55
from signal import Signals
66

7-
from ..utils import native_stringify_dict
7+
from ..utils import format_datetime_object, native_stringify_dict
88
from scrapyd_k8s.joblogs import joblogs_init
99

1010
class K8s:
@@ -125,11 +125,14 @@ def enable_joblogs(self, config):
125125
joblogs_init(config)
126126

127127
def _parse_job(self, job):
128+
state = self._k8s_job_to_scrapyd_status(job)
128129
return {
129130
'id': job.metadata.labels.get(self.LABEL_JOB_ID),
130-
'state': self._k8s_job_to_scrapyd_status(job),
131+
'state': state,
131132
'project': job.metadata.labels.get(self.LABEL_PROJECT),
132-
'spider': job.metadata.labels.get(self.LABEL_SPIDER)
133+
'spider': job.metadata.labels.get(self.LABEL_SPIDER),
134+
'start_time': format_datetime_object(job.status.start_time) if state in ['running', 'finished'] else None,
135+
'end_time': format_datetime_object(job.status.completion_time) if job.status.completion_time and state == 'finished' else None,
133136
}
134137

135138
def _get_job(self, project, job_id):

‎scrapyd_k8s/tests/__init__.py

Whitespace-only changes.

‎scrapyd_k8s/tests/integration/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)