Skip to content

Commit 655a763

Browse files
committed
Initial Kubernetes instructions
1 parent 6040202 commit 655a763

File tree

3 files changed

+139
-1
lines changed

3 files changed

+139
-1
lines changed

README.md

+4
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ runs as root. It may be useful to try things out.
4848

4949
### Kubernetes
5050

51+
1. Create the spider namespace: `kubectl create namespace scrapyd`
52+
2. Adapt the spider configuration in [`kubernetes.yaml`](./kubernetes.yaml) (`scrapyd_k8s.conf` in configmap)
53+
3. Create the resources: `kubectl create -f kubernetes.yaml`
54+
5155
### Local
5256

5357
For development, or just a quick start, you can also run this application locally.

kubernetes.yaml

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: scrapyd-k8s
6+
name: scrapyd-k8s
7+
spec:
8+
selector:
9+
matchLabels:
10+
app.kubernetes.io/name: scrapyd-k8s
11+
template:
12+
metadata:
13+
labels:
14+
app.kubernetes.io/name: scrapyd-k8s
15+
spec:
16+
serviceAccountName: scrapyd-k8s
17+
containers:
18+
- image: ghcr.io/q-m/scrapyd-k8s:main # TODO latest when published
19+
imagePullPolicy: IfNotPresent
20+
name: scrapyd-k8s
21+
ports:
22+
- containerPort: 6800
23+
name: http
24+
protocol: TCP
25+
readinessProbe:
26+
failureThreshold: 3
27+
httpGet:
28+
path: /
29+
port: http
30+
livenessProbe:
31+
failureThreshold: 30
32+
httpGet:
33+
path: /
34+
port: http
35+
resources:
36+
limits:
37+
memory: 128Mi # TODO check
38+
requests:
39+
memory: 64Mi # TODO check
40+
volumeMounts:
41+
- mountPath: /opt/app/scrapyd_k8s.conf
42+
name: scrapyd-k8s-config
43+
readOnly: true
44+
subPath: scrapyd_k8s.conf
45+
volumes:
46+
- configMap:
47+
name: scrapyd-k8s-config
48+
name: scrapyd-k8s-config
49+
---
50+
apiVersion: v1
51+
kind: ConfigMap
52+
metadata:
53+
name: scrapyd-k8s-config
54+
labels:
55+
app.kubernetes.io/name: scrapyd-k8s
56+
data:
57+
scrapyd_k8s.conf: |-
58+
[scrapyd]
59+
bind_address = 0.0.0.0
60+
http_port = 6800
61+
62+
repository = scrapyd_k8s.repository.Remote
63+
launcher = scrapyd_k8s.launcher.K8s
64+
65+
namespace = scrapyd
66+
67+
# adapt the spider config to your use-case
68+
[project.example]
69+
env_secret = spider-example-env
70+
repository = ghcr.io/example-org/example-spider
71+
---
72+
apiVersion: v1
73+
kind: Secret
74+
metadata:
75+
name: spider-example-env
76+
labels:
77+
app.kubernetes.io/name: spider-example
78+
stringData:
79+
FOO_API_KEY: 1234567890abcdef
80+
---
81+
apiVersion: v1
82+
kind: Service
83+
metadata:
84+
name: scrapyd-k8s
85+
labels:
86+
app.kubernetes.io/name: scrapyd-k8s
87+
spec:
88+
type: ClusterIP
89+
ports:
90+
- name: http
91+
port: 6800
92+
protocol: TCP
93+
targetPort: http
94+
selector:
95+
app.kubernetes.io/name: scrapyd-k8s
96+
---
97+
apiVersion: v1
98+
kind: ServiceAccount
99+
metadata:
100+
name: scrapyd-k8s
101+
namespace: scrapyd
102+
---
103+
apiVersion: rbac.authorization.k8s.io/v1
104+
kind: Role
105+
metadata:
106+
name: scrapyd-k8s
107+
namespace: scrapyd
108+
rules:
109+
- apiGroups: [""]
110+
resources: ["pods"]
111+
verbs: ["get", "list"]
112+
- apiGroups: [""]
113+
resources: ["pods/exec"]
114+
verbs: ["create"]
115+
- apiGroups: ["batch"]
116+
resources: ["jobs"]
117+
verbs: ["get", "list", "create", "delete"]
118+
---
119+
apiVersion: rbac.authorization.k8s.io/v1
120+
kind: RoleBinding
121+
metadata:
122+
name: scrapyd-k8s
123+
namespace: scrapyd
124+
subjects:
125+
- kind: ServiceAccount
126+
name: scrapyd-k8s
127+
roleRef:
128+
kind: Role
129+
name: scrapyd-k8s
130+
apiGroup: rbac.authorization.k8s.io

scrapyd_k8s/launcher/k8s.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@ class K8s:
2222
def __init__(self, config):
2323
self._namespace = config.scrapyd().get('namespace', 'default')
2424
self._pull_secret = config.scrapyd().get('pull_secret')
25-
kubernetes.config.load_kube_config() # TODO figure out where to put this
25+
# TODO figure out where to put Kubernetes initialisation
26+
try:
27+
kubernetes.config.load_incluster_config()
28+
except kubernetes.config.config_exception.ConfigException:
29+
kubernetes.config.load_kube_config()
2630
self._k8s = kubernetes.client.CoreV1Api()
2731
self._k8s_batch = kubernetes.client.BatchV1Api()
2832

0 commit comments

Comments
 (0)