Initial Kubernetes instructions

Author: wvengen

README.md

@@ -48,6 +48,10 @@ runs as root. It may be useful to try things out.
 
 ### Kubernetes
 
+1. Create the spider namespace: `kubectl create namespace scrapyd`
+2. Adapt the spider configuration in [`kubernetes.yaml`](./kubernetes.yaml) (`scrapyd_k8s.conf` in configmap)
+3. Create the resources: `kubectl create -f kubernetes.yaml`
+
 ### Local
 
 For development, or just a quick start, you can also run this application locally.

kubernetes.yaml

@@ -0,0 +1,130 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: scrapyd-k8s
+  name: scrapyd-k8s
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: scrapyd-k8s
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: scrapyd-k8s
+    spec:
+      serviceAccountName: scrapyd-k8s
+      containers:
+        - image: ghcr.io/q-m/scrapyd-k8s:main # TODO latest when published
+          imagePullPolicy: IfNotPresent
+          name: scrapyd-k8s
+          ports:
+            - containerPort: 6800
+              name: http
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /
+              port: http
+          livenessProbe:
+            failureThreshold: 30
+            httpGet:
+              path: /
+              port: http
+          resources:
+            limits:
+              memory: 128Mi # TODO check
+            requests:
+              memory: 64Mi # TODO check
+          volumeMounts:
+            - mountPath: /opt/app/scrapyd_k8s.conf
+              name: scrapyd-k8s-config
+              readOnly: true
+              subPath: scrapyd_k8s.conf
+      volumes:
+        - configMap:
+            name: scrapyd-k8s-config
+          name: scrapyd-k8s-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scrapyd-k8s-config
+  labels:
+    app.kubernetes.io/name: scrapyd-k8s
+data:
+  scrapyd_k8s.conf: |-
+    [scrapyd]
+    bind_address = 0.0.0.0
+    http_port    = 6800
+
+    repository   = scrapyd_k8s.repository.Remote
+    launcher     = scrapyd_k8s.launcher.K8s
+
+    namespace    = scrapyd
+
+    # adapt the spider config to your use-case
+    [project.example]
+    env_secret   = spider-example-env
+    repository   = ghcr.io/example-org/example-spider
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: spider-example-env
+  labels:
+    app.kubernetes.io/name: spider-example
+stringData:
+  FOO_API_KEY: 1234567890abcdef
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: scrapyd-k8s
+  labels:
+    app.kubernetes.io/name: scrapyd-k8s
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: 6800
+      protocol: TCP
+      targetPort: http
+  selector:
+    app.kubernetes.io/name: scrapyd-k8s
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scrapyd-k8s
+  namespace: scrapyd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: scrapyd-k8s
+  namespace: scrapyd
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: scrapyd-k8s
+  namespace: scrapyd
+subjects:
+  - kind: ServiceAccount
+    name: scrapyd-k8s
+roleRef:
+  kind: Role
+  name: scrapyd-k8s
+  apiGroup: rbac.authorization.k8s.io

scrapyd_k8s/launcher/k8s.py

@@ -22,7 +22,11 @@ class K8s:
     def __init__(self, config):
         self._namespace = config.scrapyd().get('namespace', 'default')
         self._pull_secret = config.scrapyd().get('pull_secret')
-        kubernetes.config.load_kube_config() # TODO figure out where to put this
+        # TODO figure out where to put Kubernetes initialisation
+        try:
+            kubernetes.config.load_incluster_config()
+        except kubernetes.config.config_exception.ConfigException:
+            kubernetes.config.load_kube_config()
         self._k8s = kubernetes.client.CoreV1Api()
         self._k8s_batch = kubernetes.client.BatchV1Api()