kubernetes.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: scrapyd-k8s
  name: scrapyd-k8s
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: scrapyd-k8s
  template:
    metadata:
      labels:
        app.kubernetes.io/name: scrapyd-k8s
    spec:
      securityContext:
        fsGroup: 1000
      serviceAccountName: scrapyd-k8s
      containers:
        - image: ghcr.io/q-m/scrapyd-k8s:latest
          imagePullPolicy: IfNotPresent
          name: scrapyd-k8s
          ports:
            - containerPort: 6800
              name: http
              protocol: TCP
          env:
            - name: MY_POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: MY_DEPLOYMENT_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.labels['app.kubernetes.io/name']
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: http
          livenessProbe:
            failureThreshold: 30
            httpGet:
              path: /healthz
              port: http
          resources:
            limits:
              memory: 128Mi # TODO check
            requests:
              memory: 64Mi # TODO check
          volumeMounts:
            - name: scrapyd-k8s-config
              mountPath: /opt/app/scrapyd_k8s.conf
              readOnly: true
              subPath: scrapyd_k8s.conf
            #- name: joblogs
            #  mountPath: /data
            # Enable if your spider repository needs a pull secret
            # - name: scrapyd-k8s-pull-secret
            #   mountPath: /opt/app/.docker
            #   readOnly: true
      volumes:
        - configMap:
            name: scrapyd-k8s-config
          name: scrapyd-k8s-config
        #- name: joblogs
        #  persistentVolumeClaim:
        #    claimName: pv-claim
        # Enable if your spider repository needs a pull secret
        # - secret:
        #     secretName: pull-secret
        #     items:
        #       - key: .dockerconfigjson
        #         path: config.json
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: scrapyd-k8s-config
  labels:
    app.kubernetes.io/name: scrapyd-k8s
data:
  scrapyd_k8s.conf: |-
    [scrapyd]
    bind_address = 0.0.0.0
    http_port    = 6800

    repository   = scrapyd_k8s.repository.Remote
    launcher     = scrapyd_k8s.launcher.K8s
    
    namespace    = default
    
    max_proc     = 2

    # This is an example spider that should work out of the box.
    # Adapt the spider config to your use-case.
    [project.example]
    env_secret   = spider-example-env
    env_config   = spider-example-env
    repository   = ghcr.io/q-m/scrapyd-k8s-spider-example

    # It is strongly recomended to set resource requests and limits on production.
    # They can be overridden on the project and spider level.
    [default.resources]
    requests_cpu = 0.2
    requests_memory = 0.2G
    limits_cpu = 0.8
    limits_memory = 0.5G
    
    [joblogs]
    logs_dir = /data/joblogs
---
apiVersion: v1
kind: Secret
metadata:
  name: spider-example-env
  labels:
    app.kubernetes.io/name: spider-example
stringData:
  FOO_API_KEY: "1234567890abcdef"
#---
#apiVersion: v1
#kind: PersistentVolumeClaim
#metadata:
#  name: pv-claim
#spec:
#  accessModes:
#    - ReadWriteOnce
#  resources:
#    requests:
#      storage: 5Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: spider-example-env
  labels:
    app.kubernetes.io/name: spider-example
data:
  BAR_VALUE: "baz"
---
apiVersion: v1
kind: Service
metadata:
  name: scrapyd-k8s
  labels:
    app.kubernetes.io/name: scrapyd-k8s
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 6800
      protocol: TCP
      targetPort: http
  selector:
    app.kubernetes.io/name: scrapyd-k8s
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: scrapyd-k8s
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: scrapyd-k8s
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get"]
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["get", "list", "create", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: scrapyd-k8s
subjects:
  - kind: ServiceAccount
    name: scrapyd-k8s
roleRef:
  kind: Role
  name: scrapyd-k8s
  apiGroup: rbac.authorization.k8s.io