Allow both configmap and secret for spider environment variables on Kubernetes

Author: wvengen

app.py

@@ -33,7 +33,8 @@ def api_schedule():
     _version = request.form.get('_version', 'latest') # TODO allow customizing latest tag
     # any other parameter is passed as spider argument
     args = { k: v for k, v in request.form.items() if k not in ('project', 'spider', 'setting', 'jobid', 'priority', '_version') }
-    jobid = launcher.schedule(project['repository'], project_id, _version, spider, job_id, project.get('env_secret'), settings, args)
+    env_config, env_secret = project.get('env_config'), project.get('env_secret')
+    jobid = launcher.schedule(project['repository'], project_id, _version, spider, job_id, env_config, env_secret, settings, args)
     return { 'status': 'ok', 'jobid': job_id }
 
 @app.post("/cancel.json")

scrapyd_k8s.sample-k8s.conf

@@ -17,7 +17,8 @@ pull_secret  = ghcr-registry
 # For each project, define a project section.
 # This contains a repository that points to the remote container repository.
 # An optional env_secret is the name of a secret with additional environment
-# variables to run the spiders with.
+# variables to run the spiders with; similarly env_config for a configmap.
 [project.example]
-env_secret   = example-env
+env_secret   = example-env-secret
+env_config   = example-env-configmap
 repository   = ghcr.io/example-org/example-spider

scrapyd_k8s/launcher/docker.py

@@ -22,7 +22,7 @@ def listjobs(self, project=None):
         jobs = [self._parse_job(j) for j in jobs]
         return jobs
 
-    def schedule(self, repository, project, version, spider, job_id, env_source, settings, args):
+    def schedule(self, repository, project, version, spider, job_id, env_config, env_secret, settings, args):
         _settings = [i for s in settings for i in ['-s', s]]
         _args = [i for a in args for i in ['-a', a]]
         env = {

scrapyd_k8s/launcher/k8s.py

@@ -36,7 +36,7 @@ def listjobs(self, project=None):
         jobs = [self._parse_job(j) for j in jobs.items]
         return jobs
 
-    def schedule(self, repository, project, version, spider, job_id, env_source, settings, args):
+    def schedule(self, repository, project, version, spider, job_id, env_config, env_secret, settings, args):
         job_name = self._k8s_job_name(project, job_id)
         _settings = [i for s in settings for i in ['-s', s]]
         _args = [i for a in args for i in ['-a', a]]
@@ -50,14 +50,21 @@ def schedule(self, repository, project, version, spider, job_id, env_source, set
             self.LABEL_PROJECT: project,
             self.LABEL_SPIDER: spider,
         }
+        env_from = []
+        if env_config:
+            env_from.append(kubernetes.client.V1EnvFromSource(
+                config_map_ref=kubernetes.client.V1ConfigMapEnvSource(name=env_config, optional=False)
+            ))
+        if env_secret:
+            env_from.append(kubernetes.client.V1EnvFromSource(
+                secret_ref=kubernetes.client.V1SecretEnvSource(name=env_secret, optional=False)
+            ))
         container = kubernetes.client.V1Container(
             name=job_name,
             image=repository + ':' + version,
             args=['scrapy', 'crawl', spider, *_args, *_settings],
             env=[kubernetes.client.V1EnvVar(k, v) for k, v in env.items()],
-            env_from=[kubernetes.client.V1EnvFromSource(
-                secret_ref=kubernetes.client.V1SecretEnvSource(name=env_source, optional=False)
-            )] if env_source else None
+            env_from=env_from
         )
         pod_template = kubernetes.client.V1PodTemplateSpec(
             metadata=kubernetes.client.V1ObjectMeta(name=job_name, labels=labels),