Skip to content

Commit 4cd36b2

Browse files
committed
Allow both configmap and secret for spider environment variables on Kubernetes
1 parent 7c69cf5 commit 4cd36b2

File tree

4 files changed

+17
-8
lines changed

4 files changed

+17
-8
lines changed

app.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ def api_schedule():
3333
_version = request.form.get('_version', 'latest') # TODO allow customizing latest tag
3434
# any other parameter is passed as spider argument
3535
args = { k: v for k, v in request.form.items() if k not in ('project', 'spider', 'setting', 'jobid', 'priority', '_version') }
36-
jobid = launcher.schedule(project['repository'], project_id, _version, spider, job_id, project.get('env_secret'), settings, args)
36+
env_config, env_secret = project.get('env_config'), project.get('env_secret')
37+
jobid = launcher.schedule(project['repository'], project_id, _version, spider, job_id, env_config, env_secret, settings, args)
3738
return { 'status': 'ok', 'jobid': job_id }
3839

3940
@app.post("/cancel.json")

scrapyd_k8s.sample-k8s.conf

+3-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ pull_secret = ghcr-registry
1717
# For each project, define a project section.
1818
# This contains a repository that points to the remote container repository.
1919
# An optional env_secret is the name of a secret with additional environment
20-
# variables to run the spiders with.
20+
# variables to run the spiders with; similarly env_config for a configmap.
2121
[project.example]
22-
env_secret = example-env
22+
env_secret = example-env-secret
23+
env_config = example-env-configmap
2324
repository = ghcr.io/example-org/example-spider

scrapyd_k8s/launcher/docker.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def listjobs(self, project=None):
2222
jobs = [self._parse_job(j) for j in jobs]
2323
return jobs
2424

25-
def schedule(self, repository, project, version, spider, job_id, env_source, settings, args):
25+
def schedule(self, repository, project, version, spider, job_id, env_config, env_secret, settings, args):
2626
_settings = [i for s in settings for i in ['-s', s]]
2727
_args = [i for a in args for i in ['-a', a]]
2828
env = {

scrapyd_k8s/launcher/k8s.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ def listjobs(self, project=None):
3636
jobs = [self._parse_job(j) for j in jobs.items]
3737
return jobs
3838

39-
def schedule(self, repository, project, version, spider, job_id, env_source, settings, args):
39+
def schedule(self, repository, project, version, spider, job_id, env_config, env_secret, settings, args):
4040
job_name = self._k8s_job_name(project, job_id)
4141
_settings = [i for s in settings for i in ['-s', s]]
4242
_args = [i for a in args for i in ['-a', a]]
@@ -50,14 +50,21 @@ def schedule(self, repository, project, version, spider, job_id, env_source, set
5050
self.LABEL_PROJECT: project,
5151
self.LABEL_SPIDER: spider,
5252
}
53+
env_from = []
54+
if env_config:
55+
env_from.append(kubernetes.client.V1EnvFromSource(
56+
config_map_ref=kubernetes.client.V1ConfigMapEnvSource(name=env_config, optional=False)
57+
))
58+
if env_secret:
59+
env_from.append(kubernetes.client.V1EnvFromSource(
60+
secret_ref=kubernetes.client.V1SecretEnvSource(name=env_secret, optional=False)
61+
))
5362
container = kubernetes.client.V1Container(
5463
name=job_name,
5564
image=repository + ':' + version,
5665
args=['scrapy', 'crawl', spider, *_args, *_settings],
5766
env=[kubernetes.client.V1EnvVar(k, v) for k, v in env.items()],
58-
env_from=[kubernetes.client.V1EnvFromSource(
59-
secret_ref=kubernetes.client.V1SecretEnvSource(name=env_source, optional=False)
60-
)] if env_source else None
67+
env_from=env_from
6168
)
6269
pod_template = kubernetes.client.V1PodTemplateSpec(
6370
metadata=kubernetes.client.V1ObjectMeta(name=job_name, labels=labels),

0 commit comments

Comments
 (0)