5
5
from flask_basicauth import BasicAuth
6
6
from natsort import natsort_keygen , ns
7
7
8
- # setup logging before anything else
9
8
from .config import Config
10
- from .logging import setup_logging
11
9
config = Config ()
12
- log_level = config .scrapyd ().get ('log_level' , 'INFO' )
13
- setup_logging (log_level )
14
10
15
11
app = Flask (__name__ )
16
- repository = (config .repository_cls ())(config )
17
- launcher = (config .launcher_cls ())(config )
18
- scrapyd_config = config .scrapyd ()
19
-
20
12
21
13
@app .get ("/" )
22
14
def home ():
@@ -28,9 +20,9 @@ def healthz():
28
20
29
21
@app .get ("/daemonstatus.json" )
30
22
def api_daemonstatus ():
31
- jobs = list (launcher .listjobs ())
23
+ jobs = list (config . launcher () .listjobs ())
32
24
return {
33
- "node_name" : config .scrapyd ().get ("node_name" , launcher .get_node_name ()),
25
+ "node_name" : config .scrapyd ().get ("node_name" , config . launcher () .get_node_name ()),
34
26
"status" : "ok" ,
35
27
"pending" : len ([j for j in jobs if j ['state' ] == 'pending' ]),
36
28
"running" : len ([j for j in jobs if j ['state' ] == 'running' ]),
@@ -55,7 +47,7 @@ def api_schedule():
55
47
# any other parameter is passed as spider argument
56
48
args = { k : v for k , v in request .form .items () if k not in ('project' , 'spider' , 'setting' , 'jobid' , 'priority' , '_version' ) }
57
49
env_config , env_secret = project .env_config (), project .env_secret ()
58
- jobid = launcher .schedule (project , _version , spider , job_id , settings , args )
50
+ jobid = config . launcher () .schedule (project , _version , spider , job_id , settings , args )
59
51
return { 'status' : 'ok' , 'jobid' : job_id }
60
52
61
53
@app .post ("/cancel.json" )
@@ -67,7 +59,7 @@ def api_cancel():
67
59
if not job_id :
68
60
return error ('job missing in form parameters' , status = 400 )
69
61
signal = request .form .get ('signal' , 'TERM' ) # TODO validate signal?
70
- prevstate = launcher .cancel (project_id , job_id , signal )
62
+ prevstate = config . launcher () .cancel (project_id , job_id , signal )
71
63
if not prevstate :
72
64
return error ('job not found' , status = 404 )
73
65
return { 'status' : 'ok' , 'prevstate' : prevstate }
@@ -84,7 +76,7 @@ def api_listversions():
84
76
project = config .project (project_id )
85
77
if not project :
86
78
return error ('project not found in configuration' , status = 404 )
87
- tags = repository .listtags (project .repository ())
79
+ tags = config . repository () .listtags (project .repository ())
88
80
tags = [t for t in tags if not t .startswith ('sha-' )]
89
81
tags .sort (key = natsort_keygen (alg = ns .NUMAFTER ))
90
82
return { 'status' : 'ok' , 'versions' : tags }
@@ -98,15 +90,15 @@ def api_listspiders():
98
90
if not project :
99
91
return error ('project not found in configuration' , status = 404 )
100
92
_version = request .args .get ('_version' , 'latest' ) # TODO allow customizing latest tag
101
- spiders = repository .listspiders (project .repository (), project_id , _version )
93
+ spiders = config . repository () .listspiders (project .repository (), project_id , _version )
102
94
if spiders is None :
103
95
return error ('project version not found in repository' , status = 404 )
104
96
return { 'status' : 'ok' , 'spiders' : spiders }
105
97
106
98
@app .get ("/listjobs.json" )
107
99
def api_listjobs ():
108
100
project_id = request .args .get ('project' )
109
- jobs = launcher .listjobs (project_id )
101
+ jobs = config . launcher () .listjobs (project_id )
110
102
pending = [j for j in jobs if j ['state' ] == 'pending' ]
111
103
running = [j for j in jobs if j ['state' ] == 'running' ]
112
104
finished = [j for j in jobs if j ['state' ] == 'finished' ]
@@ -133,21 +125,29 @@ def api_delproject():
133
125
def after_request (response : Response ):
134
126
if response .is_json :
135
127
data = response .json
136
- data ["node_name" ] = config .scrapyd ().get ("node_name" , launcher .get_node_name ())
128
+ data ["node_name" ] = config .scrapyd ().get ("node_name" , config . launcher () .get_node_name ())
137
129
response .data = jsonify (data ).data
138
130
return response
139
131
140
132
def error (msg , status = 200 ):
141
133
return { 'status' : 'error' , 'message' : msg }, status
142
134
143
135
def enable_authentication (app , config_username , config_password ):
144
- basic_auth = BasicAuth (app )
136
+
137
+ # workaround for https://github.com/jpvanhal/flask-basicauth/issues/11
138
+ class BasicAuthExceptHealthz (BasicAuth ):
139
+ def authenticate (self ):
140
+ return request .path == "/healthz" or super ().authenticate ()
141
+
142
+ basic_auth = BasicAuthExceptHealthz (app )
145
143
app .config ["BASIC_AUTH_USERNAME" ] = config_username
146
144
app .config ["BASIC_AUTH_PASSWORD" ] = config_password
147
145
app .config ["BASIC_AUTH_FORCE" ] = True
148
146
return basic_auth
149
147
150
148
def run ():
149
+ scrapyd_config = config .scrapyd ()
150
+
151
151
# where to listen
152
152
host = scrapyd_config .get ('bind_address' , '127.0.0.1' )
153
153
port = scrapyd_config .get ('http_port' , '6800' )
0 commit comments