Skip to content

Commit 7a81737

Browse files
authored
added functionality to remove successfully ploaded files; additionaly on a check when old jobs are present on the cluster and the code checks that the files of those jobs already exist in the S3 storage, added a file removal if it is still in the volume (#44)
1 parent 249910d commit 7a81737

File tree

1 file changed

+6
-0
lines changed

1 file changed

+6
-0
lines changed

scrapyd_k8s/joblogs/log_handler_k8s.py

+6
Original file line numberDiff line numberDiff line change
@@ -285,8 +285,14 @@ def handle_events(self, event):
285285
if log_filename is not None and os.path.isfile(log_filename) and os.path.getsize(log_filename) > 0:
286286
if self.object_storage_provider.object_exists(job_id):
287287
logger.info(f"Log file for job '{job_id}' already exists in storage.")
288+
if os.path.exists(log_filename):
289+
os.remove(log_filename)
290+
logger.info(
291+
f"Removed local log file '{log_filename}' since it already exists in storage.")
288292
else:
289293
self.object_storage_provider.upload_file(log_filename)
294+
os.remove(log_filename)
295+
logger.info(f"Removed local log file '{log_filename}' after successful upload.")
290296
else:
291297
logger.info(f"Logfile not found for job '{job_id}'")
292298
else:

0 commit comments

Comments
 (0)