Skip to content

Commit 5f91165

Browse files
authored
Fix persistent logs check (#28, PR #35)
Fixes the check for logs already being present on container storage.
1 parent 7975a70 commit 5f91165

File tree

2 files changed

+13
-17
lines changed

2 files changed

+13
-17
lines changed

scrapyd_k8s/joblogs/log_handler_k8s.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ def watch_pods(self):
268268
elif pod.status.phase in ['Succeeded', 'Failed']:
269269
log_filename = self.pod_tmp_mapping.get(pod_name)
270270
if log_filename is not None and os.path.isfile(log_filename) and os.path.getsize(log_filename) > 0:
271-
if self.object_storage_provider.object_exists(log_filename):
271+
if self.object_storage_provider.object_exists(job_id):
272272
logger.info(f"Log file for job '{job_id}' already exists in storage.")
273273
else:
274274
self.object_storage_provider.upload_file(log_filename)

scrapyd_k8s/object_storage/libcloud_driver.py

+12-16
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
from libcloud.storage.types import (
88
ObjectError,
99
ContainerDoesNotExistError,
10-
ObjectDoesNotExistError,
1110
InvalidContainerNameError,
1211
)
1312
from libcloud.storage.providers import get_driver
@@ -148,38 +147,35 @@ def upload_file(self, local_path):
148147
except Exception as e:
149148
logger.exception(f"An unexpected error occurred while uploading '{object_name}': {e}")
150149

151-
def object_exists(self, local_path):
150+
def object_exists(self, prefix):
152151
"""
153-
Checks if an object exists in the object storage container.
152+
Checks if any object exists in the container that starts with the given prefix.
154153
155154
Parameters
156155
----------
157-
local_path : str
158-
The local file path corresponding to the object name.
156+
prefix : str
157+
The prefix to match object names against.
159158
160159
Returns
161160
-------
162161
bool
163-
True if the object exists, False otherwise.
162+
True if at least one object with the given prefix exists, False otherwise.
164163
165164
Logs
166165
----
167166
Logs information about the existence check or errors encountered.
168167
"""
169-
object_name = os.path.basename(local_path)
170168
try:
171-
self.driver.get_object(
172-
container_name=self._container_name,
173-
object_name=object_name
174-
)
175-
logger.debug(f"Object '{object_name}' exists in container '{self._container_name}'.")
176-
return True
177-
except ObjectDoesNotExistError:
178-
logger.debug(f"Object '{object_name}' does not exist in container '{self._container_name}'.")
169+
objects = self.driver.list_container_objects(container=self._container_name, prefix=prefix)
170+
if objects:
171+
logger.debug(f"At least one object with prefix '{prefix}' exists in container '{self._container_name}'.")
172+
return True
173+
else:
174+
logger.debug(f"No objects with prefix '{prefix}' found in container '{self._container_name}'.")
179175
except ContainerDoesNotExistError:
180176
logger.error(f"Container '{self._container_name}' does not exist in the cloud storage.")
181177
except InvalidContainerNameError:
182178
logger.error(f"Invalid container name '{self._container_name}'.")
183179
except Exception as e:
184-
logger.exception(f"An unexpected error occurred while checking for object '{object_name}': {e}")
180+
logger.exception(f"An unexpected error occurred while listing objects with prefix '{prefix}': {e}")
185181
return False

0 commit comments

Comments
 (0)