K8s API connection tweaks, logging configuration (PR #39)

* remove limited reconnection attempts; cap backoff time with 15 min; configure logs in a separate file logging_config.py that is imported to the __main__.py to trigger logs configuration before other modules inheret it; add debug logs to the watcher to better monitor each step

* refactor code to make config import and logging configuration separate modules; implement logging to be configured globally but keep separate logger for each module; add level for logging to be configurable in the config file

---------

Co-authored-by: wvengen <[email protected]>

Author: vlerkin

CONFIG.md

@@ -12,6 +12,7 @@ stick to [scrapyd's configuration](https://scrapyd.readthedocs.io/en/latest/conf
 * `launcher`     - Python class for managing jobs on the cluster, defaults to `scrapyd_k8s.launcher.K8s`
 * `username`     - Set this and `password` to enable basic authentication ([➽](https://scrapyd.readthedocs.io/en/latest/config.html#username))
 * `password`     - Set this and `username` to enable basic authentication ([➽](https://scrapyd.readthedocs.io/en/latest/config.html#password))
+* `log_level`    - Log level, defaults to `INFO`
 
 The Docker and Kubernetes launchers have their own additional options.
 

scrapyd_k8s/__main__.py

@@ -1,16 +1,4 @@
-import logging
-import sys
 from .api import run
 
-def setup_logging():
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s %(name)s [%(levelname)s]: %(message)s',
-        handlers=[
-            logging.StreamHandler(sys.stdout)
-        ]
-    )
-
 if __name__ == "__main__":
-    setup_logging()
     run()

scrapyd_k8s/api.py

@@ -1,19 +1,22 @@
 #!/usr/bin/env python3
-import uuid
 import logging
-
+import uuid
 from flask import Flask, request, Response, jsonify
 from flask_basicauth import BasicAuth
 from natsort import natsort_keygen, ns
 
+# setup logging before anything else
 from .config import Config
+from .logging import setup_logging
+config = Config()
+log_level = config.scrapyd().get('log_level', 'INFO')
+setup_logging(log_level)
 
 app = Flask(__name__)
-config = Config()
 repository = (config.repository_cls())(config)
 launcher = (config.launcher_cls())(config)
 scrapyd_config = config.scrapyd()
-logger = logging.getLogger(__name__)
+
 
 @app.get("/")
 def home():

scrapyd_k8s/joblogs/log_handler_k8s.py

@@ -226,7 +226,6 @@ def handle_events(self, event):
         -------
         None
         """
-        self.object_storage_provider = LibcloudObjectStorage(self.config)
         try:
 
             pod = event['object']

scrapyd_k8s/k8s_resource_watcher.py

@@ -1,5 +1,5 @@
-import threading
 import logging
+import threading
 import time
 from kubernetes import client, watch
 from typing import Callable, List
@@ -29,7 +29,6 @@ def __init__(self, namespace, config):
             Kubernetes namespace to watch pods in.
         """
         self.namespace = namespace
-        self.reconnection_attempts = int(config.scrapyd().get('reconnection_attempts', 5))
         self.backoff_time = int(config.scrapyd().get('backoff_time', 5))
         self.backoff_coefficient = int(config.scrapyd().get('backoff_coefficient', 2))
         self.subscribers: List[Callable] = []
@@ -90,8 +89,7 @@ def watch_pods(self):
 
         logger.info(f"Started watching pods in namespace '{self.namespace}'.")
         backoff_time = self.backoff_time
-        reconnection_attempts = self.reconnection_attempts
-        while not self._stop_event.is_set() and reconnection_attempts > 0:
+        while not self._stop_event.is_set():
             try:
                 kwargs = {
                     'namespace': self.namespace,
@@ -101,9 +99,10 @@ def watch_pods(self):
                     kwargs['resource_version'] = resource_version
                 first_event = True
                 for event in w.stream(v1.list_namespaced_pod, **kwargs):
+                    logging.debug("Connected to the k8s API, received an event from k8s API")
                     if first_event:
                         # Reset reconnection attempts and backoff time upon successful reconnection
-                        reconnection_attempts = self.reconnection_attempts
+                        logging.debug("This is the first event in the stream in the established connection, setting reconnection attempts to default")
                         backoff_time = self.backoff_time
                         first_event = False  # Ensure this only happens once per connection
                     pod_name = event['object'].metadata.name
@@ -114,32 +113,29 @@ def watch_pods(self):
             except (urllib3.exceptions.ProtocolError,
                     urllib3.exceptions.ReadTimeoutError,
                     urllib3.exceptions.ConnectionError) as e:
-                reconnection_attempts -= 1
                 logger.exception(f"Encountered network error: {e}")
                 logger.info(f"Retrying to watch pods after {backoff_time} seconds...")
                 time.sleep(backoff_time)
-                backoff_time *= self.backoff_coefficient
+                backoff_time = min(backoff_time*self.backoff_coefficient, 900)
             except client.ApiException as e:
                 # Resource version is too old and cannot be accessed anymore
                 if e.status == 410:
                     logger.error("Received 410 Gone error, resetting resource_version and restarting watch.")
                     resource_version = None
                     continue
                 else:
-                    reconnection_attempts -= 1
                     logger.exception(f"Encountered ApiException: {e}")
                     logger.info(f"Retrying to watch pods after {backoff_time} seconds...")
                     time.sleep(backoff_time)
-                    backoff_time *= self.backoff_coefficient
+                    backoff_time = min(backoff_time*self.backoff_coefficient, 900)
             except StopIteration:
                 logger.info("Watch stream ended, restarting watch.")
                 continue
             except Exception as e:
-                reconnection_attempts -= 1
                 logger.exception(f"Watcher encountered exception: {e}")
                 logger.info(f"Retrying to watch pods after {backoff_time} seconds...")
                 time.sleep(backoff_time)
-                backoff_time *= self.backoff_coefficient
+                backoff_time = min(backoff_time*self.backoff_coefficient, 900)
 
 
     def stop(self):

scrapyd_k8s/launcher/docker.py

@@ -1,6 +1,6 @@
+import logging
 import re
 import socket
-import logging
 
 import docker
 from ..utils import format_iso_date_string, native_stringify_dict

scrapyd_k8s/logging.py

@@ -0,0 +1,13 @@
+import logging
+import sys
+
+def setup_logging(log_level):
+    level_name = str(log_level).upper()
+    numeric_level = logging.getLevelName(level_name)
+    if not isinstance(numeric_level, int):
+        raise ValueError(f"Invalid log_level '{log_level}'.")
+    logging.basicConfig(
+        level=numeric_level,
+        format='%(asctime)s %(name)s [%(levelname)s]: %(message)s',
+        handlers=[logging.StreamHandler(sys.stdout)]
+    )

scrapyd_k8s/object_storage/libcloud_driver.py

@@ -1,8 +1,6 @@
 import os
-import logging
 import re
-
-logger = logging.getLogger(__name__)
+import logging
 
 from libcloud.storage.types import (
     ObjectError,
@@ -11,6 +9,8 @@
 )
 from libcloud.storage.providers import get_driver
 
+logger = logging.getLogger(__name__)
+
 class LibcloudObjectStorage:
     """
     A class to interact with cloud object storage using Apache Libcloud.