Skip to content

Commit 6aee8e9

Browse files
committed
refactor: switch to loguru, add correlation ID
1 parent 3c1d3b9 commit 6aee8e9

22 files changed

+125
-53
lines changed

.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
**/.toolstarget
99
**/.vs
1010
**/.vscode
11+
**/.venv
1112
**/*.*proj.user
1213
**/*.dbmdl
1314
**/*.jfm

nazurin/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
from nazurin import config
2+
from nazurin.utils.logging import configure_logging
23

34
from .bot import NazurinBot
45
from .dispatcher import NazurinDispatcher
56

7+
configure_logging()
8+
69
bot = NazurinBot(token=config.TOKEN, proxy=config.PROXY)
710
dp = NazurinDispatcher(bot)
811

nazurin/__main__.py

+3-6
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import os
44
import shutil
55
import traceback
6-
from html import escape
76

87
from aiogram.dispatcher.filters import IDFilter
98
from aiogram.types import ChatActions, Message, Update
@@ -14,6 +13,7 @@
1413
from nazurin.utils import logger
1514
from nazurin.utils.decorators import Cache, chat_action
1615
from nazurin.utils.exceptions import NazurinError
16+
from nazurin.utils.helpers import format_error
1717

1818
@dp.message_handler(commands=['start', 'help'])
1919
@chat_action(ChatActions.TYPING)
@@ -67,12 +67,9 @@ async def on_error(update: Update, exception: Exception):
6767
traceback.print_exc()
6868
await update.message.reply('Error: Timeout, please try again.')
6969
except Exception as error: # pylint: disable=broad-except
70-
logger.error('Update %s caused %s: %s', update, type(error), error)
71-
traceback.print_exc()
70+
logger.exception('Update {} caused {}: {}', update, type(error), error)
7271
if not isinstance(error, TelegramAPIError):
73-
error_type = type(error).__name__
74-
error_msg = escape(str(error), quote=False)
75-
await update.message.reply(f'Error: ({error_type}) {error_msg}')
72+
await update.message.reply(f'Error: {format_error(error)}')
7673

7774
return True
7875

nazurin/bot.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ async def update_collection(self,
106106
result = self.sites.match(urls)
107107
if not result:
108108
raise NazurinError('No source matched')
109-
logger.info('Collection update: site=%s, match=%s', result['site'],
109+
logger.info('Collection update: site={}, match={}', result['site'],
110110
result['match'].groups())
111111

112112
illust = await self.sites.handle_update(result)

nazurin/config.py

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
1-
import logging
21
from os import path
32

43
from environs import Env
54

6-
logging.basicConfig(
7-
format='%(asctime)s - %(name)s.%(module)s - %(levelname)s - %(message)s',
8-
level=logging.INFO)
9-
105
env = Env()
116
# read config from .env file if exists
127
env.read_env()
@@ -49,3 +44,4 @@
4944
# Local directory to store database and temporary files
5045
DATA_DIR = 'data'
5146
TEMP_DIR = path.join(DATA_DIR, 'temp')
47+
ACCESS_LOG_FORMAT = '%a "%r" %s %b "%{Referer}i" "%{User-Agent}i"'

nazurin/dispatcher.py

+12-5
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,22 @@
22
from typing import List
33

44
from aiogram import Dispatcher, executor
5-
from aiogram.types import AllowedUpdates, ContentType, Message
5+
from aiogram.types import AllowedUpdates, ContentType, Message, Update
66
from aiogram.utils.executor import Executor
77

88
from nazurin import config
99
from nazurin.utils import logger
1010
from nazurin.utils.filters import URLFilter
1111

1212
from .bot import NazurinBot
13-
from .middleware import AuthMiddleware
13+
from .middleware import AuthMiddleware, LoggingMiddleware
1414
from .server import NazurinServer
1515

1616
class NazurinDispatcher(Dispatcher):
1717
def __init__(self, bot: NazurinBot):
1818
super().__init__(bot)
1919
self.middleware.setup(AuthMiddleware())
20+
self.middleware.setup(LoggingMiddleware())
2021
self.filters_factory.bind(URLFilter,
2122
event_handlers=[self.message_handlers])
2223
self.server = NazurinServer(bot)
@@ -35,8 +36,13 @@ def register_message_handler(self, callback, *args, **kwargs):
3536
*args, **kwargs)
3637

3738
async def on_startup(self, *_args):
38-
await self.bot.set_webhook(config.WEBHOOK_URL + config.TOKEN,
39-
allowed_updates=AllowedUpdates.MESSAGE)
39+
if config.ENV == 'production':
40+
await self.bot.set_webhook(config.WEBHOOK_URL + config.TOKEN,
41+
allowed_updates=AllowedUpdates.MESSAGE)
42+
43+
async def process_update(self, update: Update):
44+
with logger.contextualize(request=f"update:{update.update_id}"):
45+
return await super().process_update(update)
4046

4147
def start(self):
4248
self.init()
@@ -50,7 +56,8 @@ def start(self):
5056
# resulting in RuntimeError: Task attached to different loop
5157
self.executor.run_app(host=config.HOST,
5258
port=config.PORT,
53-
loop=asyncio.get_event_loop())
59+
loop=asyncio.get_event_loop(),
60+
access_log_format=config.ACCESS_LOG_FORMAT)
5461
else:
5562
# self.server.start()
5663
executor.start_polling(self, skip_updates=True)

nazurin/middleware.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@
33
from aiogram.types import Message
44

55
from nazurin import config
6+
from nazurin.utils import logger
67

78
class AuthMiddleware(BaseMiddleware):
8-
async def on_process_message(self, message: Message, _data: dict):
9+
async def on_pre_process_message(self, message: Message, _data: dict):
910
if config.IS_PUBLIC:
1011
return
1112
allowed_chats = config.ALLOW_ID + config.ALLOW_GROUP + [
@@ -16,3 +17,7 @@ async def on_process_message(self, message: Message, _data: dict):
1617
or message.from_user.username in config.ALLOW_USERNAME:
1718
return
1819
raise CancelHandler()
20+
21+
class LoggingMiddleware(BaseMiddleware):
22+
async def on_process_message(self, message: Message, _data: dict):
23+
logger.info('Message {}: {}', message.message_id, message.text)

nazurin/models/image.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,17 @@ async def chosen_url(self) -> str:
3434
not self.height) or self.width + self.height > 10000:
3535
self._chosen_url = self.thumbnail
3636
logger.info(
37-
'Use thumbnail (Unkown image size or width + height > 10000 [%s, %s]): %s',
38-
self.width, self.height, self._chosen_url)
37+
'Use thumbnail (Unkown image size or width + height > 10000 '
38+
'[{width}, {height}]): {url}',
39+
width=self.width,
40+
height=self.height,
41+
url=self._chosen_url)
3942
else:
4043
size = await self.size()
4144
if (not size) or size > 5 * 1024 * 1024:
4245
self._chosen_url = self.thumbnail
4346
logger.info(
44-
'Use thumbnail (Unknown size or size > 5MB [%s]): %s',
47+
'Use thumbnail (Unknown size or size > 5MB [{}]): {}',
4548
size, self._chosen_url)
4649
return self._chosen_url
4750

@@ -54,7 +57,7 @@ async def size(self, **kwargs) -> int:
5457
headers = response.headers
5558
if 'Content-Length' in headers:
5659
self._size = int(headers['Content-Length'])
57-
logger.info('Got image size: %s', self._size)
60+
logger.info('Got image size: {}', self._size)
5861
else:
5962
logger.info('Failed to get image size')
6063
return self._size

nazurin/server.py

+23-5
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
import asyncio
22
import traceback
3+
from json import JSONDecodeError
34

45
import aiohttp_cors
56
import aiojobs
67
from aiohttp import web
78

89
from nazurin import config
10+
from nazurin.utils import logger
11+
from nazurin.utils.exceptions import NazurinError
12+
from nazurin.utils.helpers import format_error
913

1014
class NazurinServer(web.Application):
1115
def __init__(self, bot):
@@ -21,26 +25,40 @@ def __init__(self, bot):
2125
})
2226
self.on_startup.append(self.init_jobs)
2327
self.on_shutdown.append(self.shutdown_jobs)
28+
self.request_id = 1
2429

2530
def start(self):
26-
web.run_app(self)
31+
web.run_app(self, access_log_format=config.ACCESS_LOG_FORMAT)
2732

2833
async def do_update(self, url):
2934
try:
35+
logger.info("API request: {}", url)
3036
await self.bot.update_collection([url])
3137
await self.bot.send_message(config.ADMIN_ID,
3238
f'Successfully collected {url}')
33-
# pylint: disable=broad-except
39+
except NazurinError as error:
40+
await self.bot.send_message(
41+
config.ADMIN_ID,
42+
f'Error processing {url}: {format_error(error)}')
43+
# pylint: disable-next=broad-exception-caught
3444
except Exception as error:
3545
traceback.print_exc()
3646
if isinstance(error, asyncio.TimeoutError):
3747
error = 'Timeout, please try again.'
3848
await self.bot.send_message(
39-
config.ADMIN_ID, f'Error processing {url}: {str(error)}')
49+
config.ADMIN_ID,
50+
f'Error processing {url}: {format_error(error)}')
4051

4152
async def update_handler(self, request):
42-
data = await request.json()
43-
await request.app['jobs'].spawn(self.do_update(data.get('url')))
53+
try:
54+
data = await request.json()
55+
except JSONDecodeError:
56+
return web.HTTPBadRequest()
57+
if 'url' not in data:
58+
return web.HTTPBadRequest()
59+
with logger.contextualize(request=f"request:{self.request_id}"):
60+
await request.app['jobs'].spawn(self.do_update(data.get('url')))
61+
self.request_id += 1
4462
return web.json_response({'error': 0})
4563

4664
async def init_jobs(self, app):

nazurin/sites/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ def load(self):
3434
handle = getattr(module, 'handle')
3535
self.sources.append((PRIORITY, patterns, handle, module_name))
3636
self.sources.sort(key=lambda s: s[0], reverse=True)
37-
logger.info("Loaded %s sites", len(self.sites))
37+
logger.info("Loaded {} sites", len(self.sites))
3838

3939
def api(self, site):
4040
return self.sites[site]

nazurin/sites/deviant_art/api.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -214,5 +214,5 @@ async def require_csrf_token(self, refresh: bool = False) -> None:
214214
# CSRF token must be used along with the cookies returned,
215215
# otherwise will be considered invalid
216216
DeviantArt.cookies = response.cookies
217-
logger.info("Fetched CSRF token: %s, cookies: %s",
217+
logger.info("Fetched CSRF token: {}, cookies: {}",
218218
DeviantArt.csrf_token, DeviantArt.cookies)

nazurin/sites/moebooru/api.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ async def get_post(self, post_id: int):
5050
info = json.loads(info)
5151
post = info['posts'][0]
5252
tags = info['tags']
53-
except json.decoder.JSONDecodeError as err:
54-
logger.error(err)
53+
except json.decoder.JSONDecodeError:
54+
logger.exception("Failed to decode JSON")
5555
return post, tags
5656

5757
async def view(self, post_id: int) -> Illust:

nazurin/sites/pixiv/api.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ async def bookmark(self,
129129
logger.error(response)
130130
raise NazurinError(response.error.user_message
131131
or response.error.message)
132-
logger.info('Bookmarked artwork %s, privacy = %s', artwork_id,
132+
logger.info('Bookmarked artwork {}, privacy = {}', artwork_id,
133133
privacy.value)
134134
return True
135135

@@ -171,14 +171,14 @@ def convert(config: File, output: File):
171171
# thus we convert to YUV420P colorspace for better compatibility.
172172
cmd = f'ffmpeg -i "{config.path}" -vcodec libx264 -pix_fmt yuv420p '\
173173
f'-vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -y "{output.path}"'
174-
logger.info('Calling FFmpeg with command: %s', cmd)
174+
logger.info('Calling FFmpeg with command: {}', cmd)
175175
args = shlex.split(cmd)
176176
try:
177177
output = subprocess.check_output(args,
178178
stderr=subprocess.STDOUT,
179179
shell=False)
180180
except subprocess.CalledProcessError as error:
181-
logger.error('FFmpeg failed with code %s, output:\n %s',
181+
logger.error('FFmpeg failed with code {}, output:\n {}',
182182
error.returncode, error.output.decode())
183183
raise NazurinError(
184184
'Failed to convert ugoira to mp4.') from None
@@ -303,7 +303,7 @@ async def auth(self, retry=True):
303303
if retry:
304304
random_ua = f"PixivAndroidApp/6.{random.randrange(0, 60)}.0"
305305
logger.info(
306-
"Blocked by CloudFlare, retry with random UA: %s",
306+
"Blocked by CloudFlare, retry with random UA: {}",
307307
random_ua)
308308
Pixiv.api.additional_headers = {"User-Agent": random_ua}
309309
return await self.auth(retry=False)

nazurin/storage/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ def load(self):
1616
driver = importlib.import_module('nazurin.storage.' +
1717
driver_name.lower())
1818
self.disks.append(getattr(driver, driver_name)())
19-
logger.info("Loaded %s storage(s), using: %s", len(self.disks),
19+
logger.info("Loaded {} storage(s), using: {}", len(self.disks),
2020
STORAGE)
2121

2222
async def store(self, illust: Illust):

nazurin/storage/googledrive.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ async def store(self, files: List[File]):
6868
tasks = [
6969
self.create_folders(destination) for destination in destinations
7070
]
71-
logger.info("Creating folders: %s", destinations)
71+
logger.info("Creating folders: {}", destinations)
7272
folder_ids = await asyncio.gather(*tasks)
7373
folders = {}
7474
for destination, folder_id in zip(destinations, folder_ids):

nazurin/storage/mega.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ async def store(self, files: List[File]):
8282
tasks = [
8383
self.ensure_existence(destination) for destination in destinations
8484
]
85-
logger.info("Creating folders: %s", destinations)
85+
logger.info("Creating folders: {}", destinations)
8686
folder_ids = await asyncio.gather(*tasks)
8787
folders = {}
8888
for destination, folder_id in zip(destinations, folder_ids):

nazurin/storage/onedrive.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ async def store(self, files: List[File]):
5858
tasks = [
5959
self.ensure_existence(destination) for destination in destinations
6060
]
61-
logger.info("Creating folders: %s", destinations)
61+
logger.info("Creating folders: {}", destinations)
6262
await asyncio.gather(*tasks)
6363

6464
tasks = [self.upload(file) for file in files]
@@ -183,7 +183,7 @@ async def _request(self, method, url, headers=None, **kwargs):
183183
async with Request(headers=_headers) as session:
184184
async with session.request(method, url, **kwargs) as response:
185185
if not response.ok:
186-
logger.info(await response.text())
186+
logger.error(await response.text())
187187
response.raise_for_status()
188188
if 'application/json' in response.headers['Content-Type']:
189189
return await response.json()
@@ -194,15 +194,15 @@ async def stream_upload(self, file: File, url: str):
194194
async def upload_chunk(url: str, chunk: bytes):
195195
async with session.put(url, data=chunk) as response:
196196
if not response.ok:
197-
logger.info(await response.text())
197+
logger.error(await response.text())
198198
response.raise_for_status()
199199

200200
# Must be a multiple of 320 KB
201201
CHUNK_SIZE = 16 * 320 * 1024 # 5MB
202202
headers = self.with_credentials()
203203
range_start = 0
204204
total_size = await file.size()
205-
logger.info("Uploading file, total size: %s...", total_size)
205+
logger.info("Uploading file, total size: {}...", total_size)
206206

207207
async with Request(headers=headers) as session:
208208
async for chunk in read_by_chunks(file.path, CHUNK_SIZE):
@@ -214,7 +214,7 @@ async def upload_chunk(url: str, chunk: bytes):
214214
})
215215
await upload_chunk(url, chunk)
216216
range_start += content_length
217-
logger.info("Uploaded %s", range_start)
217+
logger.info("Uploaded {}", range_start)
218218
logger.info("Upload completed")
219219

220220
def with_credentials(self, headers: dict = None) -> dict:

nazurin/utils/__init__.py

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
import logging
2-
1+
from .logging import logger
32
from .network import Request
43

5-
logger = logging.getLogger('nazurin')
6-
74
__all__ = ['logger', 'Request']

0 commit comments

Comments
 (0)