✨ add context to log http error

src/plugins/nonebot_bison/platform/bilibili.py

@@ -1,21 +1,17 @@
-import functools
 import json
 import re
 from copy import deepcopy
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
-from typing import Any, Callable, Optional
+from typing import Any, Optional
 
-import httpx
 from httpx import AsyncClient
 from nonebot.log import logger
 from typing_extensions import Self
 
-from ..plugin_config import plugin_config
 from ..post import Post
-from ..types import Category, RawPost, Tag, Target
+from ..types import ApiError, Category, RawPost, Tag, Target
 from ..utils import SchedulerConfig
-from ..utils.http import http_args
 from .platform import CategoryNotSupport, NewMessage, StatusChange
 
 
@@ -105,7 +101,7 @@ class Bilibili(NewMessage):
         if res_dict["code"] == 0:
             return res_dict["data"].get("cards")
         else:
-            return []
+            raise ApiError(res.request.url)
 
     def get_id(self, post: RawPost) -> Any:
         return post["desc"]["dynamic_id"]
@@ -306,7 +302,7 @@ class Bilibililive(StatusChange):
             self.name,
             text=title,
             url=url,
-            pics=pic,
+            pics=list(pic),
             target_name=target_name,
             compress=True,
         )
@@ -384,14 +380,14 @@ class BilibiliBangumi(StatusChange):
             lastest_episode = detail_dict["result"]["episodes"]
 
         url = lastest_episode["link"]
-        pic = [lastest_episode["cover"]]
+        pic: list[str] = [lastest_episode["cover"]]
         target_name = detail_dict["result"]["season_title"]
         text = lastest_episode["share_copy"]
         return Post(
             self.name,
             text=text,
             url=url,
-            pics=pic,
+            pics=list(pic),
             target_name=target_name,
             compress=True,
         )

src/plugins/nonebot_bison/platform/ff14.py

@@ -4,7 +4,7 @@ from httpx import AsyncClient
 
 from ..post import Post
 from ..types import RawPost, Target
-from ..utils import http_client, scheduler
+from ..utils import scheduler
 from .platform import NewMessage
 
 
@@ -27,11 +27,10 @@ class FF14(NewMessage):
         return "最终幻想XIV官方公告"
 
     async def get_sub_list(self, _) -> list[RawPost]:
-        async with http_client() as client:
-            raw_data = await client.get(
-                "https://ff.web.sdo.com/inc/newdata.ashx?url=List?gameCode=ff&category=5309,5310,5311,5312,5313&pageIndex=0&pageSize=5"
-            )
-            return raw_data.json()["Data"]
+        raw_data = await self.client.get(
+            "https://ff.web.sdo.com/inc/newdata.ashx?url=List?gameCode=ff&category=5309,5310,5311,5312,5313&pageIndex=0&pageSize=5"
+        )
+        return raw_data.json()["Data"]
 
     def get_id(self, post: RawPost) -> Any:
         """用发布时间当作 ID

src/plugins/nonebot_bison/platform/ncm.py

@@ -4,8 +4,8 @@ from typing import Any, Optional
 from httpx import AsyncClient
 
 from ..post import Post
-from ..types import RawPost, Target
-from ..utils import SchedulerConfig, http_client
+from ..types import ApiError, RawPost, Target
+from ..utils import SchedulerConfig
 from .platform import NewMessage
 
 
@@ -32,15 +32,14 @@ class NcmArtist(NewMessage):
     async def get_target_name(
         cls, client: AsyncClient, target: Target
     ) -> Optional[str]:
-        async with http_client() as client:
-            res = await client.get(
-                "https://music.163.com/api/artist/albums/{}".format(target),
-                headers={"Referer": "https://music.163.com/"},
-            )
-            res_data = res.json()
-            if res_data["code"] != 200:
-                return
-            return res_data["artist"]["name"]
+        res = await client.get(
+            "https://music.163.com/api/artist/albums/{}".format(target),
+            headers={"Referer": "https://music.163.com/"},
+        )
+        res_data = res.json()
+        if res_data["code"] != 200:
+            raise ApiError(res.request.url)
+        return res_data["artist"]["name"]
 
     @classmethod
     async def parse_target(cls, target_text: str) -> Target:
@@ -54,16 +53,15 @@ class NcmArtist(NewMessage):
             raise cls.ParseTargetException()
 
     async def get_sub_list(self, target: Target) -> list[RawPost]:
-        async with http_client() as client:
-            res = await client.get(
-                "https://music.163.com/api/artist/albums/{}".format(target),
-                headers={"Referer": "https://music.163.com/"},
-            )
-            res_data = res.json()
-            if res_data["code"] != 200:
-                return []
-            else:
-                return res_data["hotAlbums"]
+        res = await self.client.get(
+            "https://music.163.com/api/artist/albums/{}".format(target),
+            headers={"Referer": "https://music.163.com/"},
+        )
+        res_data = res.json()
+        if res_data["code"] != 200:
+            return []
+        else:
+            return res_data["hotAlbums"]
 
     def get_id(self, post: RawPost) -> Any:
         return post["id"]
@@ -97,16 +95,15 @@ class NcmRadio(NewMessage):
     async def get_target_name(
         cls, client: AsyncClient, target: Target
     ) -> Optional[str]:
-        async with http_client() as client:
-            res = await client.post(
-                "http://music.163.com/api/dj/program/byradio",
-                headers={"Referer": "https://music.163.com/"},
-                data={"radioId": target, "limit": 1000, "offset": 0},
-            )
-            res_data = res.json()
-            if res_data["code"] != 200 or res_data["programs"] == 0:
-                return
-            return res_data["programs"][0]["radio"]["name"]
+        res = await client.post(
+            "http://music.163.com/api/dj/program/byradio",
+            headers={"Referer": "https://music.163.com/"},
+            data={"radioId": target, "limit": 1000, "offset": 0},
+        )
+        res_data = res.json()
+        if res_data["code"] != 200 or res_data["programs"] == 0:
+            return
+        return res_data["programs"][0]["radio"]["name"]
 
     @classmethod
     async def parse_target(cls, target_text: str) -> Target:
@@ -120,17 +117,16 @@ class NcmRadio(NewMessage):
             raise cls.ParseTargetException()
 
     async def get_sub_list(self, target: Target) -> list[RawPost]:
-        async with http_client() as client:
-            res = await client.post(
-                "http://music.163.com/api/dj/program/byradio",
-                headers={"Referer": "https://music.163.com/"},
-                data={"radioId": target, "limit": 1000, "offset": 0},
-            )
-            res_data = res.json()
-            if res_data["code"] != 200:
-                return []
-            else:
-                return res_data["programs"]
+        res = await self.client.post(
+            "http://music.163.com/api/dj/program/byradio",
+            headers={"Referer": "https://music.163.com/"},
+            data={"radioId": target, "limit": 1000, "offset": 0},
+        )
+        res_data = res.json()
+        if res_data["code"] != 200:
+            return []
+        else:
+            return res_data["programs"]
 
     def get_id(self, post: RawPost) -> Any:
         return post["id"]

src/plugins/nonebot_bison/platform/platform.py

@@ -14,7 +14,7 @@ from nonebot.log import logger
 from ..plugin_config import plugin_config
 from ..post import Post
 from ..types import Category, RawPost, Tag, Target, User, UserSubInfo
-from ..utils.scheduler_config import SchedulerConfig
+from ..utils import ProcessContext, SchedulerConfig
 
 
 class CategoryNotSupport(Exception):
@@ -57,6 +57,7 @@ class PlatformABCMeta(PlatformMeta, ABC):
 class Platform(metaclass=PlatformABCMeta, base=True):
 
     scheduler: Type[SchedulerConfig]
+    ctx: ProcessContext
     is_common: bool
     enabled: bool
     name: str
@@ -99,7 +100,7 @@ class Platform(metaclass=PlatformABCMeta, base=True):
             return []
         except json.JSONDecodeError as err:
             logger.warning(f"json error, parsing: {err.doc}")
-            return []
+            raise err
 
     @abstractmethod
     async def parse(self, raw_post: RawPost) -> Post:
@@ -109,9 +110,10 @@ class Platform(metaclass=PlatformABCMeta, base=True):
         "actually function called"
         return await self.parse(raw_post)
 
-    def __init__(self, client: AsyncClient):
+    def __init__(self, context: ProcessContext, client: AsyncClient):
         super().__init__()
         self.client = client
+        self.ctx = context
 
     class ParseTargetException(Exception):
         pass
@@ -209,8 +211,8 @@ class Platform(metaclass=PlatformABCMeta, base=True):
 class MessageProcess(Platform, abstract=True):
     "General message process fetch, parse, filter progress"
 
-    def __init__(self, client: AsyncClient):
-        super().__init__(client)
+    def __init__(self, ctx: ProcessContext, client: AsyncClient):
+        super().__init__(ctx, client)
         self.parse_cache: dict[Any, Post] = dict()
 
     @abstractmethod
@@ -254,6 +256,9 @@ class MessageProcess(Platform, abstract=True):
             try:
                 self.get_category(raw_post)
             except CategoryNotSupport:
+                msgs = self.ctx.gen_req_records()
+                for m in msgs:
+                    logger.warning(m)
                 continue
             except NotImplementedError:
                 pass
@@ -342,7 +347,7 @@ class StatusChange(Platform, abstract=True):
             new_status = await self.get_status(target)
         except self.FetchError as err:
             logger.warning(f"fetching {self.name}-{target} error: {err}")
-            return []
+            raise
         res = []
         if old_status := self.get_stored_data(target):
             diff = self.compare_status(target, old_status, new_status)
@@ -420,11 +425,11 @@ def make_no_target_group(platform_list: list[Type[Platform]]) -> Type[Platform]:
                 "Platform scheduler for {} not fit".format(platform_name)
             )
 
-    def __init__(self: "NoTargetGroup", client: AsyncClient):
-        Platform.__init__(self, client)
+    def __init__(self: "NoTargetGroup", ctx: ProcessContext, client: AsyncClient):
+        Platform.__init__(self, ctx, client)
         self.platform_obj_list = []
         for platform_class in self.platform_list:
-            self.platform_obj_list.append(platform_class(client))
+            self.platform_obj_list.append(platform_class(ctx, client))
 
     def __str__(self: "NoTargetGroup") -> str:
         return "[" + " ".join(map(lambda x: x.name, self.platform_list)) + "]"

src/plugins/nonebot_bison/platform/rss.py

@@ -7,7 +7,7 @@ from httpx import AsyncClient
 
 from ..post import Post
 from ..types import RawPost, Target
-from ..utils import http_client, scheduler
+from ..utils import scheduler
 from .platform import NewMessage
 
 
@@ -26,10 +26,9 @@ class Rss(NewMessage):
     async def get_target_name(
         cls, client: AsyncClient, target: Target
     ) -> Optional[str]:
-        async with http_client() as client:
-            res = await client.get(target, timeout=10.0)
-            feed = feedparser.parse(res.text)
-            return feed["feed"]["title"]
+        res = await client.get(target, timeout=10.0)
+        feed = feedparser.parse(res.text)
+        return feed["feed"]["title"]
 
     def get_date(self, post: RawPost) -> int:
         return calendar.timegm(post.published_parsed)
@@ -38,13 +37,12 @@ class Rss(NewMessage):
         return post.id
 
     async def get_sub_list(self, target: Target) -> list[RawPost]:
-        async with http_client() as client:
-            res = await client.get(target, timeout=10.0)
-            feed = feedparser.parse(res)
-            entries = feed.entries
-            for entry in entries:
-                entry["_target_name"] = feed.feed.title
-            return feed.entries
+        res = await self.client.get(target, timeout=10.0)
+        feed = feedparser.parse(res)
+        entries = feed.entries
+        for entry in entries:
+            entry["_target_name"] = feed.feed.title
+        return feed.entries
 
     async def parse(self, raw_post: RawPost) -> Post:
         text = raw_post.get("title", "") + "\n" if raw_post.get("title") else ""

src/plugins/nonebot_bison/platform/weibo.py

@@ -68,7 +68,7 @@ class Weibo(NewMessage):
         )
         res_data = json.loads(res.text)
         if not res_data["ok"]:
-            return []
+            raise ApiError(res.request.url)
         custom_filter: Callable[[RawPost], bool] = lambda d: d["card_type"] == 9
         return list(filter(custom_filter, res_data["data"]["cards"]))
 

src/plugins/nonebot_bison/scheduler/scheduler.py

@@ -9,7 +9,7 @@ from ..config import config
 from ..platform import platform_manager
 from ..send import send_msgs
 from ..types import Target
-from ..utils import SchedulerConfig
+from ..utils import ProcessContext, SchedulerConfig
 from .aps import aps
 
 
@@ -78,6 +78,7 @@ class Scheduler:
         return cur_max_schedulable
 
     async def exec_fetch(self):
+        context = ProcessContext()
         if not (schedulable := await self.get_next_schedulable()):
             return
         logger.debug(
@@ -86,12 +87,22 @@ class Scheduler:
         send_userinfo_list = await config.get_platform_target_subscribers(
             schedulable.platform_name, schedulable.target
         )
-        platform_obj = platform_manager[schedulable.platform_name](
-            await self.scheduler_config_obj.get_client(schedulable.target)
-        )
-        to_send = await platform_obj.do_fetch_new_post(
-            schedulable.target, send_userinfo_list
-        )
+
+        client = await self.scheduler_config_obj.get_client(schedulable.target)
+        context.register_to_client(client)
+
+        try:
+            platform_obj = platform_manager[schedulable.platform_name](context, client)
+            to_send = await platform_obj.do_fetch_new_post(
+                schedulable.target, send_userinfo_list
+            )
+        except Exception as err:
+            records = context.gen_req_records()
+            for record in records:
+                logger.warning("API request record: " + record)
+            err.args += (records,)
+            raise
+
         if not to_send:
             return
         bot = nonebot.get_bot()

src/plugins/nonebot_bison/types.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from datetime import time
 from typing import Any, Literal, NamedTuple, NewType
 
+from httpx import URL
 from pydantic import BaseModel
 
 RawPost = NewType("RawPost", Any)
@@ -45,3 +46,9 @@ class PlatformWeightConfigResp(BaseModel):
     target_name: str
     platform_name: str
     weight: WeightConfig
+
+
+class ApiError(Exception):
+    def __init__(self, url: URL) -> None:
+        msg = f"api {url} error"
+        super().__init__(msg)

src/plugins/nonebot_bison/utils/__init__.py

@@ -9,6 +9,7 @@ from nonebot.log import default_format, logger
 from nonebot.plugin import require
 
 from ..plugin_config import plugin_config
+from .context import ProcessContext
 from .http import http_client
 from .scheduler_config import SchedulerConfig, scheduler
 
@@ -16,6 +17,7 @@ __all__ = [
     "http_client",
     "Singleton",
     "parse_text",
+    "ProcessContext",
     "html_to_text",
     "SchedulerConfig",
     "scheduler",

src/plugins/nonebot_bison/utils/context.py

@@ -0,0 +1,40 @@
+from base64 import b64encode
+
+from httpx import AsyncClient, Response
+
+
+class ProcessContext:
+    reqs: list[Response]
+
+    def __init__(self) -> None:
+        self.reqs = []
+
+    def log_response(self, resp: Response):
+        self.reqs.append(resp)
+
+    def register_to_client(self, client: AsyncClient):
+        async def _log_to_ctx(r: Response):
+            self.log_response(r)
+
+        hooks = {
+            "response": [_log_to_ctx],
+        }
+        client.event_hooks = hooks
+
+    def _should_print_content(self, r: Response) -> bool:
+        content_type = r.headers["content-type"]
+        if content_type.startswith("text"):
+            return True
+        if "json" in content_type:
+            return True
+        return False
+
+    def gen_req_records(self) -> list[str]:
+        res = []
+        for req in self.reqs:
+            if self._should_print_content(req):
+                log_content = f"{req.request.url} {req.request.headers} | [{req.status_code}] {req.headers} {req.text}"
+            else:
+                log_content = f"{req.request.url} {req.request.headers} | [{req.status_code}] {req.headers} b64encoded: {b64encode(req.content[:50]).decode()}"
+            res.append(log_content)
+        return res