🎨 按ruff的检查调整程序代码

This commit is contained in:
Azide
2023-07-16 00:22:20 +08:00
committed by felinae98
parent f232ce4c3e
commit dba8f2a9cb
42 changed files with 414 additions and 757 deletions
+5 -6
View File
@@ -1,23 +1,22 @@
from collections import defaultdict
from importlib import import_module
from pathlib import Path
from pkgutil import iter_modules
from typing import DefaultDict, Type
from collections import defaultdict
from importlib import import_module
from .platform import Platform, make_no_target_group
_package_dir = str(Path(__file__).resolve().parent)
for (_, module_name, _) in iter_modules([_package_dir]):
for _, module_name, _ in iter_modules([_package_dir]):
import_module(f"{__name__}.{module_name}")
_platform_list: DefaultDict[str, list[Type[Platform]]] = defaultdict(list)
_platform_list: defaultdict[str, list[type[Platform]]] = defaultdict(list)
for _platform in Platform.registry:
if not _platform.enabled:
continue
_platform_list[_platform.platform_name].append(_platform)
platform_manager: dict[str, Type[Platform]] = dict()
platform_manager: dict[str, type[Platform]] = {}
for name, platform_list in _platform_list.items():
if len(platform_list) == 1:
platform_manager[name] = platform_list[0]
+17 -48
View File
@@ -1,25 +1,23 @@
import json
from typing import Any, Optional
from typing import Any
from bs4 import BeautifulSoup as bs
from httpx import AsyncClient
from nonebot.plugin import require
from bs4 import BeautifulSoup as bs
from ..post import Post
from ..types import Category, RawPost, Target
from ..types import Target, RawPost, Category
from ..utils.scheduler_config import SchedulerConfig
from .platform import CategoryNotRecognize, NewMessage, StatusChange
from .platform import NewMessage, StatusChange, CategoryNotRecognize
class ArknightsSchedConf(SchedulerConfig):
name = "arknights"
schedule_type = "interval"
schedule_setting = {"seconds": 30}
class Arknights(NewMessage):
categories = {1: "游戏公告"}
platform_name = "arknights"
name = "明日方舟游戏信息"
@@ -30,9 +28,7 @@ class Arknights(NewMessage):
has_target = False
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
return "明日方舟游戏信息"
async def get_sub_list(self, _) -> list[RawPost]:
@@ -92,7 +88,6 @@ class Arknights(NewMessage):
class AkVersion(StatusChange):
categories = {2: "更新信息"}
platform_name = "arknights"
name = "明日方舟游戏信息"
@@ -103,15 +98,11 @@ class AkVersion(StatusChange):
has_target = False
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
return "明日方舟游戏信息"
async def get_status(self, _):
res_ver = await self.client.get(
"https://ak-conf.hypergryph.com/config/prod/official/IOS/version"
)
res_ver = await self.client.get("https://ak-conf.hypergryph.com/config/prod/official/IOS/version")
res_preanounce = await self.client.get(
"https://ak-conf.hypergryph.com/config/prod/announce_meta/IOS/preannouncement.meta.json"
)
@@ -121,20 +112,10 @@ class AkVersion(StatusChange):
def compare_status(self, _, old_status, new_status):
res = []
if (
old_status.get("preAnnounceType") == 2
and new_status.get("preAnnounceType") == 0
):
res.append(
Post("arknights", text="登录界面维护公告上线(大概是开始维护了)", target_name="明日方舟更新信息")
)
elif (
old_status.get("preAnnounceType") == 0
and new_status.get("preAnnounceType") == 2
):
res.append(
Post("arknights", text="登录界面维护公告下线(大概是开服了,冲!)", target_name="明日方舟更新信息")
)
if old_status.get("preAnnounceType") == 2 and new_status.get("preAnnounceType") == 0:
res.append(Post("arknights", text="登录界面维护公告上线(大概是开始维护了)", target_name="明日方舟更新信息")) # noqa: E501
elif old_status.get("preAnnounceType") == 0 and new_status.get("preAnnounceType") == 2:
res.append(Post("arknights", text="登录界面维护公告下线(大概是开服了,冲!)", target_name="明日方舟更新信息")) # noqa: E501
if old_status.get("clientVersion") != new_status.get("clientVersion"):
res.append(Post("arknights", text="游戏本体更新(大更新)", target_name="明日方舟更新信息"))
if old_status.get("resVersion") != new_status.get("resVersion"):
@@ -149,7 +130,6 @@ class AkVersion(StatusChange):
class MonsterSiren(NewMessage):
categories = {3: "塞壬唱片新闻"}
platform_name = "arknights"
name = "明日方舟游戏信息"
@@ -160,15 +140,11 @@ class MonsterSiren(NewMessage):
has_target = False
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
return "明日方舟游戏信息"
async def get_sub_list(self, _) -> list[RawPost]:
raw_data = await self.client.get(
"https://monster-siren.hypergryph.com/api/news"
)
raw_data = await self.client.get("https://monster-siren.hypergryph.com/api/news")
return raw_data.json()["data"]["list"]
def get_id(self, post: RawPost) -> Any:
@@ -182,14 +158,12 @@ class MonsterSiren(NewMessage):
async def parse(self, raw_post: RawPost) -> Post:
url = f'https://monster-siren.hypergryph.com/info/{raw_post["cid"]}'
res = await self.client.get(
f'https://monster-siren.hypergryph.com/api/news/{raw_post["cid"]}'
)
res = await self.client.get(f'https://monster-siren.hypergryph.com/api/news/{raw_post["cid"]}')
raw_data = res.json()
content = raw_data["data"]["content"]
content = content.replace("</p>", "</p>\n")
soup = bs(content, "html.parser")
imgs = list(map(lambda x: x["src"], soup("img")))
imgs = [x["src"] for x in soup("img")]
text = f'{raw_post["title"]}\n{soup.text.strip()}'
return Post(
"monster-siren",
@@ -203,7 +177,6 @@ class MonsterSiren(NewMessage):
class TerraHistoricusComic(NewMessage):
categories = {4: "泰拉记事社漫画"}
platform_name = "arknights"
name = "明日方舟游戏信息"
@@ -214,15 +187,11 @@ class TerraHistoricusComic(NewMessage):
has_target = False
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
return "明日方舟游戏信息"
async def get_sub_list(self, _) -> list[RawPost]:
raw_data = await self.client.get(
"https://terra-historicus.hypergryph.com/api/recentUpdate"
)
raw_data = await self.client.get("https://terra-historicus.hypergryph.com/api/recentUpdate")
return raw_data.json()["data"]
def get_id(self, post: RawPost) -> Any:
+17 -45
View File
@@ -1,14 +1,14 @@
import json
import re
import json
from typing import Any
from copy import deepcopy
from datetime import datetime, timedelta
from enum import Enum, unique
from typing import Any, Literal, Optional
from typing_extensions import Self
from datetime import datetime, timedelta
from httpx import AsyncClient
from nonebot.log import logger
from pydantic import BaseModel, Field
from typing_extensions import Self
from pydantic import Field, BaseModel
from ..post import Post
from ..types import ApiError, Category, RawPost, Tag, Target
@@ -25,9 +25,7 @@ class BilibiliSchedConf(SchedulerConfig):
cookie_expire_time = timedelta(hours=5)
def __init__(self):
self._client_refresh_time = datetime(
year=2000, month=1, day=1
) # an expired time
self._client_refresh_time = datetime(year=2000, month=1, day=1) # an expired time
super().__init__()
async def _init_session(self):
@@ -69,12 +67,8 @@ class Bilibili(NewMessage):
parse_target_promot = "请输入用户主页的链接"
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
res = await client.get(
"https://api.bilibili.com/x/web-interface/card", params={"mid": target}
)
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.get("https://api.bilibili.com/x/web-interface/card", params={"mid": target})
res.raise_for_status()
res_data = res.json()
if res_data["code"]:
@@ -129,12 +123,7 @@ class Bilibili(NewMessage):
return self._do_get_category(post_type)
def get_tags(self, raw_post: RawPost) -> list[Tag]:
return [
*map(
lambda tp: tp["topic_name"],
raw_post["display"]["topic_info"]["topic_details"],
)
]
return [*(tp["topic_name"] for tp in raw_post["display"]["topic_info"]["topic_details"])]
def _get_info(self, post_type: Category, card) -> tuple[str, list]:
if post_type == 1:
@@ -178,24 +167,16 @@ class Bilibili(NewMessage):
url = ""
if post_type == 1:
# 一般动态
url = "https://t.bilibili.com/{}".format(
raw_post["desc"]["dynamic_id_str"]
)
url = "https://t.bilibili.com/{}".format(raw_post["desc"]["dynamic_id_str"])
elif post_type == 2:
# 专栏文章
url = "https://www.bilibili.com/read/cv{}".format(
raw_post["desc"]["rid"]
)
url = "https://www.bilibili.com/read/cv{}".format(raw_post["desc"]["rid"])
elif post_type == 3:
# 视频
url = "https://www.bilibili.com/video/{}".format(
raw_post["desc"]["bvid"]
)
url = "https://www.bilibili.com/video/{}".format(raw_post["desc"]["bvid"])
elif post_type == 4:
# 纯文字
url = "https://t.bilibili.com/{}".format(
raw_post["desc"]["dynamic_id_str"]
)
url = "https://t.bilibili.com/{}".format(raw_post["desc"]["dynamic_id_str"])
text, pic = self._get_info(post_type, card_content)
elif post_type == 5:
# 转发
@@ -261,10 +242,7 @@ class Bilibililive(StatusChange):
def get_live_action(self, old_info: Self) -> "Bilibililive.LiveAction":
status = Bilibililive.LiveStatus
action = Bilibililive.LiveAction
if (
old_info.live_status in [status.OFF, status.CYCLE]
and self.live_status == status.ON
):
if old_info.live_status in [status.OFF, status.CYCLE] and self.live_status == status.ON:
return action.TURN_ON
elif old_info.live_status == status.ON and self.live_status in [
status.OFF,
@@ -281,12 +259,8 @@ class Bilibililive(StatusChange):
return action.OFF
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
res = await client.get(
"https://api.bilibili.com/x/web-interface/card", params={"mid": target}
)
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.get("https://api.bilibili.com/x/web-interface/card", params={"mid": target})
res_data = json.loads(res.text)
if res_data["code"]:
return None
@@ -382,9 +356,7 @@ class BilibiliBangumi(StatusChange):
_url = "https://api.bilibili.com/pgc/review/user"
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.get(cls._url, params={"media_id": target})
res_data = res.json()
if res_data["code"]:
+3 -6
View File
@@ -1,15 +1,14 @@
from typing import Any, Optional
from typing import Any
from httpx import AsyncClient
from ..post import Post
from ..types import RawPost, Target
from ..utils import scheduler
from .platform import NewMessage
from ..types import Target, RawPost
class FF14(NewMessage):
categories = {}
platform_name = "ff14"
name = "最终幻想XIV官方公告"
@@ -21,9 +20,7 @@ class FF14(NewMessage):
has_target = False
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
return "最终幻想XIV官方公告"
async def get_sub_list(self, _) -> list[RawPost]:
+7 -7
View File
@@ -2,15 +2,15 @@ import re
import time
import traceback
from bs4 import BeautifulSoup, Tag
from httpx import AsyncClient
from nonebot.log import logger
from bs4 import Tag, BeautifulSoup
from nonebot.plugin import require
from ..post import Post
from ..types import Category, RawPost, Target
from ..types import Target, RawPost, Category
from ..utils import SchedulerConfig, http_client
from .platform import CategoryNotRecognize, CategoryNotSupport, NewMessage
from .platform import NewMessage, CategoryNotSupport, CategoryNotRecognize
class McbbsnewsSchedConf(SchedulerConfig):
@@ -134,9 +134,9 @@ class McbbsNews(NewMessage):
if categoty_name in category_values:
category_id = category_keys[category_values.index(categoty_name)]
elif categoty_name in known_category_values:
raise CategoryNotSupport("McbbsNews订阅暂不支持 {}".format(categoty_name))
raise CategoryNotSupport(f"McbbsNews订阅暂不支持 {categoty_name}")
else:
raise CategoryNotRecognize("Mcbbsnews订阅尚未识别 {}".format(categoty_name))
raise CategoryNotRecognize(f"Mcbbsnews订阅尚未识别 {categoty_name}")
return category_id
async def parse(self, post: RawPost) -> Post:
@@ -170,7 +170,7 @@ class McbbsNews(NewMessage):
一般而言每条新闻的长度都很可观,图片生成时间比较喜人
"""
require("nonebot_plugin_htmlrender")
from nonebot_plugin_htmlrender import capture_element, text_to_pic
from nonebot_plugin_htmlrender import text_to_pic, capture_element
try:
assert url
@@ -181,7 +181,7 @@ class McbbsNews(NewMessage):
device_scale_factor=3,
)
assert pic_data
except:
except Exception:
err_info = traceback.format_exc()
logger.warning(f"渲染错误:{err_info}")
+10 -23
View File
@@ -1,23 +1,21 @@
import re
from typing import Any, Optional
from typing import Any
from httpx import AsyncClient
from ..post import Post
from ..types import ApiError, RawPost, Target
from ..utils import SchedulerConfig
from .platform import NewMessage
from ..utils import SchedulerConfig
from ..types import Target, RawPost, ApiError
class NcmSchedConf(SchedulerConfig):
name = "music.163.com"
schedule_type = "interval"
schedule_setting = {"minutes": 1}
class NcmArtist(NewMessage):
categories = {}
platform_name = "ncm-artist"
enable_tag = False
@@ -29,11 +27,9 @@ class NcmArtist(NewMessage):
parse_target_promot = "请输入歌手主页(包含数字ID)的链接"
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.get(
"https://music.163.com/api/artist/albums/{}".format(target),
f"https://music.163.com/api/artist/albums/{target}",
headers={"Referer": "https://music.163.com/"},
)
res_data = res.json()
@@ -45,16 +41,14 @@ class NcmArtist(NewMessage):
async def parse_target(cls, target_text: str) -> Target:
if re.match(r"^\d+$", target_text):
return Target(target_text)
elif match := re.match(
r"(?:https?://)?music\.163\.com/#/artist\?id=(\d+)", target_text
):
elif match := re.match(r"(?:https?://)?music\.163\.com/#/artist\?id=(\d+)", target_text):
return Target(match.group(1))
else:
raise cls.ParseTargetException()
async def get_sub_list(self, target: Target) -> list[RawPost]:
res = await self.client.get(
"https://music.163.com/api/artist/albums/{}".format(target),
f"https://music.163.com/api/artist/albums/{target}",
headers={"Referer": "https://music.163.com/"},
)
res_data = res.json()
@@ -74,13 +68,10 @@ class NcmArtist(NewMessage):
target_name = raw_post["artist"]["name"]
pics = [raw_post["picUrl"]]
url = "https://music.163.com/#/album?id={}".format(raw_post["id"])
return Post(
"ncm-artist", text=text, url=url, pics=pics, target_name=target_name
)
return Post("ncm-artist", text=text, url=url, pics=pics, target_name=target_name)
class NcmRadio(NewMessage):
categories = {}
platform_name = "ncm-radio"
enable_tag = False
@@ -92,9 +83,7 @@ class NcmRadio(NewMessage):
parse_target_promot = "请输入主播电台主页(包含数字ID)的链接"
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.post(
"http://music.163.com/api/dj/program/byradio",
headers={"Referer": "https://music.163.com/"},
@@ -109,9 +98,7 @@ class NcmRadio(NewMessage):
async def parse_target(cls, target_text: str) -> Target:
if re.match(r"^\d+$", target_text):
return Target(target_text)
elif match := re.match(
r"(?:https?://)?music\.163\.com/#/djradio\?id=(\d+)", target_text
):
elif match := re.match(r"(?:https?://)?music\.163\.com/#/djradio\?id=(\d+)", target_text):
return Target(match.group(1))
else:
raise cls.ParseTargetException()
+30 -52
View File
@@ -1,29 +1,32 @@
import json
import ssl
import json
import time
import typing
from typing import Any
from dataclasses import dataclass
from abc import ABC, abstractmethod
from collections import defaultdict
from dataclasses import dataclass
from typing import Any, Collection, Optional, Type
from collections.abc import Collection
import httpx
from httpx import AsyncClient
from nonebot.log import logger
from nonebot_plugin_saa import PlatformTarget
from ..plugin_config import plugin_config
from ..post import Post
from ..types import Category, RawPost, Tag, Target, UserSubInfo
from ..plugin_config import plugin_config
from ..utils import ProcessContext, SchedulerConfig
from ..types import Tag, Target, RawPost, Category, UserSubInfo
class CategoryNotSupport(Exception):
"raise in get_category, when you know the category of the post but don't want to support it or don't support its parsing yet"
"""raise in get_category, when you know the category of the post
but don't want to support it or don't support its parsing yet
"""
class CategoryNotRecognize(Exception):
"raise in get_category, when you don't know the category of post"
"""raise in get_category, when you don't know the category of post"""
class RegistryMeta(type):
@@ -42,7 +45,6 @@ class RegistryMeta(type):
class PlatformMeta(RegistryMeta):
categories: dict[Category, str]
store: dict[Target, Any]
@@ -60,8 +62,7 @@ class PlatformABCMeta(PlatformMeta, ABC):
class Platform(metaclass=PlatformABCMeta, base=True):
scheduler: Type[SchedulerConfig]
scheduler: type[SchedulerConfig]
ctx: ProcessContext
is_common: bool
enabled: bool
@@ -70,16 +71,14 @@ class Platform(metaclass=PlatformABCMeta, base=True):
categories: dict[Category, str]
enable_tag: bool
platform_name: str
parse_target_promot: Optional[str] = None
registry: list[Type["Platform"]]
parse_target_promot: str | None = None
registry: list[type["Platform"]]
client: AsyncClient
reverse_category: dict[str, Category]
@classmethod
@abstractmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
...
@abstractmethod
@@ -95,11 +94,7 @@ class Platform(metaclass=PlatformABCMeta, base=True):
return await self.fetch_new_post(target, users)
except httpx.RequestError as err:
if plugin_config.bison_show_network_warning:
logger.warning(
"network connection error: {}, url: {}".format(
type(err), err.request.url
)
)
logger.warning(f"network connection error: {type(err)}, url: {err.request.url}")
return []
except ssl.SSLError as err:
if plugin_config.bison_show_network_warning:
@@ -130,7 +125,7 @@ class Platform(metaclass=PlatformABCMeta, base=True):
return Target(target_string)
@abstractmethod
def get_tags(self, raw_post: RawPost) -> Optional[Collection[Tag]]:
def get_tags(self, raw_post: RawPost) -> Collection[Tag] | None:
"Return Tag list of given RawPost"
@classmethod
@@ -201,9 +196,7 @@ class Platform(metaclass=PlatformABCMeta, base=True):
) -> list[tuple[PlatformTarget, list[Post]]]:
res: list[tuple[PlatformTarget, list[Post]]] = []
for user, cats, required_tags in users:
user_raw_post = await self.filter_user_custom(
new_posts, cats, required_tags
)
user_raw_post = await self.filter_user_custom(new_posts, cats, required_tags)
user_post: list[Post] = []
for raw_post in user_raw_post:
user_post.append(await self.do_parse(raw_post))
@@ -211,7 +204,7 @@ class Platform(metaclass=PlatformABCMeta, base=True):
return res
@abstractmethod
def get_category(self, post: RawPost) -> Optional[Category]:
def get_category(self, post: RawPost) -> Category | None:
"Return category of given Rawpost"
raise NotImplementedError()
@@ -221,7 +214,7 @@ class MessageProcess(Platform, abstract=True):
def __init__(self, ctx: ProcessContext, client: AsyncClient):
super().__init__(ctx, client)
self.parse_cache: dict[Any, Post] = dict()
self.parse_cache: dict[Any, Post] = {}
@abstractmethod
def get_id(self, post: RawPost) -> Any:
@@ -246,7 +239,7 @@ class MessageProcess(Platform, abstract=True):
"Get post list of the given target"
@abstractmethod
def get_date(self, post: RawPost) -> Optional[int]:
def get_date(self, post: RawPost) -> int | None:
"Get post timestamp and return, return None if can't get the time"
async def filter_common(self, raw_post_list: list[RawPost]) -> list[RawPost]:
@@ -286,9 +279,7 @@ class NewMessage(MessageProcess, abstract=True):
inited: bool
exists_posts: set[Any]
async def filter_common_with_diff(
self, target: Target, raw_post_list: list[RawPost]
) -> list[RawPost]:
async def filter_common_with_diff(self, target: Target, raw_post_list: list[RawPost]) -> list[RawPost]:
filtered_post = await self.filter_common(raw_post_list)
store = self.get_stored_data(target) or self.MessageStorage(False, set())
res = []
@@ -297,11 +288,7 @@ class NewMessage(MessageProcess, abstract=True):
for raw_post in filtered_post:
post_id = self.get_id(raw_post)
store.exists_posts.add(post_id)
logger.info(
"init {}-{} with {}".format(
self.platform_name, target, store.exists_posts
)
)
logger.info(f"init {self.platform_name}-{target} with {store.exists_posts}")
store.inited = True
else:
for raw_post in filtered_post:
@@ -400,12 +387,11 @@ class SimplePost(MessageProcess, abstract=True):
return res
def make_no_target_group(platform_list: list[Type[Platform]]) -> Type[Platform]:
def make_no_target_group(platform_list: list[type[Platform]]) -> type[Platform]:
if typing.TYPE_CHECKING:
class NoTargetGroup(Platform, abstract=True):
platform_list: list[Type[Platform]]
platform_list: list[type[Platform]]
platform_obj_list: list[Platform]
DUMMY_STR = "_DUMMY"
@@ -418,24 +404,18 @@ def make_no_target_group(platform_list: list[Type[Platform]]) -> Type[Platform]:
for platform in platform_list:
if platform.has_target:
raise RuntimeError(
"Platform {} should have no target".format(platform.name)
)
raise RuntimeError(f"Platform {platform.name} should have no target")
if name == DUMMY_STR:
name = platform.name
elif name != platform.name:
raise RuntimeError("Platform name for {} not fit".format(platform_name))
raise RuntimeError(f"Platform name for {platform_name} not fit")
platform_category_key_set = set(platform.categories.keys())
if platform_category_key_set & categories_keys:
raise RuntimeError(
"Platform categories for {} duplicate".format(platform_name)
)
raise RuntimeError(f"Platform categories for {platform_name} duplicate")
categories_keys |= platform_category_key_set
categories.update(platform.categories)
if platform.scheduler != scheduler:
raise RuntimeError(
"Platform scheduler for {} not fit".format(platform_name)
)
raise RuntimeError(f"Platform scheduler for {platform_name} not fit")
def __init__(self: "NoTargetGroup", ctx: ProcessContext, client: AsyncClient):
Platform.__init__(self, ctx, client)
@@ -444,15 +424,13 @@ def make_no_target_group(platform_list: list[Type[Platform]]) -> Type[Platform]:
self.platform_obj_list.append(platform_class(ctx, client))
def __str__(self: "NoTargetGroup") -> str:
return "[" + " ".join(map(lambda x: x.name, self.platform_list)) + "]"
return "[" + " ".join(x.name for x in self.platform_list) + "]"
@classmethod
async def get_target_name(cls, client: AsyncClient, target: Target):
return await platform_list[0].get_target_name(client, target)
async def fetch_new_post(
self: "NoTargetGroup", target: Target, users: list[UserSubInfo]
):
async def fetch_new_post(self: "NoTargetGroup", target: Target, users: list[UserSubInfo]):
res = defaultdict(list)
for platform in self.platform_obj_list:
platform_res = await platform.fetch_new_post(target=target, users=users)
+4 -7
View File
@@ -1,10 +1,10 @@
import calendar
import time
from typing import Any, Optional
from typing import Any
import feedparser
from bs4 import BeautifulSoup as bs
from httpx import AsyncClient
from bs4 import BeautifulSoup as bs
from ..post import Post
from ..types import RawPost, Target
@@ -20,7 +20,6 @@ class RssSchedConf(SchedulerConfig):
class Rss(NewMessage):
categories = {}
enable_tag = False
platform_name = "rss"
@@ -31,9 +30,7 @@ class Rss(NewMessage):
has_target = True
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
res = await client.get(target, timeout=10.0)
feed = feedparser.parse(res.text)
return feed["feed"]["title"]
@@ -69,7 +66,7 @@ class Rss(NewMessage):
else:
text = f"{title}\n\n{desc}"
pics = list(map(lambda x: x.attrs["src"], soup("img")))
pics = [x.attrs["src"] for x in soup("img")]
if raw_post.get("media_content"):
for media in raw_post["media_content"]:
if media.get("medium") == "image" and media.get("url"):
+32 -58
View File
@@ -1,17 +1,16 @@
import json
import re
from collections.abc import Callable
import json
from typing import Any
from datetime import datetime
from typing import Any, Optional
from bs4 import BeautifulSoup as bs
from httpx import AsyncClient
from nonebot.log import logger
from bs4 import BeautifulSoup as bs
from ..post import Post
from ..types import *
from ..utils import SchedulerConfig, http_client
from .platform import NewMessage
from ..utils import SchedulerConfig, http_client
from ..types import Tag, Target, RawPost, ApiError, Category
class WeiboSchedConf(SchedulerConfig):
@@ -21,7 +20,6 @@ class WeiboSchedConf(SchedulerConfig):
class Weibo(NewMessage):
categories = {
1: "转发",
2: "视频",
@@ -38,13 +36,9 @@ class Weibo(NewMessage):
parse_target_promot = "请输入用户主页(包含数字UID)的链接"
@classmethod
async def get_target_name(
cls, client: AsyncClient, target: Target
) -> Optional[str]:
async def get_target_name(cls, client: AsyncClient, target: Target) -> str | None:
param = {"containerid": "100505" + target}
res = await client.get(
"https://m.weibo.cn/api/container/getIndex", params=param
)
res = await client.get("https://m.weibo.cn/api/container/getIndex", params=param)
res_dict = json.loads(res.text)
if res_dict.get("ok") == 1:
return res_dict["data"]["userInfo"]["screen_name"]
@@ -63,13 +57,14 @@ class Weibo(NewMessage):
async def get_sub_list(self, target: Target) -> list[RawPost]:
params = {"containerid": "107603" + target}
res = await self.client.get(
"https://m.weibo.cn/api/container/getIndex?", params=params, timeout=4.0
)
res = await self.client.get("https://m.weibo.cn/api/container/getIndex?", params=params, timeout=4.0)
res_data = json.loads(res.text)
if not res_data["ok"] and res_data["msg"] != "这里还没有内容":
raise ApiError(res.request.url)
custom_filter: Callable[[RawPost], bool] = lambda d: d["card_type"] == 9
def custom_filter(d: RawPost) -> bool:
return d["card_type"] == 9
return list(filter(custom_filter, res_data["data"]["cards"]))
def get_id(self, post: RawPost) -> Any:
@@ -79,44 +74,32 @@ class Weibo(NewMessage):
return raw_post["card_type"] == 9
def get_date(self, raw_post: RawPost) -> float:
created_time = datetime.strptime(
raw_post["mblog"]["created_at"], "%a %b %d %H:%M:%S %z %Y"
)
created_time = datetime.strptime(raw_post["mblog"]["created_at"], "%a %b %d %H:%M:%S %z %Y")
return created_time.timestamp()
def get_tags(self, raw_post: RawPost) -> Optional[list[Tag]]:
def get_tags(self, raw_post: RawPost) -> list[Tag] | None:
"Return Tag list of given RawPost"
text = raw_post["mblog"]["text"]
soup = bs(text, "html.parser")
res = list(
map(
lambda x: x[1:-1],
filter(
lambda s: s[0] == "#" and s[-1] == "#",
map(lambda x: x.text, soup.find_all("span", class_="surl-text")),
),
res = [
x[1:-1]
for x in filter(
lambda s: s[0] == "#" and s[-1] == "#",
(x.text for x in soup.find_all("span", class_="surl-text")),
)
)
super_topic_img = soup.find(
"img", src=re.compile(r"timeline_card_small_super_default")
)
]
super_topic_img = soup.find("img", src=re.compile(r"timeline_card_small_super_default"))
if super_topic_img:
try:
res.append(
super_topic_img.parent.parent.find("span", class_="surl-text").text # type: ignore
+ "超话"
)
except:
logger.info("super_topic extract error: {}".format(text))
res.append(super_topic_img.parent.parent.find("span", class_="surl-text").text + "超话") # type: ignore
except Exception:
logger.info(f"super_topic extract error: {text}")
return res
def get_category(self, raw_post: RawPost) -> Category:
if raw_post["mblog"].get("retweeted_status"):
return Category(1)
elif (
raw_post["mblog"].get("page_info")
and raw_post["mblog"]["page_info"].get("type") == "video"
):
elif raw_post["mblog"].get("page_info") and raw_post["mblog"]["page_info"].get("type") == "video":
return Category(2)
elif raw_post["mblog"].get("pics"):
return Category(3)
@@ -129,7 +112,8 @@ class Weibo(NewMessage):
async def parse(self, raw_post: RawPost) -> Post:
header = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,"
"*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "zh-CN,zh;q=0.9",
"authority": "m.weibo.cn",
"cache-control": "max-age=0",
@@ -147,26 +131,16 @@ class Weibo(NewMessage):
retweeted = True
pic_num = info["retweeted_status"]["pic_num"] if retweeted else info["pic_num"]
if info["isLongText"] or pic_num > 9:
res = await self.client.get(
"https://m.weibo.cn/detail/{}".format(info["mid"]), headers=header
)
res = await self.client.get(f"https://m.weibo.cn/detail/{info['mid']}", headers=header)
try:
match = re.search(r'"status": ([\s\S]+),\s+"call"', res.text)
assert match
full_json_text = match.group(1)
info = json.loads(full_json_text)
except:
logger.info(
"detail message error: https://m.weibo.cn/detail/{}".format(
info["mid"]
)
)
except Exception:
logger.info(f"detail message error: https://m.weibo.cn/detail/{info['mid']}")
parsed_text = self._get_text(info["text"])
raw_pics_list = (
info["retweeted_status"].get("pics", [])
if retweeted
else info.get("pics", [])
)
raw_pics_list = info["retweeted_status"].get("pics", []) if retweeted else info.get("pics", [])
pic_urls = [img["large"]["url"] for img in raw_pics_list]
pics = []
for pic_url in pic_urls:
@@ -174,7 +148,7 @@ class Weibo(NewMessage):
res = await client.get(pic_url)
res.raise_for_status()
pics.append(res.content)
detail_url = "https://weibo.com/{}/{}".format(info["user"]["id"], info["bid"])
detail_url = f"https://weibo.com/{info['user']['id']}/{info['bid']}"
# return parsed_text, detail_url, pic_urls
return Post(
"weibo",