suyiiyii/nonebot-bison
1
0
Fork 0
mirror of https://github.com/suyiiyii/nonebot-bison.git synced 2025-06-02 09:26:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🔥 移除 mcbbsnews platform

Browse Source
This commit is contained in:
Azide 2024-02-20 15:31:12 +08:00 committed by felinae98
parent 2a33d5143d
commit 700196c888
6 changed files with 0 additions and 13059 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
nonebot_bison/platform/mcbbsnews.py
View File

@ -1,192 +0,0 @@
import re
import time
import traceback
from httpx import AsyncClient
from nonebot.log import logger
from bs4 import Tag, BeautifulSoup
from nonebot.plugin import require
from ..post import Post
from ..types import Target, RawPost, Category
from ..utils import SchedulerConfig, http_client
from .platform import NewMessage, CategoryNotSupport, CategoryNotRecognize
class McbbsnewsSchedConf(SchedulerConfig):
name = "mcbbsnews"
schedule_type = "interval"
schedule_setting = {"minutes": 30}
class McbbsNews(NewMessage):
categories: dict[int, str] = {
1: "Java版资讯",
2: "基岩版资讯",
3: "块讯",
4: "基岩块讯",
5: "周边",
6: "主机",
7: "时评",
}
enable_tag: bool = False
platform_name: str = "mcbbsnews"
name: str = "MCBBS幻翼块讯"
enabled: bool = True
is_common: bool = False
scheduler = McbbsnewsSchedConf
has_target: bool = False
_known_cats: dict[int, str] = {
1: "Java版资讯",
2: "基岩版资讯",
3: "块讯",
4: "基岩块讯",
5: "周边",
6: "主机",
7: "时评",
}
@classmethod
async def get_target_name(cls, client: AsyncClient, target: Target) -> str:
return cls.name
async def get_sub_list(self, _: Target) -> list[RawPost]:
url: str = "https://www.mcbbs.net/forum-news-1.html"
html = await self.client.get(url)
soup = BeautifulSoup(html.text, "html.parser")
raw_post_list = soup.find_all("tbody", id=re.compile(r"normalthread_[0-9]*"))
post_list = self._gen_post_list(raw_post_list)
return post_list
def _gen_post_list(self, raw_post_list: list[Tag]) -> list[RawPost]:
"""解析生成推文列表"""
post_list = []
for raw_post in raw_post_list:
post = {}
url_tag = raw_post.find("a", class_="s xst")
if isinstance(url_tag, Tag):
post["url"] = url_tag.get("href")
title_tag = raw_post.find("a", class_="s xst")
if isinstance(title_tag, Tag):
title_string = title_tag.string
if isinstance(title_string, str):
post["title"] = self._format_text(title_string, "title")
post["category"] = raw_post.select("th em a")[0].string
post["author"] = raw_post.select("td:nth-of-type(2) cite a")[0].string
post["id"] = raw_post["id"]
raw_date = (
raw_post.select("td:nth-of-type(2) em span span")[0]["title"]
if raw_post.select("td:nth-of-type(2) em span span")
else raw_post.select("td:nth-of-type(2) em span")[0].string
)
if isinstance(raw_date, str):
post["date"] = self._stamp_date(raw_date)
post_list.append(post)
return post_list
@staticmethod
def _format_text(raw_text: str, mode: str) -> str:
"""
处理BeautifulSoup生成的string中奇怪的回车+连续空格
参数:
title: 处理标题
"""
match mode:
case "title":
ftext = re.sub(r"\n\s*", " ", raw_text)
case _:
raise NotImplementedError("不支持的处理模式: {mode}")
return ftext
@staticmethod
def _stamp_date(raw_date: str) -> int:
"""
将时间转化为时间戳:
yyyy-mm-dd -> timestamp
"""
time_stamp = int(time.mktime(time.strptime(raw_date, "%Y-%m-%d")))
return time_stamp
def get_id(self, post: RawPost) -> str:
return post["id"]
def get_date(self, _: RawPost) -> int | None:
# 获取datetime精度只到日期，故暂时舍弃
# return post["date"]
return None
def get_category(self, post: RawPost) -> Category:
categoty_name = post["category"]
category_keys = list(self.categories.keys())
category_values = list(self.categories.values())
known_category_values = list(self._known_cats.values())
if categoty_name in category_values:
category_id = category_keys[category_values.index(categoty_name)]
elif categoty_name in known_category_values:
raise CategoryNotSupport(f"McbbsNews订阅暂不支持 {categoty_name}")
else:
raise CategoryNotRecognize(f"Mcbbsnews订阅尚未识别 {categoty_name}")
return category_id
async def parse(self, post: RawPost) -> Post:
"""获取并分配正式推文交由相应的函数渲染"""
post_url = "https://www.mcbbs.net/{}".format(post["url"])
async with http_client() as client:
html = await client.get(post_url)
html.raise_for_status()
soup = BeautifulSoup(html.text, "html.parser")
post_body = soup.find("td", id=re.compile(r"postmessage_[0-9]*"))
if isinstance(post_body, Tag):
post_id = post_body.attrs.get("id")
else:
post_id = None
pics = await self._news_render(post_url, f"#{post_id}")
return Post(
self,
"{}\n│\n└由 {} 发表".format(post["title"], post["author"]),
url=post_url,
images=list(pics),
nickname=post["category"],
)
async def _news_render(self, url: str, selector: str) -> list[bytes]:
"""
将给定的url网页的指定CSS选择器部分渲染成图片
注意：
一般而言每条新闻的长度都很可观，图片生成时间比较喜人
"""
require("nonebot_plugin_htmlrender")
from nonebot_plugin_htmlrender import text_to_pic, capture_element
try:
assert url
pic_data = await capture_element(
url,
selector,
viewport={"width": 1000, "height": 6400},
device_scale_factor=3,
)
assert pic_data
except Exception:
err_info = traceback.format_exc()
logger.warning(f"渲染错误：{err_info}")
err_pic0 = await text_to_pic("错误发生！")
err_pic1 = await text_to_pic(err_info)
return [err_pic0, err_pic1]
else:
return [pic_data]

218
tests/platforms/static/mcbbsnews/mcbbsnews_raw_post_list_update.json vendored
View File

@ -1,218 +0,0 @@
[
{
"url": "thread-1340927-1-1.html",
"title": "Minecraft Java版 1.19-pre1 发布",
"category": "Java版资讯",
"author": "希铁石z",
"id": "normalthread_1340927",
"date": 1652889600
},
{
"url": "thread-1340080-1-1.html",
"title": "Mojang Status：服务器出现一些小问题",
"category": "块讯",
"author": "DreamVoid",
"id": "normalthread_1340080",
"date": 1652630400
},
{
"url": "thread-1339940-1-1.html",
"title": "kinbdogz 就近期荒野更新的风波发表看法",
"category": "块讯",
"author": "卡狗",
"id": "normalthread_1339940",
"date": 1652630400
},
{
"url": "thread-1339097-1-1.html",
"title": "Minecraft 基岩版 1.18.33 发布（仅 Switch）",
"category": "基岩版资讯",
"author": "电量量",
"id": "normalthread_1339097",
"date": 1652457600
},
{
"url": "thread-1338607-1-1.html",
"title": "Minecraft Java版 22w19a 发布",
"category": "Java版资讯",
"author": "寂华",
"id": "normalthread_1338607",
"date": 1652371200
},
{
"url": "thread-1338592-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.32/33 发布",
"category": "基岩版资讯",
"author": "苦力怕553",
"id": "normalthread_1338592",
"date": 1652371200
},
{
"url": "thread-1338588-1-1.html",
"title": "请给我们一个真正的“荒野更新”",
"category": "时评",
"author": "斯乌",
"id": "normalthread_1338588",
"date": 1652371200
},
{
"url": "thread-1338496-1-1.html",
"title": "slicedlime：周三无快照，推迟至周四",
"category": "块讯",
"author": "橄榄Chan",
"id": "normalthread_1338496",
"date": 1652198400
},
{
"url": "thread-1336371-1-1.html",
"title": "Minecraft 基岩版 1.18.32 发布（仅 Android、NS）【新增 NS 平台】",
"category": "基岩版资讯",
"author": "电量量",
"id": "normalthread_1336371",
"date": 1651766400
},
{
"url": "thread-1335897-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.30/31 发布",
"category": "基岩版资讯",
"author": "AzureZeng",
"id": "normalthread_1335897",
"date": 1651680000
},
{
"url": "thread-1335891-1-1.html",
"title": "Minecraft Java版 22w18a 发布",
"category": "Java版资讯",
"author": "Aurora_Feather",
"id": "normalthread_1335891",
"date": 1651680000
},
{
"url": "thread-1333196-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.28/29 发布",
"category": "基岩版资讯",
"author": "希铁石z",
"id": "normalthread_1333196",
"date": 1651161600
},
{
"url": "thread-1332834-1-1.html",
"title": "Minecraft 基岩版 1.18.31 发布",
"category": "基岩版资讯",
"author": "希铁石z",
"id": "normalthread_1332834",
"date": 1651075200
},
{
"url": "thread-1332811-1-1.html",
"title": "Minecraft Java版 22w17a 发布",
"category": "Java版资讯",
"author": "卡狗",
"id": "normalthread_1332811",
"date": 1651075200
},
{
"url": "thread-1332424-1-1.html",
"title": "Mojang Status：正在寻找1.18.30更新问题的解决方案",
"category": "基岩块讯",
"author": "ArmorRush",
"id": "normalthread_1332424",
"date": 1650988800
},
{
"url": "thread-1329712-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.26/27 发布",
"category": "基岩版资讯",
"author": "ArmorRush",
"id": "normalthread_1329712",
"date": 1650470400
},
{
"url": "thread-1329651-1-1.html",
"title": "Minecraft Java版 22w16b 发布",
"category": "Java版资讯",
"author": "卡狗",
"id": "normalthread_1329651",
"date": 1650470400
},
{
"url": "thread-1329644-1-1.html",
"title": "Minecraft Java版 22w16a 发布",
"category": "Java版资讯",
"author": "希铁石z",
"id": "normalthread_1329644",
"date": 1650470400
},
{
"url": "thread-1329335-1-1.html",
"title": "Minecraft 基岩版 1.18.30 发布",
"category": "基岩版资讯",
"author": "ArmorRush",
"id": "normalthread_1329335",
"date": 1650384000
},
{
"url": "thread-1328892-1-1.html",
"title": "“海王” 杰森·莫玛 有望主演《我的世界》大电影",
"category": "块讯",
"author": "广药",
"id": "normalthread_1328892",
"date": 1650297600
},
{
"url": "thread-1327089-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.24/25 发布",
"category": "基岩版资讯",
"author": "ArmorRush",
"id": "normalthread_1327089",
"date": 1649952000
},
{
"url": "thread-1326640-1-1.html",
"title": "Minecraft Java版 22w15a 发布",
"category": "Java版资讯",
"author": "ArmorRush",
"id": "normalthread_1326640",
"date": 1649865600
},
{
"url": "thread-1323762-1-1.html",
"title": "Minecraft 基岩版 Beta & Preview 1.19.0.20 发布",
"category": "基岩版资讯",
"author": "ArmorRush",
"id": "normalthread_1323762",
"date": 1649260800
},
{
"url": "thread-1323662-1-1.html",
"title": "Minecraft Java版 22w14a 发布",
"category": "Java版资讯",
"author": "卡狗",
"id": "normalthread_1323662",
"date": 1649260800
},
{
"url": "thread-1321419-1-1.html",
"title": "[愚人节] Minecraft Java版 22w13oneBlockAtATime‎‎ 发布",
"category": "Java版资讯",
"author": "希铁石z",
"id": "normalthread_1321419",
"date": 1648742400
},
{
"url": "thread-1320986-1-1.html",
"title": "Minecraft：近期没有为主机平台添加光线追踪的计划",
"category": "基岩块讯",
"author": "ArmorRush",
"id": "normalthread_1320986",
"date": 1648742400
},
{
"url": "thread-1320931-1-1.html",
"title": "Minecraft Java版 22w13a 发布",
"category": "Java版资讯",
"author": "卡狗",
"id": "normalthread_1320931",
"date": 1648742400
}
]

7429
tests/platforms/static/mcbbsnews/mock/mcbbsnews_new_post_html.html vendored
View File

File diff suppressed because one or more lines are too long

2552
tests/platforms/static/mcbbsnews/mock/mcbbsnews_post_list_html-0.html vendored
View File

File diff suppressed because one or more lines are too long

2601
tests/platforms/static/mcbbsnews/mock/mcbbsnews_post_list_html-1.html vendored
View File

File diff suppressed because one or more lines are too long

67
tests/platforms/test_mcbbsnews.py
View File

@ -1,67 +0,0 @@
import respx
import pytest
from flaky import flaky
from nonebug.app import App
from httpx import Response, AsyncClient
from .utils import get_file, get_json
@pytest.fixture()
def mcbbsnews(app: App):
from nonebot_bison.utils import ProcessContext
from nonebot_bison.platform import platform_manager
return platform_manager["mcbbsnews"](ProcessContext(), AsyncClient())
@pytest.fixture(scope="module")
def raw_post_list():
return get_json("mcbbsnews/mcbbsnews_raw_post_list_update.json")
@pytest.mark.asyncio
@pytest.mark.render
@respx.mock
@flaky(max_runs=3, min_passes=1)
async def test_fetch_new(mcbbsnews, dummy_user_subinfo, raw_post_list):
from nonebot_bison.post import Post
news_router = respx.get("https://www.mcbbs.net/forum-news-1.html")
news_router.mock(return_value=Response(200, text=get_file("mcbbsnews/mock/mcbbsnews_post_list_html-0.html")))
new_post = respx.get("https://www.mcbbs.net/thread-1340927-1-1.html")
new_post.mock(return_value=Response(200, text=get_file("mcbbsnews/mock/mcbbsnews_new_post_html.html")))
target = ""
res = await mcbbsnews.fetch_new_post(target, [dummy_user_subinfo])
assert news_router.called
assert len(res) == 0
news_router.mock(return_value=Response(200, text=get_file("mcbbsnews/mock/mcbbsnews_post_list_html-1.html")))
res1 = await mcbbsnews.fetch_new_post(target, [dummy_user_subinfo])
assert news_router.called
post: Post = res1[0][1][0]
raw_post = raw_post_list[0]
assert post.platform.name == "MCBBS幻翼块讯"
assert post.content == "{}\n│\n└由 {} 发表".format(raw_post["title"], raw_post["author"])
assert post.url == "https://www.mcbbs.net/{}".format(raw_post["url"])
assert post.nickname == raw_post["category"]
assert post.images
assert len(post.images) == 1
@pytest.mark.asyncio
@pytest.mark.render
@respx.mock
@flaky(max_runs=3, min_passes=1)
async def test_news_render(mcbbsnews, dummy_user_subinfo):
new_post = respx.get("https://www.mcbbs.net/thread-1340927-1-1.html")
new_post.mock(return_value=Response(200, text=get_file("mcbbsnews/mock/mcbbsnews_new_post_html.html")))
pics = await mcbbsnews._news_render("https://www.mcbbs.net/thread-1340927-1-1.html", "#post_25849603")
assert len(pics) == 1
pics_err_on_assert = await mcbbsnews._news_render("", "##post_25849603")
assert len(pics_err_on_assert) == 2
pics_err_on_other = await mcbbsnews._news_render("https://www.mcbbs.net/thread-1340927-1-1.html", "#post_err")
assert len(pics_err_on_other) == 2