Merge branch 'main' into admin-page

src/plugins/nonebot_bison/config.py

@@ -14,14 +14,18 @@ from .platform import platform_manager
 supported_target_type = platform_manager.keys()
 
 def get_config_path() -> str:
-    if plugin_config.hk_reporter_config_path:
-        data_dir = plugin_config.hk_reporter_config_path
+    if plugin_config.bison_config_path:
+        data_dir = plugin_config.bison_config_path
     else:
         working_dir = os.getcwd()
         data_dir = path.join(working_dir, 'data')
     if not path.isdir(data_dir):
         os.makedirs(data_dir)
-    return path.join(data_dir, 'hk_reporter.json')
+    old_path = path.join(data_dir, 'hk_reporter.json')
+    new_path = path.join(data_dir, 'bison.json')
+    if os.path.exists(old_path) and not os.path.exists(new_path):
+        os.rename(old_path, new_path)
+    return new_path
 
 class NoSuchUserException(Exception):
     pass

src/plugins/nonebot_bison/config_manager.py

@@ -10,10 +10,14 @@ from nonebot.matcher import Matcher
 
 from .config import Config, NoSuchSubscribeException
 from .platform import platform_manager, check_sub_target
-from .send import send_msgs
 from .utils import parse_text
 from .types import Target
 
+def _gen_prompt_template(prompt: str):
+    if hasattr(Message, 'template'):
+        return Message.template(prompt)
+    return prompt
+
 common_platform = [p.platform_name for p in \
         filter(lambda platform: platform.enabled and platform.is_common,
             platform_manager.values())
@@ -29,7 +33,7 @@ async def send_help(bot: Bot, event: Event, state: T_State):
 def do_add_sub(add_sub: Type[Matcher]):
     @add_sub.handle()
     async def init_promote(bot: Bot, event: Event, state: T_State):
-        state['_prompt'] = '请输入想要订阅的平台，目前支持：\n' + \
+        state['_prompt'] = '请输入想要订阅的平台，目前支持，请输入冒号左边的名称：\n' + \
                 ''.join(['{}：{}\n'.format(platform_name, platform_manager[platform_name].name) \
                         for platform_name in common_platform]) + \
                 '要查看全部平台请输入：“全部”'
@@ -46,11 +50,11 @@ def do_add_sub(add_sub: Type[Matcher]):
         else:
             await add_sub.reject('平台输入错误')
 
-    @add_sub.got('platform', '{_prompt}', parse_platform)
+    @add_sub.got('platform', _gen_prompt_template('{_prompt}'), parse_platform)
     @add_sub.handle()
     async def init_id(bot: Bot, event: Event, state: T_State):
         if platform_manager[state['platform']].has_target:
-            state['_prompt'] = '请输入订阅用户的id，详情查阅https://nonebot-hk-reporter.vercel.app/usage/#%E6%89%80%E6%94%AF%E6%8C%81%E5%B9%B3%E5%8F%B0%E7%9A%84uid'
+            state['_prompt'] = '请输入订阅用户的id，详情查阅https://nonebot-bison.vercel.app/usage/#%E6%89%80%E6%94%AF%E6%8C%81%E5%B9%B3%E5%8F%B0%E7%9A%84uid'
         else:
             state['id'] = 'default'
             state['name'] = await platform_manager[state['platform']].get_target_name(Target(''))
@@ -63,14 +67,14 @@ def do_add_sub(add_sub: Type[Matcher]):
         state['id'] = target
         state['name'] = name
 
-    @add_sub.got('id', '{_prompt}', parse_id)
+    @add_sub.got('id', _gen_prompt_template('{_prompt}'), parse_id)
     @add_sub.handle()
     async def init_cat(bot: Bot, event: Event, state: T_State):
         if not platform_manager[state['platform']].categories:
             state['cats'] = []
             return
         state['_prompt'] = '请输入要订阅的类别，以空格分隔，支持的类别有：{}'.format(
-                ','.join(list(platform_manager[state['platform']].categories.values())))
+                ' '.join(list(platform_manager[state['platform']].categories.values())))
 
     async def parser_cats(bot: Bot, event: Event, state: T_State):
         res = []
@@ -80,7 +84,7 @@ def do_add_sub(add_sub: Type[Matcher]):
             res.append(platform_manager[state['platform']].reverse_category[cat])
         state['cats'] = res
 
-    @add_sub.got('cats', '{_prompt}', parser_cats)
+    @add_sub.got('cats', _gen_prompt_template('{_prompt}'), parser_cats)
     @add_sub.handle()
     async def init_tag(bot: Bot, event: Event, state: T_State):
         if not platform_manager[state['platform']].enable_tag:
@@ -94,7 +98,7 @@ def do_add_sub(add_sub: Type[Matcher]):
         else:
             state['tags'] = str(event.get_message()).strip().split()
 
-    @add_sub.got('tags', '{_prompt}', parser_tags)
+    @add_sub.got('tags', _gen_prompt_template('{_prompt}'), parser_tags)
     @add_sub.handle()
     async def add_sub_process(bot: Bot, event: Event, state: T_State):
         config = Config()
@@ -118,7 +122,6 @@ def do_query_sub(query_sub: Type[Matcher]):
             if platform.enable_tag:
                 res += ' {}'.format(', '.join(sub['tags']))
             res += '\n'
-        # send_msgs(bot, event.group_id, 'group', [await parse_text(res)])
         await query_sub.finish(Message(await parse_text(res)))
 
 def do_del_sub(del_sub: Type[Matcher]):

src/plugins/nonebot_bison/platform/arknights.py

@@ -1,14 +1,13 @@
-from typing import Any
-import httpx
 import json
+from typing import Any
+
 from bs4 import BeautifulSoup as bs
+import httpx
 
-from ..types import Category, RawPost, Target
-
-from .platform import NewMessage, NoTargetMixin, CategoryNotSupport, StatusChange
-
-from ..utils import Render
 from ..post import Post
+from ..types import Category, RawPost, Target
+from ..utils import Render
+from .platform import CategoryNotSupport, NewMessage, NoTargetMixin, StatusChange
 
 
 class Arknights(NewMessage, NoTargetMixin):
@@ -41,6 +40,7 @@ class Arknights(NewMessage, NoTargetMixin):
 
     async def parse(self, raw_post: RawPost) -> Post:
         announce_url = raw_post['webUrl']
+        text = ''
         async with httpx.AsyncClient() as client:
             raw_html = await client.get(announce_url)
         soup = bs(raw_html, 'html.parser')
@@ -50,12 +50,15 @@ class Arknights(NewMessage, NoTargetMixin):
             render = Render()
             viewport = {'width': 320, 'height': 6400, 'deviceScaleFactor': 3}
             pic_data = await render.render(announce_url, viewport=viewport, target='div.main')
-            pics.append(pic_data)
+            if pic_data:
+                pics.append(pic_data)
+            else:
+                text = '图片渲染失败'
         elif (pic := soup.find('img', class_='banner-image')):
             pics.append(pic['src'])
         else:
             raise CategoryNotSupport()
-        return Post('arknights', text='', url='', target_name="明日方舟游戏内公告", pics=pics, compress=True, override_use_pic=False)
+        return Post('arknights', text=text, url='', target_name="明日方舟游戏内公告", pics=pics, compress=True, override_use_pic=False)
 
 class AkVersion(NoTargetMixin, StatusChange):
 
@@ -82,9 +85,13 @@ class AkVersion(NoTargetMixin, StatusChange):
     def compare_status(self, _, old_status, new_status):
         res = []
         if old_status.get('preAnnounceType') == 2 and new_status.get('preAnnounceType') == 0:
-            res.append(Post('arknights', text='开始维护！', target_name='明日方舟更新信息'))
+            res.append(Post('arknights',
+                text='登录界面维护公告上线（大概是开始维护了)',
+                target_name='明日方舟更新信息'))
         elif old_status.get('preAnnounceType') == 0 and new_status.get('preAnnounceType') == 2:
-            res.append(Post('arknights', text='维护结束！冲！（可能不太准确）', target_name='明日方舟更新信息'))
+            res.append(Post('arknights',
+                text='登录界面维护公告下线（大概是开服了，冲！）',
+                target_name='明日方舟更新信息'))
         if old_status.get('clientVersion') != new_status.get('clientVersion'):
             res.append(Post('arknights', text='游戏本体更新（大更新）', target_name='明日方舟更新信息'))
         if old_status.get('resVersion') != new_status.get('resVersion'):
@@ -96,3 +103,45 @@ class AkVersion(NoTargetMixin, StatusChange):
 
     async def parse(self, raw_post):
         return raw_post
+
+class MonsterSiren(NewMessage, NoTargetMixin):
+
+    categories = {3: '塞壬唱片新闻'}
+    platform_name = 'arknights'
+    name = '明日方舟游戏信息'
+    enable_tag = False
+    enabled = True
+    is_common = False
+    schedule_type = 'interval'
+    schedule_kw = {'seconds': 30}
+
+    async def get_target_name(self, _: Target) -> str:
+        return '明日方舟游戏信息'
+
+    async def get_sub_list(self, _) -> list[RawPost]:
+        async with httpx.AsyncClient() as client:
+            raw_data = await client.get('https://monster-siren.hypergryph.com/api/news')
+            return raw_data.json()['data']['list']
+
+    def get_id(self, post: RawPost) -> Any:
+        return post['cid']
+
+    def get_date(self, _) -> None:
+        return None
+
+    def get_category(self, _) -> Category:
+        return Category(3)
+
+    async def parse(self, raw_post: RawPost) -> Post:
+        url = f'https://monster-siren.hypergryph.com/info/{raw_post["cid"]}'
+        async with httpx.AsyncClient() as client:
+            res = await client.get(f'https://monster-siren.hypergryph.com/api/news/{raw_post["cid"]}')
+            raw_data = res.json()
+            content = raw_data['data']['content']
+            content = content.replace('</p>', '</p>\n')
+            soup = bs(content, 'html.parser')
+            imgs = list(map(lambda x: x['src'], soup('img')))
+            text = f'{raw_post["title"]}\n{soup.text.strip()}'
+        return Post('monster-siren', text=text, pics=imgs,
+                url=url, target_name="塞壬唱片新闻", compress=True,
+                override_use_pic=False)

src/plugins/nonebot_bison/platform/platform.py

@@ -134,7 +134,7 @@ class MessageProcessMixin(PlatformNameMixin, CategoryMixin, ParsePostMixin, abst
             # if post_id in exists_posts_set:
             #     continue
             if (post_time := self.get_date(raw_post)) and time.time() - post_time > 2 * 60 * 60 and \
-                    plugin_config.hk_reporter_init_filter:
+                    plugin_config.bison_init_filter:
                 continue
             try:
                 self.get_category(raw_post)
@@ -157,7 +157,7 @@ class NewMessageProcessMixin(StorageMixinProto, MessageProcessMixin, abstract=Tr
         filtered_post = await self.filter_common(raw_post_list)
         store = self.get_stored_data(target) or self.MessageStorage(False, set())
         res = []
-        if not store.inited and plugin_config.hk_reporter_init_filter:
+        if not store.inited and plugin_config.bison_init_filter:
             # target not init
             for raw_post in filtered_post:
                 post_id = self.get_id(raw_post)
@@ -241,7 +241,7 @@ class Platform(PlatformNameMixin, UserCustomFilterMixin, base=True):
         ...
 
 class NewMessage(
-        Platform, 
+        Platform,
         NewMessageProcessMixin,
         UserCustomFilterMixin,
         abstract=True
@@ -306,6 +306,33 @@ class StatusChange(
             logger.warning("network connection error: {}, url: {}".format(type(err), err.request.url))
             return []
 
+class SimplePost(
+        Platform,
+        MessageProcessMixin,
+        UserCustomFilterMixin,
+        StorageMixinProto,
+        abstract=True
+        ):
+    "Fetch a list of messages, dispatch it to different users"
+
+    async def fetch_new_post(self, target: Target, users: list[UserSubInfo]) -> list[tuple[User, list[Post]]]:
+        try:
+            new_posts = await self.get_sub_list(target)
+            if not new_posts:
+                return []
+            else:
+                for post in new_posts:
+                    logger.info('fetch new post from {} {}: {}'.format(
+                        self.platform_name,
+                        target if self.has_target else '-',
+                        self.get_id(post)))
+            res = await self.dispatch_user_post(target, new_posts, users)
+            self.parse_cache = {}
+            return res
+        except httpx.RequestError as err:
+            logger.warning("network connection error: {}, url: {}".format(type(err), err.request.url))
+            return []
+
 class NoTargetGroup(
         Platform,
         NoTargetMixin,

src/plugins/nonebot_bison/plugin_config.py

@@ -0,0 +1,22 @@
+from pydantic import BaseSettings
+
+import warnings
+import nonebot
+
+class PlugConfig(BaseSettings):
+
+    bison_config_path: str = ""
+    bison_use_pic: bool = False
+    bison_use_local: bool = False
+    bison_browser: str = ''
+    bison_init_filter: bool = True
+    bison_use_queue: bool = True
+    bison_outer_url: str = 'http://localhost:8080/bison/'
+
+    class Config:
+        extra = 'ignore'
+
+global_config = nonebot.get_driver().config
+plugin_config = PlugConfig(**global_config.dict())
+if plugin_config.bison_use_local:
+    warnings.warn('BISON_USE_LOCAL is deprecated, please use BISON_BROWSER')

src/plugins/nonebot_bison/post.py

@@ -29,7 +29,7 @@ class Post:
     def _use_pic(self):
         if not self.override_use_pic is None:
             return self.override_use_pic
-        return plugin_config.hk_reporter_use_pic
+        return plugin_config.bison_use_pic
 
     async def _pic_url_to_image(self, data: Union[str, bytes]) -> Image.Image:
         pic_buffer = BytesIO()
@@ -110,7 +110,10 @@ class Post:
             msgs = []
             text = ''
             if self.text:
-                text += '{}'.format(self.text if len(self.text) < 500 else self.text[:500] + '...')
+                if self._use_pic():
+                    text += '{}'.format(self.text)
+                else:
+                    text += '{}'.format(self.text if len(self.text) < 500 else self.text[:500] + '...')
             text += '\n来源: {}'.format(self.target_type)
             if self.target_name:
                 text += ' {}'.format(self.target_name)

src/plugins/nonebot_bison/scheduler.py

@@ -11,6 +11,7 @@ from .platform import platform_manager
 from .send import do_send_msgs
 from .send import send_msgs
 from .types import UserSubInfo
+from .plugin_config import plugin_config
 
 scheduler = AsyncIOScheduler()
 
@@ -43,7 +44,7 @@ async def fetch_and_send(target_type: str):
             if not bot:
                 logger.warning('no bot connected')
             else:
-                send_msgs(bot, user.user, user.user_type, await send_post.generate_messages())
+                await send_msgs(bot, user.user, user.user_type, await send_post.generate_messages())
 
 for platform_name, platform in platform_manager.items():
     if platform.schedule_type in ['cron', 'interval', 'date']:
@@ -52,16 +53,17 @@ for platform_name, platform in platform_manager.items():
                 fetch_and_send, platform.schedule_type, **platform.schedule_kw,
                 args=(platform_name,))
 
-scheduler.add_job(do_send_msgs, 'interval', seconds=0.3, coalesce=True)
+if plugin_config.bison_use_queue:
+    scheduler.add_job(do_send_msgs, 'interval', seconds=0.3, coalesce=True)
 
-class SchedulerLogFilter(logging.Filter):
+    class SchedulerLogFilter(logging.Filter):
 
-    def filter(self, record: logging.LogRecord) -> bool:
-        logger.debug("logRecord", record, record.getMessage())
-        return not (record.name == "apscheduler" and 'skipped: maximum number of running instances reached' in record.getMessage())
+        def filter(self, record: logging.LogRecord) -> bool:
+            logger.debug("logRecord", record, record.getMessage())
+            return not (record.name == "apscheduler" and 'skipped: maximum number of running instances reached' in record.getMessage())
 
-aps_logger = logging.getLogger("apscheduler")
-aps_logger.setLevel(30)
-aps_logger.addFilter(SchedulerLogFilter())
-aps_logger.handlers.clear()
-aps_logger.addHandler(LoguruHandler())
+    aps_logger = logging.getLogger("apscheduler")
+    aps_logger.setLevel(30)
+    aps_logger.addFilter(SchedulerLogFilter())
+    aps_logger.handlers.clear()
+    aps_logger.addHandler(LoguruHandler())

src/plugins/nonebot_bison/send.py

@@ -0,0 +1,43 @@
+import time
+
+from nonebot import logger
+from nonebot.adapters.cqhttp.bot import Bot
+
+from .plugin_config import plugin_config
+
+QUEUE = []
+LAST_SEND_TIME = time.time()
+
+async def _do_send(bot: 'Bot', user: str, user_type: str, msg):
+    if user_type == 'group':
+        await bot.call_api('send_group_msg', group_id=user, message=msg)
+    elif user_type == 'private':
+        await bot.call_api('send_private_msg', user_id=user, message=msg)
+
+async def do_send_msgs():
+    global LAST_SEND_TIME
+    if time.time() - LAST_SEND_TIME < 1.5:
+        return
+    if QUEUE:
+        bot, user, user_type, msg, retry_time = QUEUE.pop(0)
+        try:
+            await _do_send(bot, user, user_type, msg)
+        except Exception as e:
+            if retry_time > 0:
+                QUEUE.insert(0, (bot, user, user_type, msg, retry_time - 1))
+            else:
+                msg_str = str(msg)
+                if len(msg_str) > 50:
+                    msg_str = msg_str[:50] + '...'
+                logger.warning(f'send msg err {e} {msg_str}')
+        LAST_SEND_TIME = time.time()
+
+async def send_msgs(bot, user, user_type, msgs):
+    if plugin_config.bison_use_queue:
+        for msg in msgs:
+            QUEUE.append((bot, user, user_type, msg, 2))
+    else:
+        for msg in msgs:
+            await _do_send(bot, user, user_type, msg)
+
+

src/plugins/nonebot_bison/utils.py

@@ -1,15 +1,20 @@
 import asyncio
 import base64
 from html import escape
+import os
+from time import asctime
+import re
 from typing import Awaitable, Callable, Optional
-from urllib.parse import quote
-from nonebot.adapters.cqhttp.message import MessageSegment
 
+from nonebot.adapters.cqhttp.message import MessageSegment
 from nonebot.log import logger
 from pyppeteer import connect, launch
 from pyppeteer.browser import Browser
+from pyppeteer.chromium_downloader import check_chromium, download_chromium
 from pyppeteer.page import Page
 
+from bs4 import BeautifulSoup as bs
+
 from .plugin_config import plugin_config
 
 class Singleton(type):
@@ -19,6 +24,10 @@ class Singleton(type):
             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]
 
+if not plugin_config.bison_browser and not plugin_config.bison_use_local \
+        and not check_chromium():
+    os.environ['PYPPETEER_DOWNLOAD_HOST'] = 'http://npm.taobao.org/mirrors'
+    download_chromium()
 
 class Render(metaclass=Singleton):
 
@@ -29,15 +38,15 @@ class Render(metaclass=Singleton):
         self.remote_browser = False
 
     async def get_browser(self) -> Browser:
-        if plugin_config.hk_reporter_browser:
-            if plugin_config.hk_reporter_browser.startswith('local:'):
-                path = plugin_config.hk_reporter_browser.split('local:', 1)[1]
+        if plugin_config.bison_browser:
+            if plugin_config.bison_browser.startswith('local:'):
+                path = plugin_config.bison_browser.split('local:', 1)[1]
                 return await launch(executablePath=path, args=['--no-sandbox'])
-            if plugin_config.hk_reporter_browser.startswith('ws:'):
+            if plugin_config.bison_browser.startswith('ws:'):
                 self.remote_browser = True
-                return await connect(browserWSEndpoint=plugin_config.hk_reporter_browser)
-            raise RuntimeError('HK_REPORTER_BROWSER error')
-        if plugin_config.hk_reporter_use_local:
+                return await connect(browserWSEndpoint=plugin_config.bison_browser)
+            raise RuntimeError('bison_BROWSER error')
+        if plugin_config.bison_use_local:
             return await launch(executablePath='/usr/bin/chromium', args=['--no-sandbox'])
         return await launch(args=['--no-sandbox'])
 
@@ -60,8 +69,7 @@ class Render(metaclass=Singleton):
                 #     self.lock.release()
 
     def _inter_log(self, message: str) -> None:
-        # self.interval_log += asctime() + '' + message + '\n'
-        logger.debug(message)
+        self.interval_log += asctime() + '' + message + '\n'
 
     async def do_render(self, url: str, viewport: Optional[dict] = None, target: Optional[str] = None,
             operation: Optional[Callable[[Page], Awaitable[None]]] = None) -> Optional[bytes]:
@@ -111,8 +119,20 @@ class Render(metaclass=Singleton):
 
 async def parse_text(text: str) -> MessageSegment:
     'return raw text if don\'t use pic, otherwise return rendered opcode'
-    if plugin_config.hk_reporter_use_pic:
+    if plugin_config.bison_use_pic:
         render = Render()
         return await render.text_to_pic_cqcode(text)
     else:
         return MessageSegment.text(text)
+
+def html_to_text(html: str, query_dict: dict = {}) -> str:
+    html = re.sub(r'<br\s*/?>', '<br>\n', html)
+    html = html.replace('</p>', '</p>\n')
+    soup = bs(html, 'html.parser')
+    if query_dict:
+        node = soup.find(**query_dict)
+    else:
+        node = soup
+    assert node is not None
+    return node.text.strip()
+

src/plugins/nonebot_hk_reporter/platform/monster_siren.py

@@ -1,38 +0,0 @@
-from typing import Any
-import httpx
-
-from .platform import NewMessage, NoTargetMixin
-from ..types import RawPost
-from ..post import Post
-
-class MonsterSiren(NewMessage, NoTargetMixin):
-
-    categories = {}
-    platform_name = 'monster-siren'
-    enable_tag = False
-    enabled = True
-    is_common = False
-    schedule_type = 'interval'
-    schedule_kw = {'seconds': 30}
-    name = '塞壬唱片官网新闻'
-
-    @staticmethod
-    async def get_target_name(_) -> str:
-        return '塞壬唱片新闻'
-
-    async def get_sub_list(self, _) -> list[RawPost]:
-        async with httpx.AsyncClient() as client:
-            raw_data = await client.get('https://monster-siren.hypergryph.com/api/news')
-            return raw_data.json()['data']['list']
-
-    def get_id(self, post: RawPost) -> Any:
-        return post['cid']
-
-    def get_date(self, _) -> None:
-        return None
-
-    async def parse(self, raw_post: RawPost) -> Post:
-        url = f'https://monster-siren.hypergryph.com/info/{raw_post["cid"]}'
-        return Post('monster-siren', text=raw_post['title'],
-                url=url, target_name="塞壬唱片新闻", compress=True,
-                override_use_pic=False)

src/plugins/nonebot_hk_reporter/plugin_config.py

@@ -1,21 +0,0 @@
-from pydantic import BaseSettings
-
-import warnings
-import nonebot
-
-class PlugConfig(BaseSettings):
-
-    hk_reporter_config_path: str = ""
-    hk_reporter_use_pic: bool = False
-    hk_reporter_use_local: bool = False
-    hk_reporter_browser: str = ''
-    hk_reporter_init_filter: bool = True
-    hk_reporter_outer_url: str = 'http://localhost:8080/hk_reporter/'
-
-    class Config:
-        extra = 'ignore'
-
-global_config = nonebot.get_driver().config
-plugin_config = PlugConfig(**global_config.dict())
-if plugin_config.hk_reporter_use_local:
-    warnings.warn('HK_REPORTER_USE_LOCAL is deprecated, please use HK_REPORTER_BROWSER')

src/plugins/nonebot_hk_reporter/send.py

@@ -1,30 +0,0 @@
-from nonebot import logger
-import time
-
-QUEUE = []
-LAST_SEND_TIME = time.time()
-
-
-async def do_send_msgs():
-    global LAST_SEND_TIME
-    if time.time() - LAST_SEND_TIME < 1.5:
-        return
-    if QUEUE:
-        bot, user, user_type, msg, retry_time = QUEUE.pop(0)
-        try:
-            if user_type == 'group':
-                await bot.call_api('send_group_msg', group_id=user, message=msg)
-            elif user_type == 'private':
-                await bot.call_api('send_private_msg', user_id=user, message=msg)
-        except:
-            if retry_time > 0:
-                QUEUE.insert(0, (bot, user, user_type, msg, retry_time - 1))
-            else:
-                logger.warning('send msg err {}'.format(msg))
-        LAST_SEND_TIME = time.time()
-
-def send_msgs(bot, user, user_type, msgs):
-    for msg in msgs:
-        QUEUE.append((bot, user, user_type, msg, 2))
-
-