diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml index 9c14210..94de36a 100644 --- a/.github/actions/setup-python/action.yml +++ b/.github/actions/setup-python/action.yml @@ -18,23 +18,14 @@ runs: - name: Install poetry uses: Gr1N/setup-poetry@v7 - - name: Cache Windows dependencies - uses: actions/cache@v2 - if: ${{ runner.os == 'Windows' }} - with: - path: ~/AppData/Local/pypoetry/Cache/virtualenvs - key: ${{ runner.os }}-poetry-${{ inputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + - id: poetry-cache + run: echo "::set-output name=dir::$(poetry config virtualenvs.path)" + shell: bash - - name: Cache Linux dependencies - uses: actions/cache@v2 - if: ${{ runner.os == 'Linux' }} + - uses: actions/cache@v2 with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-${{ inputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-poetry-${{ steps.python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - name: Cache macOS dependencies - uses: actions/cache@v2 - if: ${{ runner.os == 'macOS' }} - with: - path: ~/Library/Caches/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-${{ inputs.python-version }}-${{ hashFiles('**/poetry.lock') }} \ No newline at end of file + - run: poetry install + shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3725705..5f3fa00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,21 @@ on: push: branches: - main + paths: + - admin-frontend/** + - docker/** + - src/** + - tests/** + - pyproject.toml + - poetry.lock pull_request: + paths: + - admin-frontend/** + - docker/** + - src/** + - tests/** + - pyproject.toml + - poetry.lock concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/website-preview.yml b/.github/workflows/website-preview.yml index d6946b3..668a096 100644 --- a/.github/workflows/website-preview.yml +++ b/.github/workflows/website-preview.yml @@ -2,6 +2,10 @@ name: Site Deploy(Preview) on: pull_request_target: + paths: + - docs/** + - package.json + - yarn.lock jobs: preview: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e511ed..3ecf05f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,20 @@ ### 新功能 +- 添加bilibili开播提醒 [@Sichongzou](https://github.com/Sichongzou) ([#60](https://github.com/felinae98/nonebot-bison/pull/60)) +- 添加User-Agent配置 [@felinae98](https://github.com/felinae98) ([#78](https://github.com/felinae98/nonebot-bison/pull/78)) - 增加代理设置 [@felinae98](https://github.com/felinae98) ([#71](https://github.com/felinae98/nonebot-bison/pull/71)) - 增加Parse Target功能 [@felinae98](https://github.com/felinae98) ([#72](https://github.com/felinae98/nonebot-bison/pull/72)) +### Bug 修复 + +- 捕获 JSONDecodeError [@felinae98](https://github.com/felinae98) ([#82](https://github.com/felinae98/nonebot-bison/pull/82)) +- 捕获SSL异常 [@felinae98](https://github.com/felinae98) ([#75](https://github.com/felinae98/nonebot-bison/pull/75)) + +### 文档 + +- 完善开发文档 [@AzideCupric](https://github.com/AzideCupric) ([#80](https://github.com/felinae98/nonebot-bison/pull/80)) + ## v0.5.3 - on_command 设置 block=True (#63) @MeetWq diff --git a/README.md b/README.md index 473c664..feb59d8 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ - 专栏 - 转发 - 纯文字 +- Bilibili 直播 + - 开播提醒 - RSS - 富文本转换为纯文本 - 提取出所有图片 diff --git a/docs/dev/README.md b/docs/dev/README.md index 945e47c..f18891c 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -31,6 +31,11 @@ sidebar: auto 本插件需要你的帮助！只需要会写简单的爬虫，就能给本插件适配新的网站。 +::: danger +Nonebot 项目使用了全异步的处理方式，所以你需要对异步，Python asyncio 的机制有一定了解，当然， +依葫芦画瓢也是足够的 +::: + ## 基本概念 - `nonebot_bison.post.Post`: 可以理解为推送内容，其中包含需要发送的文字，图片，链接，平台信息等 @@ -53,7 +58,7 @@ sidebar: auto 例如：微博，Bilibili - `nonebot_bison.platform.platform.StatusChange` 每次爬虫获取一个状态，在状态改变时发布推送 例如：游戏开服提醒，主播上播提醒 -- `nonebot_bison.platform.platform.SimplePost` 与`NewMessage`相似，但是不过滤新的消息 +- `nonebot_bison.platform.platform.SimplePost` 与`NewMessage`相似，但是不过滤之前发过的 ，每次发送全部消息 例如：每日榜单定时发送 @@ -64,10 +69,61 @@ sidebar: auto - 没有账号的概念 例如：游戏公告，教务处公告 +## 实现方法 + 现在你需要在`src/plugins/nonebot_bison/platform`下新建一个 py 文件， 在里面新建一个类，继承推送类型的基类，重载一些关键的函数，然后……就完成了，不需要修改别的东西了。 -任何一种订阅类型需要实现的方法/字段如下： +### 不同类型 Platform 的实现适配以及逻辑 + +- `nonebot_bison.platform.platform.NewMessage` + 需要实现： + + - `async get_sub_list(Target) -> list[RawPost]` + - `get_id(RawPost)` + - `get_date(RawPost)` (可选) + + ::: details 大致流程 + + 1. 调用`get_sub_list`拿到 RawPost 列表 + 2. 调用`get_id`判断是否重复，如果没有重复就说明是新的 RawPost + 3. 如果有`get_category`和`get_date`，则调用判断 RawPost 是否满足条件 + 4. 调用`parse`生成正式推文 + ::: + + 参考[nonebot_bison.platform.Weibo](https://github.com/felinae98/nonebot-bison/blob/v0.5.3/src/plugins/nonebot_bison/platform/weibo.py) + +- `nonebot_bison.platform.platform.StatusChange` + 需要实现： + + - `async get_status(Target) -> Any` + - `compare_status(Target, old_status, new_status) -> list[RawPost]` + + :::details 大致流程 + + 1. `get_status`获取当前状态 + 2. 传入`compare_status`比较前状态 + 3. 通过则进入`parser`生成 Post + ::: + + 参考[nonenot_bison.platform.AkVersion](https://github.com/felinae98/nonebot-bison/blob/v0.5.3/src/plugins/nonebot_bison/platform/arknights.py#L86) + +- `nonebot_bison.platform.platform.SimplePost` + 需要实现： + + - `async get_sub_list(Target) -> list[RawPost]` + - `get_date(RawPost)` (可选) + + ::: details 大致流程 + + 1. 调用`get_sub_list`拿到 RawPost 列表 + 2. 如果有`get_category`和`get_date`，则调用判断 RawPost 是否满足条件 + 3. 调用`parse`生成正式推文 + ::: + +### 公共方法/成员 + +任何一种订阅类型需要实现的方法/成员如下： - `schedule_type`, `schedule_kw` 调度的参数，本质是使用 apscheduler 的[trigger 参数](https://apscheduler.readthedocs.io/en/3.x/userguide.html?highlight=trigger#choosing-the-right-scheduler-job-store-s-executor-s-and-trigger-s)，`schedule_type`可以是`date`,`interval`和`cron`， `schedule_kw`是对应的参数，一个常见的配置是`schedule_type=interval`, `schedule_kw={'seconds':30}` @@ -80,11 +136,40 @@ sidebar: auto - `enable_tag` 平台发布内容是否带 Tag，例如微博 - `platform_name` 唯一的，英文的识别标识，比如`weibo` - `async get_target_name(Target) -> Optional[str]` 通常用于获取帐号的名称，如果平台没有帐号概念，可以直接返回平台的`name` -- `async parse(RawPost) -> Post`将获取到的 RawPost 处理成 Post - `get_tags(RawPost) -> Optional[Collection[Tag]]` （可选） 从 RawPost 中提取 Tag - `get_category(RawPos) -> Optional[Category]` （可选）从 RawPost 中提取 Category +- `async parse(RawPost) -> Post` 将获取到的 RawPost 处理成 Post +- `async parse_target(str) -> Target` （可选）定制化处理传入用户输入的 Target 字符串，返回 Target（一般是把用户的主页链接解析为 Target），如果输入本身就是 Target，则直接返回 Target +- `parse_target_promot` （可选）在要求用户输入 Target 的时候显示的提示文字 -例如要适配微博，我希望 bot 搬运新的消息，所以微博的类应该这样定义： +### 特有的方法/成员 + +- `async get_sub_list(Target) -> list[RawPost]` 输入一个`Target`，输出一个`RawPost`的 list + - 对于`nonebot_bison.platform.platform.NewMessage` + `get_sub_list(Target) -> list[RawPost]` 用于获取对应 Target 的 RawPost 列表，与上一次`get_sub_list`获取的列表比较，过滤出新的 RawPost + - 对于`nonebot_bison.platform.platform.SimplePost` + `get_sub_list` 用于获取对应 Target 的 RawPost 列表，但不会与上次获取的结果进行比较，而是直接进行发送 +- `get_id(RawPost) -> Any` 输入一个`RawPost`，从`RawPost`中获取一个唯一的 ID，这个 ID 会用来判断这条`RawPost`是不是之前收到过 +- `get_date(RawPost) -> Optional[int]` 输入一个`RawPost`，如果可以从`RawPost`中提取出发文的时间，返回发文时间的 timestamp，否则返回`None` +- `async get_status(Target) -> Any` + - 对于`nonebot_bison.platform.platform.StatusChange` + `get_status`用于获取对应 Target 当前的状态，随后将获取的状态作为参数`new_status`传入`compare_status`中 +- `compare_status(self, target: Target, old_status, new_status) -> list[RawPost]` + - 对于`nonebot_bison.platform.platform.StatusChange` + `compare_status` 用于比较储存的`old_status`与新传入的`new_status`，并返回发生变更的 RawPost 列表 + +### 单元测试 + +当然我们非常希望你对自己适配的平台写一些单元测试 + +你可以参照`tests/platforms/test_*.py`中的内容对单元测试进行编写。 + +为保证多次运行测试的一致性，可以 mock http 的响应，测试的内容应包括[获取 RawPost](https://github.com/felinae98/nonebot-bison/blob/v0.5.3/tests/platforms/test_weibo.py#L59)，处理成 Post +，测试分类以及提取 tag 等，当然最好和 rsshub 做一个交叉验证。 + +## 一些例子 + +例如要适配微博，我希望 bot 搬运新的消息，所以微博的类应该这样实现： ```python class Weibo(NewMessage): @@ -103,17 +188,26 @@ class Weibo(NewMessage): schedule_type = "interval" schedule_kw = {"seconds": 3} has_target = True + + async def get_target_name(self, target: Target) -> Optional[str]: + #获取Target对应的用户名 + ... + async def get_sub_list(self, target: Target) -> list[RawPost]: + #获取对应Target的RawPost列表，会与上一次get_sub_list获取的列表比较，过滤出新的RawPost + ... + def get_id(self, post: RawPost) -> Any: + #获取可以标识每个Rawpost的，不与之前RawPost重复的id，用于过滤出新的RawPost + ... + def get_date(self, raw_post: RawPost) -> float: + #获取RawPost的发布时间，若bot过滤出的新RawPost发布时间与当前时间差超过2小时，该RawPost将被忽略，可以返回None + ... + def get_tags(self, raw_post: RawPost) -> Optional[list[Tag]]: + #获取RawPost中包含的微博话题（#xxx#中的内容） + ... + def get_category(self, raw_post: RawPost) -> Category: + #获取该RawPost在该类定义categories的具体分类(转发？视频？图文？...？) + ... + async def parse(self, raw_post: RawPost) -> Post: + #将需要bot推送的RawPost处理成正式推送的Post + ... ``` - -当然我们非常希望你对自己适配的平台写一些单元测试，你可以模仿`tests/platforms/test_*.py`中的内容写 -一些单元测试。为保证多次运行测试的一致性，可以 mock http 的响应，测试的内容包括获取 RawPost，处理成 Post -，测试分类以及提取 tag 等，当然最好和 rsshub 做一个交叉验证。 - -::: danger -Nonebot 项目使用了全异步的处理方式，所以你需要对异步，Python asyncio 的机制有一定了解，当然， -依葫芦画瓢也是足够的 -::: - -## 类的方法与成员变量 - -## 方法与变量的定义 diff --git a/docs/usage/README.md b/docs/usage/README.md index 647da48..39bf8e6 100644 --- a/docs/usage/README.md +++ b/docs/usage/README.md @@ -94,23 +94,27 @@ sidebar: auto ## 配置 -可参考[源文件](https://github.com/felinae98/nonebot-bison/blob/main/src/plugins/nonebot_bison/plugin_config.py) +::: tip INFO + +- 所有配置项可参考[源文件](https://github.com/felinae98/nonebot-bison/blob/main/src/plugins/nonebot_bison/plugin_config.py) +- **配置项的配置方法** 请参考[NoneBot 配置方式](https://v2.nonebot.dev/docs/tutorial/configuration#%E9%85%8D%E7%BD%AE%E6%96%B9%E5%BC%8F)，在`.env`/`.env.*`文件中写入希望配置的 Bison 配置项 + ::: - `BISON_CONFIG_PATH`: 插件存放配置文件的位置，如果不设定默认为项目目录下的`data`目录 - `BISON_USE_PIC`: 将文字渲染成图片后进行发送，多用于规避风控 - `BISON_BROWSER`: 本插件使用 Chrome 来渲染图片 - - 使用 browserless 提供的 Chrome 管理服务，设置为`ws://xxxxxxxx`，值为 Chrome Endpoint（推荐） - - 使用 cdp 连接相关服务，设置为`wsc://xxxxxxxxx` - - 使用本地安装的 Chrome，设置为`local:`，例如`local:/usr/bin/google-chrome-stable` - 如果不进行配置，那么会在启动时候自动进行安装，在官方的 docker 镜像中已经安装了浏览器 + - 使用本地安装的 Chrome，设置为`local:`，例如`local:/usr/bin/google-chrome-stable` + - 使用 cdp 连接相关服务，设置为`wsc://xxxxxxxxx` + - 使用 browserless 提供的 Chrome 管理服务，设置为`ws://xxxxxxxx`，值为 Chrome Endpoint ::: warning 截止发布时，本项目尚不能完全与 browserless 兼容，目前建议使用镜像内自带的浏览器，即 不要配置这个变量 ::: - `BISON_SKIP_BROWSER_CHECK`: 是否在启动时自动下载浏览器，如果选择`False`会在用到浏览器时自动下载， 默认`True` -- `BISON_OUTER_URL`: 从外部访问服务器的地址，默认为`http://localhost:8080/bison`，如果你的插件部署 - 在服务器上，建议配置为`http://<你的服务器ip>:8080/bison` +- `BISON_OUTER_URL`: 从外部访问服务器的地址，默认为`http://localhost:8080/bison/`，如果你的插件部署 + 在服务器上，建议配置为`http://<你的服务器ip>:8080/bison/` ::: warning 如果需要从外网或者 Docker 容器外访问后台页面，请确保`HOST=0.0.0.0` ::: @@ -124,7 +128,7 @@ sidebar: auto - `1`: 首条消息单独发送，剩余图片合并转发 - `2`: 所有消息全部合并转发 - ::: details 配置项示例 + ::: details BISON_USE_PIC_MERGE 配置项示例 - 当`BISON_USE_PIC_MERGE=1`时: ![simple1](/images/forward-msg-simple1.png) @@ -137,6 +141,7 @@ sidebar: auto ::: - `BISON_PROXY`: 使用的代理连接，形如`http://:`（可选） +- `BISON_UA`: 使用的 User-Agent，默认为 Chrome ## 使用 @@ -154,12 +159,16 @@ sidebar: auto 所有命令都需要@bot 触发 - 添加订阅（仅管理员和群主和 SUPERUSER）：`添加订阅` - ::: tip 关于中止订阅 - 对于[**v0.5.1**](https://github.com/felinae98/nonebot-bison/releases/tag/v0.5.1)及以上的版本中，已经为`添加订阅`命令添加了中止订阅的功能。 - 在添加订阅命令的~~几乎~~各个阶段，都可以向 Bot 发送`取消`消息来中止订阅过程(需要订阅发起者本人发送) + ::: details 关于中止添加订阅 + 对于[**v0.5.1**](https://github.com/felinae98/nonebot-bison/releases/tag/v0.5.1)及以上的版本中，已经为`添加订阅`命令添加了中止添加功能。 + 在`添加订阅`命令的~~几乎~~各个阶段，都可以向 Bot 发送`取消`消息来中止订阅过程(需要发起者本人发送) ::: - 查询订阅：`查询订阅` - 删除订阅（仅管理员和群主和 SUPERUSER）：`删除订阅` + ::: details 关于中止删除订阅 + 对于[**v0.5.3**](https://github.com/felinae98/nonebot-bison/releases/tag/v0.5.3)及以上的版本中，已经为`删除订阅`命令添加了中止删除功能。 + 在`删除订阅`命令的~~几乎~~各个阶段，都可以向 Bot 发送`取消`消息来中止订阅过程(需要发起者本人发送) + ::: #### 私聊机器人获取后台地址 @@ -178,8 +187,8 @@ sidebar: auto #### 私聊机器人进行配置（需要 SUPERUER 权限） 请私聊 bot`群管理` -::: tip 关于中止订阅 -与普通的[`添加订阅`](#在本群中进行配置)命令一样，在`群管理`命令中使用的`添加订阅`命令也可以使用`取消`来中止订阅过程 +::: details 关于中止订阅 +与普通的[`添加订阅`/`删除订阅`](#在本群中进行配置)命令一样，在`群管理`命令中使用的`添加订阅`/`删除订阅`命令也可以使用`取消`来中止订阅过程 ::: ### 所支持平台的 uid diff --git a/src/plugins/nonebot_bison/platform/bilibili.py b/src/plugins/nonebot_bison/platform/bilibili.py index c880a46..56afd42 100644 --- a/src/plugins/nonebot_bison/platform/bilibili.py +++ b/src/plugins/nonebot_bison/platform/bilibili.py @@ -5,7 +5,7 @@ from typing import Any, Optional from ..post import Post from ..types import Category, RawPost, Tag, Target from ..utils import http_client -from .platform import CategoryNotSupport, NewMessage +from .platform import CategoryNotSupport, NewMessage, StatusChange class Bilibili(NewMessage): @@ -155,3 +155,73 @@ class Bilibili(NewMessage): else: raise CategoryNotSupport(post_type) return Post("bilibili", text=text, url=url, pics=pic, target_name=target_name) + + +class Bilibililive(StatusChange): + # Author : Sichongzou + # Date : 2022-5-18 8:54 + # Description : bilibili开播提醒 + # E-mail : 1557157806@qq.com + categories = {} + platform_name = "bilibili-live" + enable_tag = True + enabled = True + is_common = True + schedule_type = "interval" + schedule_kw = {"seconds": 10} + name = "Bilibili直播" + has_target = True + + async def get_target_name(self, target: Target) -> Optional[str]: + async with http_client() as client: + res = await client.get( + "https://api.bilibili.com/x/space/acc/info", params={"mid": target} + ) + res_data = json.loads(res.text) + if res_data["code"]: + return None + return res_data["data"]["name"] + + async def get_status(self, target: Target): + async with http_client() as client: + params = {"mid": target} + res = await client.get( + "https://api.bilibili.com/x/space/acc/info", + params=params, + timeout=4.0, + ) + res_dict = json.loads(res.text) + if res_dict["code"] == 0: + info = {} + info["uid"] = res_dict["data"]["mid"] + info["uname"] = res_dict["data"]["name"] + info["live_state"] = res_dict["data"]["live_room"]["liveStatus"] + info["room_id"] = res_dict["data"]["live_room"]["roomid"] + info["title"] = res_dict["data"]["live_room"]["title"] + info["cover"] = res_dict["data"]["live_room"]["cover"] + return info + else: + return [] + + def compare_status(self, target: Target, old_status, new_status) -> list[RawPost]: + if ( + new_status["live_state"] != old_status["live_state"] + and new_status["live_state"] == 1 + ): + return [new_status] + else: + return [] + + async def parse(self, raw_post: RawPost) -> Post: + url = "https://live.bilibili.com/{}".format(raw_post["room_id"]) + pic = [raw_post["cover"]] + target_name = raw_post["uname"] + title = raw_post["title"] + return Post( + self.name, + text=title, + url=url, + pics=pic, + target_name=target_name, + compress=True, + ) diff --git a/src/plugins/nonebot_bison/platform/platform.py b/src/plugins/nonebot_bison/platform/platform.py index 19f002d..22c1200 100644 --- a/src/plugins/nonebot_bison/platform/platform.py +++ b/src/plugins/nonebot_bison/platform/platform.py @@ -1,3 +1,5 @@ +import json +import ssl import time from abc import ABC, abstractmethod from collections import defaultdict @@ -59,6 +61,25 @@ class Platform(metaclass=RegistryABCMeta, base=True): ) -> list[tuple[User, list[Post]]]: ... + async def do_fetch_new_post( + self, target: Target, users: list[UserSubInfo] + ) -> list[tuple[User, list[Post]]]: + try: + return await self.fetch_new_post(target, users) + except httpx.RequestError as err: + logger.warning( + "network connection error: {}, url: {}".format( + type(err), err.request.url + ) + ) + return [] + except ssl.SSLError as err: + logger.warning(f"ssl error: {err}") + return [] + except json.JSONDecodeError as err: + logger.warning(f"json error, parsing: {err.doc}") + return [] + @abstractmethod async def parse(self, raw_post: RawPost) -> Post: ... @@ -226,30 +247,22 @@ class NewMessage(MessageProcess, abstract=True): async def fetch_new_post( self, target: Target, users: list[UserSubInfo] ) -> list[tuple[User, list[Post]]]: - try: - post_list = await self.get_sub_list(target) - new_posts = await self.filter_common_with_diff(target, post_list) - if not new_posts: - return [] - else: - for post in new_posts: - logger.info( - "fetch new post from {} {}: {}".format( - self.platform_name, - target if self.has_target else "-", - self.get_id(post), - ) - ) - res = await self.dispatch_user_post(target, new_posts, users) - self.parse_cache = {} - return res - except httpx.RequestError as err: - logger.warning( - "network connection error: {}, url: {}".format( - type(err), err.request.url - ) - ) + post_list = await self.get_sub_list(target) + new_posts = await self.filter_common_with_diff(target, post_list) + if not new_posts: return [] + else: + for post in new_posts: + logger.info( + "fetch new post from {} {}: {}".format( + self.platform_name, + target if self.has_target else "-", + self.get_id(post), + ) + ) + res = await self.dispatch_user_post(target, new_posts, users) + self.parse_cache = {} + return res class StatusChange(Platform, abstract=True): @@ -270,30 +283,22 @@ class StatusChange(Platform, abstract=True): async def fetch_new_post( self, target: Target, users: list[UserSubInfo] ) -> list[tuple[User, list[Post]]]: - try: - new_status = await self.get_status(target) - res = [] - if old_status := self.get_stored_data(target): - diff = self.compare_status(target, old_status, new_status) - if diff: - logger.info( - "status changes {} {}: {} -> {}".format( - self.platform_name, - target if self.has_target else "-", - old_status, - new_status, - ) + new_status = await self.get_status(target) + res = [] + if old_status := self.get_stored_data(target): + diff = self.compare_status(target, old_status, new_status) + if diff: + logger.info( + "status changes {} {}: {} -> {}".format( + self.platform_name, + target if self.has_target else "-", + old_status, + new_status, ) - res = await self.dispatch_user_post(target, diff, users) - self.set_stored_data(target, new_status) - return res - except httpx.RequestError as err: - logger.warning( - "network connection error: {}, url: {}".format( - type(err), err.request.url ) - ) - return [] + res = await self.dispatch_user_post(target, diff, users) + self.set_stored_data(target, new_status) + return res class SimplePost(MessageProcess, abstract=True): @@ -302,29 +307,21 @@ class SimplePost(MessageProcess, abstract=True): async def fetch_new_post( self, target: Target, users: list[UserSubInfo] ) -> list[tuple[User, list[Post]]]: - try: - new_posts = await self.get_sub_list(target) - if not new_posts: - return [] - else: - for post in new_posts: - logger.info( - "fetch new post from {} {}: {}".format( - self.platform_name, - target if self.has_target else "-", - self.get_id(post), - ) - ) - res = await self.dispatch_user_post(target, new_posts, users) - self.parse_cache = {} - return res - except httpx.RequestError as err: - logger.warning( - "network connection error: {}, url: {}".format( - type(err), err.request.url - ) - ) + new_posts = await self.get_sub_list(target) + if not new_posts: return [] + else: + for post in new_posts: + logger.info( + "fetch new post from {} {}: {}".format( + self.platform_name, + target if self.has_target else "-", + self.get_id(post), + ) + ) + res = await self.dispatch_user_post(target, new_posts, users) + self.parse_cache = {} + return res class NoTargetGroup(Platform, abstract=True): diff --git a/src/plugins/nonebot_bison/plugin_config.py b/src/plugins/nonebot_bison/plugin_config.py index dfd43f3..0a539f5 100644 --- a/src/plugins/nonebot_bison/plugin_config.py +++ b/src/plugins/nonebot_bison/plugin_config.py @@ -18,6 +18,7 @@ class PlugConfig(BaseSettings): # 0：不启用；1：首条消息单独发送，剩余照片合并转发；2以及以上：所有消息全部合并转发 bison_resend_times: int = 0 bison_proxy: Optional[str] + bison_ua: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" class Config: extra = "ignore" diff --git a/src/plugins/nonebot_bison/scheduler.py b/src/plugins/nonebot_bison/scheduler.py index 0219f74..64ec4e4 100644 --- a/src/plugins/nonebot_bison/scheduler.py +++ b/src/plugins/nonebot_bison/scheduler.py @@ -58,7 +58,7 @@ async def fetch_and_send(target_type: str): send_user_list, ) ) - to_send = await platform_manager[target_type].fetch_new_post( + to_send = await platform_manager[target_type].do_fetch_new_post( target, send_userinfo_list ) if not to_send: diff --git a/src/plugins/nonebot_bison/utils/http.py b/src/plugins/nonebot_bison/utils/http.py index f46af30..082aa55 100644 --- a/src/plugins/nonebot_bison/utils/http.py +++ b/src/plugins/nonebot_bison/utils/http.py @@ -4,9 +4,8 @@ import httpx from ..plugin_config import plugin_config -if plugin_config.bison_proxy: - http_client = functools.partial( - httpx.AsyncClient, proxies=plugin_config.bison_proxy - ) -else: - http_client = httpx.AsyncClient +http_client = functools.partial( + httpx.AsyncClient, + proxies=plugin_config.bison_proxy or None, + headers={"user-agent": plugin_config.bison_ua}, +) diff --git a/tests/platforms/static/bili_live_status.json b/tests/platforms/static/bili_live_status.json new file mode 100644 index 0000000..e907dc9 --- /dev/null +++ b/tests/platforms/static/bili_live_status.json @@ -0,0 +1,114 @@ +{ + "code": 0, + "message": "0", + "ttl": 1, + "data": { + "mid": 13164144, + "name": "魔法Zc目录", + "sex": "男", + "face": "http://i0.hdslb.com/bfs/face/a84fa10f90f7060d0336384954ee1cde7a8e9bc6.jpg", + "face_nft": 0, + "sign": "每日18:00~22:00欢乐直播！请勿在任何乌有相关内容中刷Zc，尊重角色；商务合作qq271374252", + "rank": 10000, + "level": 6, + "jointime": 0, + "moral": 0, + "silence": 0, + "coins": 0, + "fans_badge": true, + "fans_medal": { + "show": false, + "wear": false, + "medal": null + }, + "official": { + "role": 1, + "title": "bilibili 2021百大UP主、知名游戏UP主、直播高能主播", + "desc": "", + "type": 0 + }, + "vip": { + "type": 2, + "status": 1, + "due_date": 1702051200000, + "vip_pay_type": 0, + "theme_type": 0, + "label": { + "path": "", + "text": "年度大会员", + "label_theme": "annual_vip", + "text_color": "#FFFFFF", + "bg_style": 1, + "bg_color": "#FB7299", + "border_color": "" + }, + "avatar_subscript": 1, + "nickname_color": "#FB7299", + "role": 3, + "avatar_subscript_url": "http://i0.hdslb.com/bfs/vip/icon_Certification_big_member_22_3x.png" + }, + "pendant": { + "pid": 3399, + "name": "2233幻星集", + "image": "http://i0.hdslb.com/bfs/garb/item/20c07ded13498a5b12db99660c766ddd92ecfe31.png", + "expire": 0, + "image_enhance": "http://i0.hdslb.com/bfs/garb/item/20c07ded13498a5b12db99660c766ddd92ecfe31.png", + "image_enhance_frame": "" + }, + "nameplate": { + "nid": 1, + "name": "黄金殿堂", + "image": "http://i2.hdslb.com/bfs/face/82896ff40fcb4e7c7259cb98056975830cb55695.png", + "image_small": "http://i0.hdslb.com/bfs/face/627e342851dfda6fe7380c2fa0cbd7fae2e61533.png", + "level": "稀有勋章", + "condition": "单个自制视频总播放数\u003e=100万" + }, + "user_honour_info": { + "mid": 0, + "colour": null, + "tags": [] + }, + "is_followed": true, + "top_photo": "http://i2.hdslb.com/bfs/space/853fea2728651588a2cdef0a1e586bcefff8e3d8.png", + "theme": {}, + "sys_notice": {}, + "live_room": { + "roomStatus": 1, + "liveStatus": 0, + "url": "https://live.bilibili.com/3044248?broadcast_type=0\u0026is_room_feed=1", + "title": "【Zc】早朝危机合约！", + "cover": "http://i0.hdslb.com/bfs/live/new_room_cover/cf7d4d3b2f336c6dba299644c3af952c5db82612.jpg", + "roomid": 3044248, + "roundStatus": 1, + "broadcast_type": 0, + "watched_show": { + "switch": true, + "num": 13753, + "text_small": "1.3万", + "text_large": "1.3万人看过", + "icon": "https://i0.hdslb.com/bfs/live/a725a9e61242ef44d764ac911691a7ce07f36c1d.png", + "icon_location": "", + "icon_web": "https://i0.hdslb.com/bfs/live/8d9d0f33ef8bf6f308742752d13dd0df731df19c.png" + } + }, + "birthday": "07-21", + "school": { + "name": "" + }, + "profession": { + "name": "", + "department": "", + "title": "", + "is_show": 0 + }, + "tags": [ + "评论区UP主", + "目标是星辰大海" + ], + "series": { + "user_upgrade_status": 3, + "show_upgrade_window": false + }, + "is_senior_member": 1 + } +} \ No newline at end of file diff --git a/tests/platforms/test_bilibili_live.py b/tests/platforms/test_bilibili_live.py new file mode 100644 index 0000000..53885ef --- /dev/null +++ b/tests/platforms/test_bilibili_live.py @@ -0,0 +1,44 @@ +from datetime import datetime + +import feedparser +import pytest +import respx +from httpx import Response +from nonebug.app import App +from pytz import timezone + +from .utils import get_file, get_json + + +@pytest.fixture +def bili_live(app: App): + from nonebot_bison.platform import platform_manager + + return platform_manager["bilibili-live"] + + +@pytest.mark.asyncio +@respx.mock +async def test_fetch_bilibili_live_status(bili_live, dummy_user_subinfo): + mock_bili_live_status = get_json("bili_live_status.json") + + bili_live_router = respx.get( + "https://api.bilibili.com/x/space/acc/info?mid=13164144" + ) + bili_live_router.mock(return_value=Response(200, json=mock_bili_live_status)) + target = "13164144" + res = await bili_live.fetch_new_post(target, [dummy_user_subinfo]) + assert bili_live_router.called + assert len(res) == 0 + mock_bili_live_status["data"]["live_room"]["liveStatus"] = 1 + bili_live_router.mock(return_value=Response(200, json=mock_bili_live_status)) + res2 = await bili_live.fetch_new_post(target, [dummy_user_subinfo]) + post = res2[0][1][0] + assert post.target_type == "Bilibili直播" + assert post.text == "【Zc】早朝危机合约！" + assert post.url == "https://live.bilibili.com/3044248" + assert post.target_name == "魔法Zc目录" + assert post.pics == [ + "http://i0.hdslb.com/bfs/live/new_room_cover/cf7d4d3b2f336c6dba299644c3af952c5db82612.jpg" + ] + assert post.compress == True diff --git a/tests/test_proxy.py b/tests/test_proxy.py index ec4a22e..44a9124 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -1,6 +1,5 @@ import pytest from nonebug import App -from nonebug.fixture import nonebug_init async def test_without_proxy(app: App): @@ -8,6 +7,8 @@ async def test_without_proxy(app: App): c = http_client() assert not c._mounts + req = c.build_request("GET", "http://example.com") + assert "Chrome" in req.headers["User-Agent"] @pytest.mark.parametrize(