✨ 提供批量 api 接口 (#290)

* 🚧 add batch api

* 🚧 support batch in scheduler

* ✨ use batch api in bilibili-live

✅ patch platform_manager directly

✨ use batch api in bilibili-live

✅ patch platform_manager directly

* ♻️ refactor var name

* 🐛 fix test

* 🐛 fix scheduler

* 🐛 fix test

nonebot_bison/scheduler/manager.py

@@ -29,7 +29,9 @@ async def init_scheduler():
     for scheduler_config, target_list in _schedule_class_dict.items():
         schedulable_args = []
         for target in target_list:
-            schedulable_args.append((target.platform_name, T_Target(target.target)))
+            schedulable_args.append(
+                (target.platform_name, T_Target(target.target), platform_manager[target.platform_name].use_batch)
+            )
         platform_name_list = _schedule_class_platform_dict[scheduler_config]
         scheduler_dict[scheduler_config] = Scheduler(scheduler_config, schedulable_args, platform_name_list)
     config.register_add_target_hook(handle_insert_new_target)

nonebot_bison/scheduler/scheduler.py

@@ -1,12 +1,13 @@
 from dataclasses import dataclass
+from collections import defaultdict
 
 from nonebot.log import logger
 from nonebot_plugin_apscheduler import scheduler
 from nonebot_plugin_saa.utils.exceptions import NoBotFound
 
-from ..types import Target
 from ..config import config
 from ..send import send_msgs
+from ..types import Target, SubUnit
 from ..platform import platform_manager
 from ..utils import ProcessContext, SchedulerConfig
 
@@ -16,15 +17,18 @@ class Schedulable:
     platform_name: str
     target: Target
     current_weight: int
+    use_batch: bool = False
 
 
 class Scheduler:
-    schedulable_list: list[Schedulable]
+    schedulable_list: list[Schedulable]  # for load weigth from db
+    batch_api_target_cache: dict[str, dict[Target, list[Target]]]  # platform_name -> (target -> [target])
+    batch_platform_name_targets_cache: dict[str, list[Target]]
 
     def __init__(
         self,
         scheduler_config: type[SchedulerConfig],
-        schedulables: list[tuple[str, Target]],
+        schedulables: list[tuple[str, Target, bool]],  # [(platform_name, target, use_batch)]
         platform_name_list: list[str],
     ):
         self.name = scheduler_config.name
@@ -33,9 +37,17 @@ class Scheduler:
             raise RuntimeError(f"{self.name} not found")
         self.scheduler_config = scheduler_config
         self.scheduler_config_obj = self.scheduler_config()
+
         self.schedulable_list = []
-        for platform_name, target in schedulables:
-            self.schedulable_list.append(Schedulable(platform_name=platform_name, target=target, current_weight=0))
+        self.batch_platform_name_targets_cache: dict[str, list[Target]] = defaultdict(list)
+        for platform_name, target, use_batch in schedulables:
+            if use_batch:
+                self.batch_platform_name_targets_cache[platform_name].append(target)
+            self.schedulable_list.append(
+                Schedulable(platform_name=platform_name, target=target, current_weight=0, use_batch=use_batch)
+            )
+        self._refresh_batch_api_target_cache()
+
         self.platform_name_list = platform_name_list
         self.pre_weight_val = 0  # 轮调度中“本轮”增加权重和的初值
         logger.info(
@@ -48,6 +60,12 @@ class Scheduler:
             **self.scheduler_config.schedule_setting,
         )
 
+    def _refresh_batch_api_target_cache(self):
+        self.batch_api_target_cache = defaultdict(dict)
+        for platform_name, targets in self.batch_platform_name_targets_cache.items():
+            for target in targets:
+                self.batch_api_target_cache[platform_name][target] = targets
+
     async def get_next_schedulable(self) -> Schedulable | None:
         if not self.schedulable_list:
             return None
@@ -69,14 +87,24 @@ class Scheduler:
         if not (schedulable := await self.get_next_schedulable()):
             return
         logger.trace(f"scheduler {self.name} fetching next target: [{schedulable.platform_name}]{schedulable.target}")
-        send_userinfo_list = await config.get_platform_target_subscribers(schedulable.platform_name, schedulable.target)
 
         client = await self.scheduler_config_obj.get_client(schedulable.target)
         context.register_to_client(client)
 
         try:
             platform_obj = platform_manager[schedulable.platform_name](context, client)
-            to_send = await platform_obj.do_fetch_new_post(schedulable.target, send_userinfo_list)
+            if schedulable.use_batch:
+                batch_targets = self.batch_api_target_cache[schedulable.platform_name][schedulable.target]
+                sub_units = []
+                for batch_target in batch_targets:
+                    userinfo = await config.get_platform_target_subscribers(schedulable.platform_name, batch_target)
+                    sub_units.append(SubUnit(batch_target, userinfo))
+                to_send = await platform_obj.do_batch_fetch_new_post(sub_units)
+            else:
+                send_userinfo_list = await config.get_platform_target_subscribers(
+                    schedulable.platform_name, schedulable.target
+                )
+                to_send = await platform_obj.do_fetch_new_post(SubUnit(schedulable.target, send_userinfo_list))
         except Exception as err:
             records = context.gen_req_records()
             for record in records:
@@ -101,9 +129,18 @@ class Scheduler:
     def insert_new_schedulable(self, platform_name: str, target: Target):
         self.pre_weight_val += 1000
         self.schedulable_list.append(Schedulable(platform_name, target, 1000))
+
+        if platform_manager[platform_name].use_batch:
+            self.batch_platform_name_targets_cache[platform_name].append(target)
+            self._refresh_batch_api_target_cache()
+
         logger.info(f"insert [{platform_name}]{target} to Schduler({self.scheduler_config.name})")
 
     def delete_schedulable(self, platform_name, target: Target):
+        if platform_manager[platform_name].use_batch:
+            self.batch_platform_name_targets_cache[platform_name].remove(target)
+            self._refresh_batch_api_target_cache()
+
         if not self.schedulable_list:
             return
         to_find_idx = None
@@ -114,4 +151,3 @@ class Scheduler:
         if to_find_idx is not None:
             deleted_schdulable = self.schedulable_list.pop(to_find_idx)
             self.pre_weight_val -= deleted_schdulable.current_weight
-        return