snowykami/testpage
1
0
Fork 0
forked from bot/app
Code Pull Requests Activity

🚀 测试文档工作流

Browse Source
This commit is contained in:
远野千束
2024-08-19 23:47:39 +08:00
parent 55a427e344
commit 6dcb085b53
48 changed files with 316 additions and 1704 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
liteyuki/comm/__init__.py
View File

@ -3,8 +3,8 @@
该模块用于轻雪主进程和Nonebot子进程之间的通信
依赖关系
event -> _
storage -> channel
rpc -> channel, storage
storage -> channel_
rpc -> channel_, storage
"""
from liteyuki.comm.channel import (
Channel,

27
liteyuki/comm/channel.py
View File

@ -5,7 +5,7 @@ Copyright (C) 2020-2024 LiteyukiStudio. All Rights Reserved
@Time : 2024/7/26 下午11:21
@Author : snowykami
@Email : snowykami@outlook.com
@File : channel.py
@File : channel_.py
@Software: PyCharm
本模块定义了一个通用的通道类，用于进程间通信
@ -38,11 +38,12 @@ class Channel(Generic[T]):
有两种接收工作方式，但是只能选择一种，主动接收和被动接收，主动接收使用 `receive` 方法，被动接收使用 `on_receive` 装饰器
"""
def __init__(self, _id: str, type_check: bool = False):
def __init__(self, _id: str, type_check: Optional[bool] = None):
"""
初始化通道
Args:
_id: 通道ID
type_check: 是否开启类型检查, 若为空，则传入泛型默认开启，否则默认关闭
"""
self.conn_send, self.conn_recv = Pipe()
self._closed = False
@ -53,9 +54,13 @@ class Channel(Generic[T]):
self.is_main_receive_loop_running = False
self.is_sub_receive_loop_running = False
if type_check:
if type_check is None:
# 若传入泛型则默认开启类型检查
type_check = self._get_generic_type() is not None
elif type_check:
if self._get_generic_type() is None:
raise TypeError("Type hint is required for enforcing type check.")
raise TypeError("Type hint is required for enforcing type_ check.")
self.type_check = type_check
def _get_generic_type(self) -> Optional[type]:
@ -110,7 +115,7 @@ class Channel(Generic[T]):
raise TypeError(f"Data must be an instance of {_type}, {type(data)} found")
if self._closed:
raise RuntimeError("Cannot send to a closed channel")
raise RuntimeError("Cannot send to a closed channel_")
self.conn_send.send(data)
def receive(self) -> T:
@ -119,7 +124,7 @@ class Channel(Generic[T]):
Args:
"""
if self._closed:
raise RuntimeError("Cannot receive from a closed channel")
raise RuntimeError("Cannot receive from a closed channel_")
while True:
data = self.conn_recv.recv()
@ -153,7 +158,7 @@ class Channel(Generic[T]):
async def wrapper(data: T) -> Any:
if filter_func is not None:
if is_coroutine_callable(filter_func):
if not (await filter_func(data)): # type: ignore
if not (await filter_func(data)): # type_: ignore
return
else:
if not filter_func(data):
@ -226,10 +231,12 @@ class Channel(Generic[T]):
"""子进程可用的主动和被动通道"""
active_channel: Optional["Channel"] = None
passive_channel: Optional["Channel"] = None
publish_channel: Channel[tuple[str, dict[str, Any]]] = Channel(_id="publish_channel")
"""通道传递通道，主进程创建单例，子进程初始化时实例化"""
channel_deliver_active_channel: Channel[Channel[Any]]
channel_deliver_passive_channel: Channel[tuple[str, dict[str, Any]]]
if IS_MAIN_PROCESS:
channel_deliver_active_channel = Channel(_id="channel_deliver_active_channel")
channel_deliver_passive_channel = Channel(_id="channel_deliver_passive_channel")
@ -237,7 +244,7 @@ if IS_MAIN_PROCESS:
@channel_deliver_passive_channel.on_receive(filter_func=lambda data: data[0] == "set_channel")
def on_set_channel(data: tuple[str, dict[str, Any]]):
name, channel = data[1]["name"], data[1]["channel"]
name, channel = data[1]["name"], data[1]["channel_"]
set_channel(name, channel)
@ -261,7 +268,7 @@ def set_channel(name: str, channel: Channel):
channel: 通道实例
"""
if not isinstance(channel, Channel):
raise TypeError(f"channel must be an instance of Channel, {type(channel)} found")
raise TypeError(f"channel_ must be an instance of Channel, {type(channel)} found")
if IS_MAIN_PROCESS:
_channel[name] = channel
@ -271,7 +278,7 @@ def set_channel(name: str, channel: Channel):
(
"set_channel", {
"name" : name,
"channel": channel,
"channel_": channel,
}
)
)

139
liteyuki/comm/storage.py
View File

@ -4,14 +4,20 @@
"""
import threading
from typing import Any, Optional
from typing import Any, Coroutine, Optional, TypeAlias, Callable
from liteyuki.comm.channel import Channel
from liteyuki.utils import IS_MAIN_PROCESS
from liteyuki.comm import channel
from liteyuki.comm.channel import Channel, ON_RECEIVE_FUNC, ASYNC_ON_RECEIVE_FUNC
from liteyuki.utils import IS_MAIN_PROCESS, is_coroutine_callable, run_coroutine
if IS_MAIN_PROCESS:
_locks = {}
_on_main_subscriber_receive_funcs: dict[str, list[ASYNC_ON_RECEIVE_FUNC]] = {} # type_: ignore
"""主进程订阅者接收函数"""
_on_sub_subscriber_receive_funcs: dict[str, list[ASYNC_ON_RECEIVE_FUNC]] = {} # type_: ignore
"""子进程订阅者接收函数"""
def _get_lock(key) -> threading.Lock:
"""
@ -25,12 +31,28 @@ def _get_lock(key) -> threading.Lock:
raise RuntimeError("Cannot get lock in sub process.")
class Subscriber:
def __init__(self):
self._subscribers = {}
def receive(self) -> Any:
pass
def unsubscribe(self) -> None:
pass
class KeyValueStore:
def __init__(self):
self._store = {}
self.active_chan = Channel[tuple[str, Optional[dict[str, Any]]]](_id="shared_memory-active")
self.passive_chan = Channel[tuple[str, Optional[dict[str, Any]]]](_id="shared_memory-passive")
self.publish_channel = Channel[tuple[str, Any]](_id="shared_memory-publish")
self.is_main_receive_loop_running = False
self.is_sub_receive_loop_running = False
def set(self, key: str, value: Any) -> None:
"""
设置键值对
@ -134,6 +156,94 @@ class KeyValueStore:
)
return recv_chan.receive()
def publish(self, channel_: str, data: Any) -> None:
"""
发布消息
Args:
channel_: 频道
data: 数据
Returns:
"""
self.active_chan.send(
(
"publish",
{
"channel_": channel_,
"data" : data
}
)
)
def on_subscriber_receive(self, channel_: str) -> Callable[[ON_RECEIVE_FUNC], ON_RECEIVE_FUNC]:
"""
订阅者接收消息时的回调
Args:
channel_: 频道
Returns:
装饰器
"""
if IS_MAIN_PROCESS and not self.is_main_receive_loop_running:
threading.Thread(target=self._start_receive_loop, daemon=True).start()
shared_memory.is_main_receive_loop_running = True
elif not IS_MAIN_PROCESS and not self.is_sub_receive_loop_running:
threading.Thread(target=self._start_receive_loop, daemon=True).start()
shared_memory.is_sub_receive_loop_running = True
def decorator(func: ON_RECEIVE_FUNC) -> ON_RECEIVE_FUNC:
async def wrapper(data: Any):
if is_coroutine_callable(func):
await func(data)
else:
func(data)
if IS_MAIN_PROCESS:
if channel_ not in _on_main_subscriber_receive_funcs:
_on_main_subscriber_receive_funcs[channel_] = []
_on_main_subscriber_receive_funcs[channel_].append(wrapper)
else:
if channel_ not in _on_sub_subscriber_receive_funcs:
_on_sub_subscriber_receive_funcs[channel_] = []
_on_sub_subscriber_receive_funcs[channel_].append(wrapper)
return wrapper
return decorator
@staticmethod
def run_subscriber_receive_funcs(channel_: str, data: Any):
"""
运行订阅者接收函数
Args:
channel_: 频道
data: 数据
"""
if IS_MAIN_PROCESS:
if channel_ in _on_main_subscriber_receive_funcs and _on_main_subscriber_receive_funcs[channel_]:
run_coroutine(*[func(data) for func in _on_main_subscriber_receive_funcs[channel_]])
else:
if channel_ in _on_sub_subscriber_receive_funcs and _on_sub_subscriber_receive_funcs[channel_]:
run_coroutine(*[func(data) for func in _on_sub_subscriber_receive_funcs[channel_]])
def _start_receive_loop(self):
"""
启动发布订阅接收器循环，在主进程中运行，若有子进程订阅则推送给子进程
"""
if IS_MAIN_PROCESS:
while True:
data = self.active_chan.receive()
if data[0] == "publish":
# 运行主进程订阅函数
self.run_subscriber_receive_funcs(data[1]["channel_"], data[1]["data"])
# 推送给子进程
self.publish_channel.send(data)
else:
while True:
data = self.publish_channel.receive()
if data[0] == "publish":
# 运行子进程订阅函数
self.run_subscriber_receive_funcs(data[1]["channel_"], data[1]["data"])
class GlobalKeyValueStore:
_instance = None
@ -141,20 +251,17 @@ class GlobalKeyValueStore:
@classmethod
def get_instance(cls):
if IS_MAIN_PROCESS:
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = KeyValueStore()
return cls._instance
else:
raise RuntimeError("Cannot get instance in sub process.")
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = KeyValueStore()
return cls._instance
shared_memory: KeyValueStore = GlobalKeyValueStore.get_instance()
# 全局单例访问点
if IS_MAIN_PROCESS:
shared_memory: KeyValueStore = GlobalKeyValueStore.get_instance()
@shared_memory.passive_chan.on_receive(lambda d: d[0] == "get")
def on_get(data: tuple[str, dict[str, Any]]):
@ -182,9 +289,13 @@ if IS_MAIN_PROCESS:
recv_chan = data[1]["recv_chan"]
recv_chan.send(shared_memory.get_all())
else:
# 子进程在入口函数中对shared_memory进行初始化
shared_memory: Optional[KeyValueStore] = None # type: ignore
@channel.publish_channel.on_receive()
def on_publish(data: tuple[str, Any]):
channel_, data = data
shared_memory.run_subscriber_receive_funcs(channel_, data)
_ref_count = 0 # import 引用计数, 防止获取空指针
if not IS_MAIN_PROCESS: