fair/matrix-puppeteer-line
fair
/
matrix-puppeteer-line
3
0
Fork 3
Code Issues 26 Pull Requests 2 Projects Releases Wiki Activity

Compare commits

base: fair:968832545ed5bb51bccd423be720d9826055de1b
Branches Tags
fair:master
fair:testing
..
compare: fair:9d4f2dafac6c7ab0cfcecb1e9c27f8c07012d88b
Branches Tags
fair:testing
fair:master

No commits in common. "968832545ed5bb51bccd423be720d9826055de1b" and "9d4f2dafac6c7ab0cfcecb1e9c27f8c07012d88b" have entirely different histories.
968832545e ... 9d4f2dafac

14 changed files with 157 additions and 486 deletions
Show Stats Expand all files Collapse all files

6
.gitignore vendored
View File

@ -1,6 +1,4 @@
/.idea/
/.*project
/.settings/
/.venv
/env/
@ -15,8 +13,8 @@ __pycache__
profiles
puppet/extension_files
/config*.yaml
/registration*.yaml
/config.yaml
/registration.yaml
*.log*
*.db
*.pickle

46
matrix_puppeteer_line/db/message.py
View File

@ -29,18 +29,20 @@ class Message:
mxid: EventID
mx_room: RoomID
mid: Optional[int]
mid: int
chat_id: str
async def insert(self) -> None:
q = "INSERT INTO message (mxid, mx_room, mid, chat_id) VALUES ($1, $2, $3, $4)"
await self.db.execute(q, self.mxid, self.mx_room, self.mid, self.chat_id)
async def update_ids(self, new_mxid: EventID, new_mid: int) -> None:
q = ("UPDATE message SET mxid=$1, mid=$2 "
"WHERE mxid=$3 AND mx_room=$4 AND chat_id=$5")
await self.db.execute(q, new_mxid, new_mid,
self.mxid, self.mx_room, self.chat_id)
async def delete(self) -> None:
q = "DELETE FROM message WHERE mid=$1"
await self.db.execute(q, self.mid)
@classmethod
async def delete_all(cls, room_id: RoomID) -> None:
await cls.db.execute("DELETE FROM message WHERE mx_room=$1", room_id)
@classmethod
async def get_max_mid(cls, room_id: RoomID) -> int:
@ -55,23 +57,6 @@ class Message:
data[row["chat_id"]] = row["max_mid"]
return data
@classmethod
async def get_num_noid_msgs(cls, room_id: RoomID) -> int:
return await cls.db.fetchval("SELECT COUNT(*) FROM message "
"WHERE mid IS NULL AND mx_room=$1", room_id)
@classmethod
async def is_last_by_mxid(cls, mxid: EventID, room_id: RoomID) -> bool:
q = ("SELECT mxid "
"FROM message INNER JOIN ( "
" SELECT mx_room, MAX(mid) AS max_mid "
" FROM message GROUP BY mx_room "
") by_room "
"ON mid=max_mid "
"WHERE by_room.mx_room=$1")
last_mxid = await cls.db.fetchval(q, room_id)
return last_mxid == mxid
@classmethod
async def get_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> Optional['Message']:
row = await cls.db.fetchrow("SELECT mxid, mx_room, mid, chat_id "
@ -87,18 +72,3 @@ class Message:
if not row:
return None
return cls(**row)
@classmethod
async def get_next_noid_msg(cls, room_id: RoomID) -> Optional['Message']:
row = await cls.db.fetchrow("SELECT mxid, mx_room, mid, chat_id FROM message "
"WHERE mid IS NULL AND mx_room=$1", room_id)
if not row:
return None
return cls(**row)
@classmethod
async def delete_all_noid_msgs(cls, room_id: RoomID) -> None:
status = await cls.db.execute("DELETE FROM message "
"WHERE mid IS NULL AND mx_room=$1", room_id)
# Skip leading "DELETE "
return int(status[7:])

9
matrix_puppeteer_line/db/upgrade.py
View File

@ -128,11 +128,4 @@ async def upgrade_strangers(conn: Connection) -> None:
FOREIGN KEY (fake_mid)
REFERENCES puppet (mid)
ON DELETE CASCADE
)""")
@upgrade_table.register(description="Track messages that lack an ID")
async def upgrade_noid_msgs(conn: Connection) -> None:
await conn.execute("ALTER TABLE message DROP CONSTRAINT IF EXISTS message_pkey")
await conn.execute("ALTER TABLE message ALTER COLUMN mid DROP NOT NULL")
await conn.execute("ALTER TABLE message ADD UNIQUE (mid)")
)""")

6
matrix_puppeteer_line/example-config.yaml
View File

@ -57,12 +57,6 @@ appservice:
as_token: "This value is generated when generating the registration"
hs_token: "This value is generated when generating the registration"
# Whether or not to receive ephemeral events via appservice transactions.
# Requires MSC2409 support (i.e. Synapse 1.22+).
# This is REQUIRED in order to bypass Puppeteer needing to "view" a LINE chat
# (thus triggering a LINE read receipt on your behalf) to sync its messages.
ephemeral_events: false
# Prometheus telemetry config. Requires prometheus-client to be installed.
metrics:
enabled: false

20
matrix_puppeteer_line/matrix.py
View File

@ -17,11 +17,9 @@ from typing import TYPE_CHECKING
from mautrix.bridge import BaseMatrixHandler
from mautrix.types import (Event, ReactionEvent, MessageEvent, StateEvent, EncryptedEvent, RedactionEvent,
ReceiptEvent, SingleReceiptEventContent,
EventID, RoomID, UserID)
from . import portal as po, puppet as pu, user as u
from .db import Message as DBMessage
if TYPE_CHECKING:
from .__main__ import MessagesBridge
@ -37,9 +35,8 @@ class MatrixHandler(BaseMatrixHandler):
super().__init__(bridge=bridge)
def filter_matrix_event(self, evt: Event) -> bool:
if isinstance(evt, ReceiptEvent):
return False
if not isinstance(evt, (MessageEvent, StateEvent, EncryptedEvent)):
if not isinstance(evt, (ReactionEvent, MessageEvent, StateEvent, EncryptedEvent,
RedactionEvent)):
return True
return (evt.sender == self.az.bot_mxid
or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
@ -62,16 +59,3 @@ class MatrixHandler(BaseMatrixHandler):
return
await portal.handle_matrix_leave(user)
async def handle_read_receipt(self, user: 'u.User', portal: 'po.Portal', event_id: EventID,
data: SingleReceiptEventContent) -> None:
# When reading a bridged message, view its chat in LINE, to make it send a read receipt.
# TODO Use *null* mids for last messages in a chat!!
# Only visit a LINE chat when its LAST bridge message has been read,
# because LINE lacks per-message read receipts--it's all or nothing!
# TODO Also view if message is non-last but for media, so it can be loaded.
#if await DBMessage.is_last_by_mxid(event_id, portal.mxid):
# Viewing a chat by updating it whole-hog, lest a ninja arrives
await user.sync_portal(portal)

111
matrix_puppeteer_line/portal.py
View File

@ -124,11 +124,6 @@ class Portal(DBPortal, BasePortal):
except Exception:
self.log.exception("Failed to send delivery receipt for %s", event_id)
async def _cleanup_noid_msgs(self) -> None:
num_noid_msgs = await DBMessage.delete_all_noid_msgs(self.mxid)
if num_noid_msgs > 0:
self.log.warn(f"Found {num_noid_msgs} messages in chat {self.chat_id} with no ID that could not be matched with a real ID")
async def handle_matrix_message(self, sender: 'u.User', message: MessageEventContent,
event_id: EventID) -> None:
if not sender.client:
@ -168,8 +163,6 @@ class Portal(DBPortal, BasePortal):
self.log.warning(f"Failed to upload media {event_id} to chat {self.chat_id}: {e}")
message_id = -1
remove(file_path)
await self._cleanup_noid_msgs()
msg = None
if message_id != -1:
try:
@ -182,18 +175,20 @@ class Portal(DBPortal, BasePortal):
if not msg and self.config["bridge.delivery_error_reports"]:
await self.main_intent.send_notice(
self.mxid,
"Posting this message to LINE may have failed.",
relates_to=RelatesTo(rel_type=RelationType.REPLY, event_id=event_id))
"Posting this message to LINE may have failed.",
relates_to=RelatesTo(rel_type=RelationType.REPLY, event_id=event_id))
async def handle_matrix_leave(self, user: 'u.User') -> None:
self.log.info(f"{user.mxid} left portal to {self.chat_id}, "
f"cleaning up and deleting...")
await self.cleanup_and_delete()
async def _bridge_own_message_pm(self, source: 'u.User', puppet: Optional['p.Puppet'], mid: str,
async def _bridge_own_message_pm(self, source: 'u.User', sender: Optional['p.Puppet'], mid: str,
invite: bool = True) -> Optional[IntentAPI]:
intent = puppet.intent if puppet else (await source.get_own_puppet()).intent
if self.is_direct and (not puppet or puppet.mid == source.mid and not puppet.is_real_user):
# Use bridge bot as puppet for own user when puppet for own user is unavailable
# TODO Use own LINE puppet instead, and create it if it's not available yet
intent = sender.intent if sender else self.az.intent
if self.is_direct and (sender is None or sender.mid == source.mid and not sender.is_real_user):
if self.invite_own_puppet_to_pm and invite:
try:
await intent.ensure_joined(self.mxid)
@ -218,55 +213,29 @@ class Portal(DBPortal, BasePortal):
if evt.is_outgoing:
if source.intent:
sender = None
intent = source.intent
else:
if not self.invite_own_puppet_to_pm:
self.log.warning(f"Ignoring message {evt.id}: double puppeting isn't enabled")
return
puppet = await p.Puppet.get_by_mid(evt.sender.id) if evt.sender else None
intent = await self._bridge_own_message_pm(source, puppet, f"message {evt.id}")
sender = p.Puppet.get_by_mid(evt.sender.id) if not self.is_direct else None
intent = await self._bridge_own_message_pm(source, sender, f"message {evt.id}")
if not intent:
return
else:
if self.is_direct:
# TODO Respond to name/avatar changes of users in a DM
intent = (await p.Puppet.get_by_mid(self.other_user)).intent
elif evt.sender:
puppet = await p.Puppet.get_by_mid(evt.sender.id)
if puppet:
await puppet.update_info(evt.sender, source.client)
sender = await p.Puppet.get_by_mid(self.other_user if self.is_direct else evt.sender.id)
# TODO Respond to name/avatar changes of users in a DM
if not self.is_direct:
if sender:
await sender.update_info(evt.sender, source.client)
else:
self.log.warning(f"Could not find ID of LINE user who sent message {evt.id or 'with no ID'}")
puppet = await p.Puppet.get_by_profile(evt.sender, source.client)
intent = puppet.intent
else:
self.log.info(f"Using bridgebot for unknown sender of message {evt.id or 'with no ID'}")
intent = self.az.intent
self.log.warning(f"Could not find ID of LINE user who sent event {evt.id}")
sender = await p.Puppet.get_by_profile(evt.sender, source.client)
intent = sender.intent
await intent.ensure_joined(self.mxid)
if evt.id:
msg = await DBMessage.get_next_noid_msg(self.mxid)
if not msg:
self.log.info(f"Handling new message {evt.id} in chat {self.mxid}")
prev_event_id = None
else:
self.log.info(f"Handling preseen message {evt.id} in chat {self.mxid}: {msg.mxid}")
if not self.is_direct:
# Non-DM previews are always sent by bridgebot.
# Must delete the bridgebot message and send a new message from the correct puppet.
await self.az.intent.redact(self.mxid, msg.mxid, "Found actual sender")
prev_event_id = None
else:
prev_event_id = msg.mxid
else:
self.log.info(f"Handling new message with no ID in chat {self.mxid}")
msg = None
prev_event_id = None
if prev_event_id and evt.html:
# No need to update a previewed text message, as their previews are accurate
event_id = prev_event_id
elif evt.image and evt.image.url:
if evt.image and evt.image.url:
if not evt.image.is_sticker or self.config["bridge.receive_stickers"]:
media_info = await self._handle_remote_media(
source, intent, evt.image.url,
@ -283,11 +252,9 @@ class Portal(DBPortal, BasePortal):
else:
media_info = None
send_sticker = self.config["bridge.use_sticker_events"] and evt.image.is_sticker and not self.encrypted and media_info
# TODO Element Web messes up text->sticker edits!!
# File a case on it
if send_sticker and not prev_event_id:
#relates_to = RelatesTo(rel_type=RelationType.REPLACE, event_id=prev_event_id) if prev_event_id else None
event_id = await intent.send_sticker(self.mxid, media_info.mxc, image_info, "<sticker>", timestamp=evt.timestamp)
if send_sticker:
event_id = await intent.send_sticker(
self.mxid, media_info.mxc, image_info, "<sticker>", timestamp=evt.timestamp)
else:
if media_info:
content = MediaMessageEventContent(
@ -299,11 +266,8 @@ class Portal(DBPortal, BasePortal):
content = TextMessageEventContent(
msgtype=MessageType.NOTICE,
body=f"<{'sticker' if evt.image.is_sticker else 'image'}>")
if prev_event_id:
content.set_edit(prev_event_id)
event_id = await self._send_message(intent, content, timestamp=evt.timestamp)
elif evt.html and not evt.html.isspace():
chunks = []
def handle_data(data):
@ -367,24 +331,17 @@ class Portal(DBPortal, BasePortal):
content = TextMessageEventContent(
msgtype=MessageType.NOTICE,
body="<Unbridgeable message>")
if prev_event_id:
content.set_edit(prev_event_id)
event_id = await self._send_message(intent, content, timestamp=evt.timestamp)
if evt.is_outgoing and evt.receipt_count:
await self._handle_receipt(event_id, evt.receipt_count)
if not msg:
msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id)
try:
await msg.insert()
#await self._send_delivery_receipt(event_id)
self.log.debug(f"Handled remote message {evt.id or 'with no ID'} -> {event_id}")
except UniqueViolationError as e:
self.log.debug(f"Failed to handle remote message {evt.id or 'with no ID'} -> {event_id}: {e}")
else:
await msg.update_ids(new_mxid=event_id, new_mid=evt.id)
self.log.debug(f"Handled preseen remote message {evt.id} -> {event_id}")
msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id)
try:
await msg.insert()
await self._send_delivery_receipt(event_id)
self.log.debug(f"Handled remote message {evt.id} -> {event_id}")
except UniqueViolationError as e:
self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}")
async def handle_remote_receipt(self, receipt: Receipt) -> None:
msg = await DBMessage.get_by_mid(receipt.id)
@ -572,12 +529,11 @@ class Portal(DBPortal, BasePortal):
continue
mid = p.Puppet.get_id_from_mxid(user_id)
is_own_puppet = p.Puppet.is_mid_for_own_puppet(mid)
if mid and mid not in current_members and not is_own_puppet:
if mid and mid not in current_members:
print(mid)
await self.main_intent.kick_user(self.mxid, user_id,
reason="User had left this chat")
elif forbid_own_puppets and is_own_puppet:
elif forbid_own_puppets and p.Puppet.is_mid_for_own_puppet(mid):
await self.main_intent.kick_user(self.mxid, user_id,
reason="Kicking own puppet")
@ -597,7 +553,6 @@ class Portal(DBPortal, BasePortal):
if not messages:
self.log.debug("Didn't get any entries from server")
await self._cleanup_noid_msgs()
return
self.log.debug("Got %d messages from server", len(messages))
@ -605,7 +560,6 @@ class Portal(DBPortal, BasePortal):
for evt in messages:
await self.handle_remote_message(source, evt)
self.log.info("Backfilled %d messages through %s", len(messages), source.mxid)
await self._cleanup_noid_msgs()
@property
def bridge_info_state_key(self) -> str:
@ -759,6 +713,9 @@ class Portal(DBPortal, BasePortal):
self._main_intent = self.az.intent
async def delete(self) -> None:
if self.mxid:
# TODO Handle this with db foreign keys instead
await DBMessage.delete_all(self.mxid)
self.by_chat_id.pop(self.chat_id, None)
self.by_mxid.pop(self.mxid, None)
await super().delete()

9
matrix_puppeteer_line/rpc/client.py
View File

@ -19,7 +19,7 @@ from base64 import b64decode
import asyncio
from .rpc import RPCClient
from .types import ChatListInfo, ChatInfo, ImageData, Message, Participant, Receipt, StartStatus
from .types import ChatListInfo, ChatInfo, Message, Receipt, ImageData, StartStatus
class LoginCommand(TypedDict):
@ -41,15 +41,12 @@ class Client(RPCClient):
async def resume(self) -> None:
await self.request("resume")
async def get_own_profile(self) -> Participant:
return Participant.deserialize(await self.request("get_own_profile"))
async def get_chats(self) -> List[ChatListInfo]:
resp = await self.request("get_chats")
return [ChatListInfo.deserialize(data) for data in resp]
async def get_chat(self, chat_id: str, force_view: bool = False) -> ChatInfo:
return ChatInfo.deserialize(await self.request("get_chat", chat_id=chat_id, force_view=force_view))
async def get_chat(self, chat_id: str) -> ChatInfo:
return ChatInfo.deserialize(await self.request("get_chat", chat_id=chat_id))
async def get_messages(self, chat_id: str) -> List[Message]:
resp = await self.request("get_messages", chat_id=chat_id)

10
matrix_puppeteer_line/rpc/rpc.py
View File

@ -33,7 +33,6 @@ class RPCClient:
log: TraceLogger = logging.getLogger("mau.rpc")
user_id: UserID
ephemeral_events: bool
_reader: Optional[asyncio.StreamReader]
_writer: Optional[asyncio.StreamWriter]
_req_id: int
@ -41,12 +40,10 @@ class RPCClient:
_response_waiters: Dict[int, asyncio.Future]
_event_handlers: Dict[str, List[EventHandler]]
def __init__(self, user_id: UserID, own_id: str, ephemeral_events: bool) -> None:
def __init__(self, user_id: UserID) -> None:
self.log = self.log.getChild(user_id)
self.loop = asyncio.get_running_loop()
self.user_id = user_id
self.own_id = own_id
self.ephemeral_events = ephemeral_events
self._req_id = 0
self._min_broadcast_id = 0
self._event_handlers = {}
@ -70,10 +67,7 @@ class RPCClient:
self._writer = w
self.loop.create_task(self._try_read_loop())
self.loop.create_task(self._command_loop())
await self.request("register",
user_id=self.user_id,
own_id = self.own_id,
ephemeral_events=self.ephemeral_events)
await self.request("register", user_id=self.user_id)
async def disconnect(self) -> None:
self._writer.write_eof()

2
matrix_puppeteer_line/rpc/types.py
View File

@ -60,7 +60,7 @@ class MessageImage(SerializableAttrs['MessageImage']):
@dataclass
class Message(SerializableAttrs['Message']):
id: Optional[int]
id: int
chat_id: int
is_outgoing: bool
sender: Optional[Participant]

25
matrix_puppeteer_line/user.py
View File

@ -69,14 +69,6 @@ class User(DBUser, BaseUser):
self.log.debug(f"Sending bridge notice: {text}")
await self.az.intent.send_notice(self.notice_room, text)
@property
def own_id(self) -> str:
# Remove characters that will conflict with mxid grammar
return f"_OWN_{self.mxid[1:].replace(':', '_ON_')}"
async def get_own_puppet(self) -> 'pu.Puppet':
return await pu.Puppet.get_by_mid(self.own_id)
async def is_logged_in(self) -> bool:
try:
return self.client and (await self.client.start()).is_logged_in
@ -103,7 +95,7 @@ class User(DBUser, BaseUser):
async def connect(self) -> None:
self.loop.create_task(self.connect_double_puppet())
self.client = Client(self.mxid, self.own_id, self.config["appservice.ephemeral_events"])
self.client = Client(self.mxid)
self.log.debug("Starting client")
await self.send_bridge_notice("Starting up...")
state = await self.client.start()
@ -134,7 +126,6 @@ class User(DBUser, BaseUser):
self._connection_check_task.cancel()
self._connection_check_task = self.loop.create_task(self._check_connection_loop())
await self.client.pause()
await self.sync_own_profile()
await self.client.set_last_message_ids(await DBMessage.get_max_mids())
limit = self.config["bridge.initial_conversation_sync"]
self.log.info("Syncing chats")
@ -153,20 +144,6 @@ class User(DBUser, BaseUser):
await self.send_bridge_notice("Synchronization complete")
await self.client.resume()
async def sync_portal(self, portal: 'po.Portal') -> None:
chat_id = portal.chat_id
self.log.info(f"Viewing (and syncing) chat {chat_id}")
await self.client.pause()
chat = await self.client.get_chat(chat_id, True)
await portal.update_matrix_room(self, chat)
await self.client.resume()
async def sync_own_profile(self) -> None:
self.log.info("Syncing own LINE profile info")
own_profile = await self.client.get_own_profile()
puppet = await self.get_own_puppet()
await puppet.update_info(own_profile, self.client)
async def stop(self) -> None:
# TODO Notices for shutdown messages
if self._connection_check_task:

2
puppet/.gitignore vendored
View File

@ -1,2 +1,2 @@
/node_modules
/config*.json
/config.json

13
puppet/src/client.js
View File

@ -99,7 +99,7 @@ export default class Client {
}
sendMessage(message) {
this.log(`Sending message ${message.id || "with no ID"} to client`)
this.log(`Sending message ${message.id} to client`)
return this._write({
id: --this.notificationID,
command: "message",
@ -164,7 +164,7 @@ export default class Client {
let started = false
if (this.puppet === null) {
this.log("Opening new puppeteer for", this.userID)
this.puppet = new MessagesPuppeteer(this.userID, this.ownID, this.sendPlaceholders, this)
this.puppet = new MessagesPuppeteer(this.userID, this)
this.manager.puppets.set(this.userID, this.puppet)
await this.puppet.start(!!req.debug)
started = true
@ -194,13 +194,11 @@ export default class Client {
handleRegister = async (req) => {
this.userID = req.user_id
this.ownID = req.own_id
this.sendPlaceholders = req.ephemeral_events
this.log(`Registered socket ${this.connID} -> ${this.userID}${!this.sendPlaceholders ? "" : " (with placeholder message support)"}`)
this.log("Registered socket", this.connID, "->", this.userID)
if (this.manager.clients.has(this.userID)) {
const oldClient = this.manager.clients.get(this.userID)
this.manager.clients.set(this.userID, this)
this.log(`Terminating previous socket ${oldClient.connID} for ${this.userID}`)
this.log("Terminating previous socket", oldClient.connID, "for", this.userID)
await oldClient.stop("Socket replaced by new connection")
} else {
this.manager.clients.set(this.userID, this)
@ -260,9 +258,8 @@ export default class Client {
set_last_message_ids: req => this.puppet.setLastMessageIDs(req.msg_ids),
pause: () => this.puppet.stopObserving(),
resume: () => this.puppet.startObserving(),
get_own_profile: () => this.puppet.getOwnProfile(),
get_chats: () => this.puppet.getRecentChats(),
get_chat: req => this.puppet.getChatInfo(req.chat_id, req.force_view),
get_chat: req => this.puppet.getChatInfo(req.chat_id),
get_messages: req => this.puppet.getMessages(req.chat_id),
read_image: req => this.puppet.readImage(req.image_url),
is_connected: async () => ({ is_connected: !await this.puppet.isDisconnected() }),

84
puppet/src/contentscript.js
View File

@ -23,7 +23,7 @@
*/
window.__chronoParseDate = function (text, ref, option) {}
/**
* @param {ChatListInfo[]} changes - The chats that changed.
* @param {string[]} changes - The hrefs of the chats that changed.
* @return {Promise<void>}
*/
window.__mautrixReceiveChanges = function (changes) {}
@ -102,7 +102,9 @@ class MautrixController {
}
setOwnID(ownID) {
this.ownID = ownID
// Remove characters that will conflict with mxid grammar
const suffix = ownID.slice(1).replace(":", "_ON_")
this.ownID = `_OWN_${suffix}`
}
// TODO Commonize with Node context
@ -525,7 +527,7 @@ class MautrixController {
* @typedef PathImage
* @type object
* @property {?string} path - The virtual path of the image (behaves like an ID). Optional.
* @property {string} url - The URL of the image. Mandatory.
* @property {string} src - The URL of the image. Mandatory.
*/
/**
@ -607,8 +609,6 @@ class MautrixController {
* May be prefixed by sender name.
* @property {string} lastMsgDate - An imprecise date for the most recent message
* (e.g. "7:16 PM", "Thu" or "Aug 4")
* @property {number} notificationCount - The number of unread messages in the chat,
* signified by the number in its notification badge.
*/
getChatListItemID(element) {
@ -624,17 +624,13 @@ class MautrixController {
}
getChatListItemLastMsg(element) {
return element.querySelector(".mdCMN04Desc").innerHTML
return element.querySelector(".mdCMN04Desc").innerText
}
getChatListItemLastMsgDate(element) {
return element.querySelector("time").innerText
}
getChatListItemNotificationCount(element) {
return Number.parseInt(element.querySelector(".MdIcoBadge01:not(.MdNonDisp)")?.innerText) || 0
}
/**
* Parse a conversation list item element.
*
@ -649,7 +645,6 @@ class MautrixController {
icon: this.getChatListItemIcon(element),
lastMsg: this.getChatListItemLastMsg(element),
lastMsgDate: this.getChatListItemLastMsgDate(element),
notificationCount: this.getChatListItemNotificationCount(element),
}
}
@ -681,56 +676,13 @@ class MautrixController {
return promise
}
/**
* Wait for updates to the active chat's message list to settle down.
* Wait an additional bit of time every time an update is observed.
* TODO Look (harder) for an explicit signal of when a chat is fully updated...
*
* @returns Promise<void>
*/
waitForMessageListStability() {
// Increase this if messages get missed on sync / chat change.
// Decrease it if response times are too slow.
const delayMillis = 500
let myResolve
const promise = new Promise(resolve => {myResolve = resolve})
let observer
const onTimeout = () => {
console.log("Message list looks stable, continue")
console.debug(`timeoutID = ${timeoutID}`)
observer.disconnect()
myResolve()
}
let timeoutID
const startTimer = () => {
timeoutID = setTimeout(onTimeout, delayMillis)
}
observer = new MutationObserver(changes => {
clearTimeout(timeoutID)
console.log("CHANGE to message list detected! Wait a bit longer...")
console.debug(`timeoutID = ${timeoutID}`)
console.debug(changes)
startTimer()
})
observer.observe(
document.querySelector("#_chat_message_area"),
{childList: true, attributes: true, subtree: true})
startTimer()
return promise
}
/**
* @param {[MutationRecord]} mutations - The mutation records that occurred
* @private
*/
_observeChatListMutations(mutations) {
// TODO Observe *added/removed* chats, not just new messages
const changedChats = new Set()
const changedChatIDs = new Set()
for (const change of mutations) {
if (change.target.id == "_chat_list_body") {
// TODO
@ -740,24 +692,26 @@ class MautrixController {
for (const node of change.addedNodes) {
}
*/
} else if (change.target.tagName == "LI" && change.addedNodes.length == 1) {
} else if (change.target.tagName == "LI") {
if (change.target.classList.contains("ExSelected")) {
console.debug("Not using chat list mutation response for currently-active chat")
continue
}
const chat = this.parseChatListItem(change.addedNodes[0])
if (chat) {
console.log("Added chat list item:", chat)
changedChats.add(chat)
} else {
console.debug("Could not parse added node as a chat list item:", node)
for (const node of change.addedNodes) {
const chat = this.parseChatListItem(node)
if (chat) {
console.log("Added chat list item:", chat)
changedChatIDs.add(chat.id)
} else {
console.debug("Could not parse added node as a chat list item:", node)
}
}
}
// change.removedNodes tells you which chats that had notifications are now read.
}
if (changedChats.size > 0) {
console.debug("Dispatching chat list mutations:", changedChats)
window.__mautrixReceiveChanges(Array.from(changedChats)).then(
if (changedChatIDs.size > 0) {
console.debug("Dispatching chat list mutations:", changedChatIDs)
window.__mautrixReceiveChanges(Array.from(changedChatIDs)).then(
() => console.debug("Chat list mutations dispatched"),
err => console.error("Error dispatching chat list mutations:", err))
}

300
puppet/src/puppet.js
View File

@ -27,7 +27,7 @@ export default class MessagesPuppeteer {
static executablePath = undefined
static devtools = false
static noSandbox = false
static viewport = { width: 960, height: 840 }
static viewport = { width: 960, height: 880 }
static url = undefined
static extensionDir = "extension_files"
@ -36,19 +36,16 @@ export default class MessagesPuppeteer {
* @param {string} id
* @param {?Client} [client]
*/
constructor(id, ownID, sendPlaceholders, client = null) {
constructor(id, client = null) {
let profilePath = path.join(MessagesPuppeteer.profileDir, id)
if (!profilePath.startsWith("/")) {
profilePath = path.join(process.cwd(), profilePath)
}
this.id = id
this.ownID = ownID
this.sendPlaceholders = sendPlaceholders
this.profilePath = profilePath
this.updatedChats = new Set()
this.sentMessageIDs = new Set()
this.mostRecentMessages = new Map()
this.numChatNotifications = new Map()
this.taskQueue = new TaskQueue(this.id)
this.client = client
}
@ -68,19 +65,18 @@ export default class MessagesPuppeteer {
async start() {
this.log("Launching browser")
const args = [
let extensionArgs = [
`--disable-extensions-except=${MessagesPuppeteer.extensionDir}`,
`--load-extension=${MessagesPuppeteer.extensionDir}`,
`--window-size=${MessagesPuppeteer.viewport.width},${MessagesPuppeteer.viewport.height+120}`,
`--load-extension=${MessagesPuppeteer.extensionDir}`
]
if (MessagesPuppeteer.noSandbox) {
args = args.concat(`--no-sandbox`)
extensionArgs = extensionArgs.concat(`--no-sandbox`)
}
this.browser = await puppeteer.launch({
executablePath: MessagesPuppeteer.executablePath,
userDataDir: this.profilePath,
args: args,
args: extensionArgs,
headless: false, // Needed to load extensions
defaultViewport: MessagesPuppeteer.viewport,
devtools: MessagesPuppeteer.devtools,
@ -93,10 +89,6 @@ export default class MessagesPuppeteer {
} else {
this.page = await this.browser.newPage()
}
this.blankPage = await this.browser.newPage()
await this.page.bringToFront()
this.log("Opening", MessagesPuppeteer.url)
await this.page.setBypassCSP(true) // Needed to load content scripts
await this._preparePage(true)
@ -127,7 +119,6 @@ export default class MessagesPuppeteer {
}
async _preparePage(navigateTo) {
await this.page.bringToFront()
if (navigateTo) {
await this.page.goto(MessagesPuppeteer.url)
} else {
@ -137,30 +128,6 @@ export default class MessagesPuppeteer {
await this.page.addScriptTag({ path: "./src/contentscript.js", type: "module" })
}
async _interactWithPage(promiser) {
await this.page.bringToFront()
try {
await promiser()
} catch (e) {
this.error(`Error while interacting with page: ${e}`)
throw e
} finally {
await this.blankPage.bringToFront()
}
}
/**
* Set the contents of a text input field to the given text.
* Works by triple-clicking the input field to select all existing text, to replace it on type.
*
* @param {ElementHandle} inputElement - The input element to type into.
* @param {string} text - The text to input.
*/
async _enterText(inputElement, text) {
await inputElement.click({clickCount: 3})
await inputElement.type(text)
}
/**
* Wait for the session to be logged in and monitor changes while it's not.
*/
@ -169,7 +136,6 @@ export default class MessagesPuppeteer {
return
}
this.loginRunning = true
await this.page.bringToFront()
const loginContentArea = await this.page.waitForSelector("#login_content")
@ -253,7 +219,7 @@ export default class MessagesPuppeteer {
this.log("Removing observers")
// TODO __mautrixController is undefined when cancelling, why?
await this.page.evaluate(ownID => window.__mautrixController.setOwnID(ownID), this.ownID)
await this.page.evaluate(ownID => window.__mautrixController.setOwnID(ownID), this.id)
await this.page.evaluate(() => window.__mautrixController.removeQRChangeObserver())
await this.page.evaluate(() => window.__mautrixController.removeQRAppearObserver())
await this.page.evaluate(() => window.__mautrixController.removeEmailAppearObserver())
@ -288,7 +254,15 @@ export default class MessagesPuppeteer {
}
this.loginRunning = false
await this.blankPage.bringToFront()
// Don't start observing yet, instead wait for explicit request.
// But at least view the most recent chat.
try {
const mostRecentChatID = await this.page.$eval("#_chat_list_body li",
element => window.__mautrixController.getChatListItemID(element.firstElementChild))
await this._switchChat(mostRecentChatID)
} catch (e) {
this.log("No chats available to focus on")
}
this.log("Login complete")
}
@ -402,11 +376,10 @@ export default class MessagesPuppeteer {
* Get info about a chat.
*
* @param {string} chatID - The chat ID whose info to get.
* @param {boolean} forceView - Whether the LINE tab should always be viewed, even if the chat is already active.
* @return {Promise<ChatInfo>} - Info about the chat.
*/
async getChatInfo(chatID, forceView) {
return await this.taskQueue.push(() => this._getChatInfoUnsafe(chatID, forceView))
async getChatInfo(chatID) {
return await this.taskQueue.push(() => this._getChatInfoUnsafe(chatID))
}
/**
@ -427,7 +400,7 @@ export default class MessagesPuppeteer {
* @return {Promise<[MessageData]>} - The messages visible in the chat.
*/
async getMessages(chatID) {
return await this.taskQueue.push(() => this._getMessagesUnsafe(chatID))
return await this.taskQueue.push(async () => this._getMessagesUnsafe(chatID))
}
setLastMessageIDs(ids) {
@ -470,46 +443,11 @@ export default class MessagesPuppeteer {
() => window.__mautrixController.removeMsgListObserver())
}
async getOwnProfile() {
return await this.taskQueue.push(() => this._getOwnProfileUnsafe())
}
async _getOwnProfileUnsafe() {
// NOTE Will send a read receipt if a chat was in view!
// Best to use this on startup when no chat is viewed.
let ownProfile
await this._interactWithPage(async () => {
this.log("Opening settings view")
await this.page.click("button.mdGHD01SettingBtn")
await this.page.waitForSelector("#context_menu li#settings", {visible: true}).then(e => e.click())
await this.page.waitForSelector("#settings_contents", {visible: true})
this.log("Getting own profile info")
ownProfile = {
id: this.ownID,
name: await this.page.$eval("#settings_basic_name_input", e => e.innerText),
avatar: {
path: null,
url: await this.page.$eval(".mdCMN09ImgInput", e => {
const imgStr = e.style?.getPropertyValue("background-image")
const matches = imgStr.match(/url\("(blob:.*)"\)/)
return matches?.length == 2 ? matches[1] : null
}),
},
}
const backSelector = "#label_setting button"
await this.page.click(backSelector)
await this.page.waitForSelector(backSelector, {visible: false})
})
return ownProfile
}
_listItemSelector(id) {
return `#_chat_list_body div[data-chatid="${id}"]`
}
async _switchChat(chatID, forceView = false) {
async _switchChat(chatID) {
// TODO Allow passing in an element directly
this.log(`Switching to chat ${chatID}`)
const chatListItem = await this.page.$(this._listItemSelector(chatID))
@ -525,63 +463,30 @@ export default class MessagesPuppeteer {
}
if (await this.page.evaluate(isCorrectChatVisible, chatName)) {
if (!forceView) {
this.log("Already viewing chat, no need to switch")
} else {
await this._interactWithPage(async () => {
this.log("Already viewing chat, but got request to view it")
this.page.waitForTimeout(500)
})
}
this.log("Already viewing chat, no need to switch")
} else {
this.log("Ensuring msg list observer is removed")
const hadMsgListObserver = await this.page.evaluate(
() => window.__mautrixController.removeMsgListObserver())
this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer")
await this._interactWithPage(async () => {
let numTries = 3
while (true) {
try {
this.log("Clicking chat list item")
chatListItem.click()
this.log(`Waiting for chat header title to be "${chatName}"`)
await this.page.waitForFunction(
isCorrectChatVisible,
{polling: "mutation", timeout: 1000},
chatName)
break
} catch (e) {
if (--numTries == 0) {
throw e
} else {
this.log("Clicking chat list item didn't work...try again")
}
}
}
await chatListItem.click()
this.log(`Waiting for chat header title to be "${chatName}"`)
await this.page.waitForFunction(
isCorrectChatVisible,
{polling: "mutation"},
chatName)
// Always show the chat details sidebar, as this makes life easier
this.log("Waiting for detail area to be auto-hidden upon entering chat")
await this.page.waitForFunction(
detailArea => detailArea.childElementCount == 0,
{},
await this.page.$("#_chat_detail_area"))
this.log("Clicking chat header to show detail area")
await this.page.click("#_chat_header_area > .mdRGT04Link")
this.log("Waiting for detail area")
await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info")
})
this.log("Waiting for any item to appear in chat")
try {
await this.page.waitForSelector("#_chat_room_msg_list div", {timeout: 2000})
this.log("Waiting for chat to stabilize")
await this.page.evaluate(() => window.__mautrixController.waitForMessageListStability())
} catch (e) {
this.log("No messages in chat found. Maybe no messages were ever sent yet?")
}
// Always show the chat details sidebar, as this makes life easier
this.log("Waiting for detail area to be auto-hidden upon entering chat")
await this.page.waitForFunction(
detailArea => detailArea.childElementCount == 0,
{},
await this.page.$("#_chat_detail_area"))
this.log("Clicking chat header to show detail area")
await this.page.click("#_chat_header_area > .mdRGT04Link")
this.log("Waiting for detail area")
await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info")
if (hadMsgListObserver) {
this.log("Restoring msg list observer")
@ -594,7 +499,7 @@ export default class MessagesPuppeteer {
}
}
async _getChatInfoUnsafe(chatID, forceView) {
async _getChatInfoUnsafe(chatID) {
const chatListInfo = await this.page.$eval(this._listItemSelector(chatID),
(element, chatID) => window.__mautrixController.parseChatListItem(element, chatID),
chatID)
@ -616,7 +521,7 @@ export default class MessagesPuppeteer {
if (!isDirect) {
this.log("Found multi-user chat, so viewing it to get participants")
// TODO This will mark the chat as "read"!
await this._switchChat(chatID, forceView)
await this._switchChat(chatID)
const participantList = await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul")
// TODO Is a group not actually created until a message is sent(?)
// If so, maybe don't create a portal until there is a message.
@ -624,10 +529,6 @@ export default class MessagesPuppeteer {
element => window.__mautrixController.parseParticipantList(element))
} else {
this.log(`Found direct chat with ${chatID}`)
if (forceView) {
this.log("Viewing chat on request")
await this._switchChat(chatID, forceView)
}
//const chatDetailArea = await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info")
//await chatDetailArea.$(".MdTxtDesc02") || // 1:1 chat with custom title - get participant's real name
participants = [{
@ -650,27 +551,21 @@ export default class MessagesPuppeteer {
// Always present, just made visible via classes
async _sendMessageUnsafe(chatID, text) {
// Sync all messages in this chat first
this._receiveMessages(chatID, await this._getMessagesUnsafe(chatID), true)
await this._switchChat(chatID)
// TODO Initiate the promise in the content script
await this.page.evaluate(
() => window.__mautrixController.promiseOwnMessage(5000, "time"))
const input = await this.page.$("#_chat_room_input")
await this._interactWithPage(async () => {
// Live-typing in the field can have its text mismatch what was requested!!
// Probably because the input element is a div instead of a real text input...ugh!
// Setting its innerText directly works fine though...
await input.click()
await input.evaluate((e, text) => e.innerText = text, text)
await input.press("Enter")
})
await input.click()
await input.type(text)
await input.press("Enter")
return await this._waitForSentMessage(chatID)
}
async _sendFileUnsafe(chatID, filePath) {
this._receiveMessages(chatID, await this._getMessagesUnsafe(chatID), true)
await this._switchChat(chatID)
await this.page.evaluate(
() => window.__mautrixController.promiseOwnMessage(
10000, // Use longer timeout for file uploads
@ -678,15 +573,13 @@ export default class MessagesPuppeteer {
"#_chat_message_fail_menu"))
try {
this._interactWithPage(async () => {
this.log(`About to ask for file chooser in ${chatID}`)
const [fileChooser] = await Promise.all([
this.page.waitForFileChooser(),
this.page.click("#_chat_room_plus_btn")
])
this.log(`About to upload ${filePath}`)
await fileChooser.accept([filePath])
})
this.log(`About to ask for file chooser in ${chatID}`)
const [fileChooser] = await Promise.all([
this.page.waitForFileChooser(),
this.page.click("#_chat_room_plus_btn")
])
this.log(`About to upload ${filePath}`)
await fileChooser.accept([filePath])
} catch (e) {
this.log(`Failed to upload file to ${chatID}`)
return -1
@ -711,11 +604,9 @@ export default class MessagesPuppeteer {
}
}
_receiveMessages(chatID, messages, skipProcessing = false) {
async _receiveMessages(chatID, messages) {
if (this.client) {
if (!skipProcessing) {
messages = this._processMessages(chatID, messages)
}
messages = await this._processMessages(chatID, messages)
for (const message of messages) {
this.client.sendMessage(message).catch(err =>
this.error("Failed to send message", message.id, "to client:", err))
@ -735,13 +626,11 @@ export default class MessagesPuppeteer {
// TODO Handle unloaded messages. Maybe scroll up
// TODO This will mark the chat as "read"!
await this._switchChat(chatID)
// TODO Is it better to reset the notification count in _switchChat instead of here?
this.numChatNotifications.set(chatID, 0)
let messages = await this.page.evaluate(
const messages = await this.page.evaluate(
mostRecentMessage => window.__mautrixController.parseMessageList(mostRecentMessage),
this.mostRecentMessages.get(chatID))
// Doing this before restoring the observer since it updates minID
messages = this._processMessages(chatID, messages)
const filteredMessages = await this._processMessages(chatID, messages)
if (hadMsgListObserver) {
this.log("Restoring msg list observer")
@ -752,10 +641,10 @@ export default class MessagesPuppeteer {
this.log("Not restoring msg list observer, as there never was one")
}
return messages
return filteredMessages
}
_processMessages(chatID, messages) {
async _processMessages(chatID, messages) {
// TODO Probably don't need minID filtering if Puppeteer context handles it now
const minID = this.mostRecentMessages.get(chatID) || 0
const filteredMessages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id))
@ -766,6 +655,7 @@ export default class MessagesPuppeteer {
this.mostRecentMessages.set(chatID, newLastID)
const range = newFirstID === newLastID ? newFirstID : `${newFirstID}-${newLastID}`
this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filteredMessages.length} newer than ${minID} (${range})`)
for (const message of filteredMessages) {
message.chat_id = chatID
}
@ -775,62 +665,19 @@ export default class MessagesPuppeteer {
}
}
async _processChatListChangeUnsafe(chatListInfo) {
const chatID = chatListInfo.id
async _processChatListChangeUnsafe(chatID) {
this.updatedChats.delete(chatID)
this.log("Processing change to", chatID)
// TODO Also process name/icon changes
const prevNumNotifications = this.numChatNotifications.get(chatID) || 0
const diffNumNotifications = chatListInfo.notificationCount - prevNumNotifications
if (chatListInfo.notificationCount == 0 && diffNumNotifications < 0) {
this.log("Notifications dropped--must have read messages from another LINE client, skip")
this.numChatNotifications.set(chatID, 0)
const messages = await this._getMessagesUnsafe(chatID)
if (messages.length === 0) {
this.log("No new messages found in", chatID)
return
}
const mustSync =
// If >1, a notification was missed. Only way to get them is to view the chat.
// If == 0, might be own message...or just a shuffled chat, or something else.
// To play it safe, just sync them. Should be no harm, as they're viewed already.
diffNumNotifications != 1
// Without placeholders, some messages require visiting their chat to be synced.
|| !this.sendPlaceholders
&& (
// Can only use previews for DMs, because sender can't be found otherwise!
chatListInfo.id.charAt(0) != 'u'
// Sync when lastMsg is a canned message for a non-previewable message type.
|| chatListInfo.lastMsg.endsWith(" sent a photo.")
|| chatListInfo.lastMsg.endsWith(" sent a sticker.")
|| chatListInfo.lastMsg.endsWith(" sent a location.")
// TODO More?
)
let messages
if (!mustSync) {
messages = [{
chat_id: chatListInfo.id,
id: null, // because sidebar messages have no ID
timestamp: null, // because this message was sent right now
is_outgoing: false, // because there's no reliable way to detect own messages...
sender: null, // because there's no way to tell who sent a message
html: chatListInfo.lastMsg,
}]
this.numChatNotifications.set(chatID, chatListInfo.notificationCount)
} else {
messages = await this._getMessagesUnsafe(chatListInfo.id)
this.numChatNotifications.set(chatID, 0)
if (messages.length === 0) {
this.log("No new messages found in", chatListInfo.id)
return
}
}
if (this.client) {
for (const message of messages) {
await this.client.sendMessage(message).catch(err =>
this.error("Failed to send message", message.id || "with no ID", "to client:", err))
this.error("Failed to send message", message.id, "to client:", err))
}
} else {
this.log("No client connected, not sending messages")
@ -838,10 +685,10 @@ export default class MessagesPuppeteer {
}
_receiveChatListChanges(changes) {
this.log(`Received chat list changes: ${changes.map(item => item.id)}`)
this.log("Received chat list changes:", changes)
for (const item of changes) {
if (!this.updatedChats.has(item.id)) {
this.updatedChats.add(item.id)
if (!this.updatedChats.has(item)) {
this.updatedChats.add(item)
this.taskQueue.push(() => this._processChatListChangeUnsafe(item))
.catch(err => this.error("Error handling chat list changes:", err))
}
@ -877,8 +724,18 @@ export default class MessagesPuppeteer {
async _sendEmailCredentials() {
this.log("Inputting login credentials")
await this._enterText(await this.page.$("#line_login_email"), this.login_email)
await this._enterText(await this.page.$("#line_login_pwd"), this.login_password)
// Triple-click input fields to select all existing text and replace it on type
let input
input = await this.page.$("#line_login_email")
await input.click({clickCount: 3})
await input.type(this.login_email)
input = await this.page.$("#line_login_pwd")
await input.click({clickCount: 3})
await input.type(this.login_password)
await this.page.click("button#login_btn")
}
@ -924,7 +781,6 @@ export default class MessagesPuppeteer {
_onLoggedOut() {
this.log("Got logged out!")
this.stopObserving()
this.page.bringToFront()
if (this.client) {
this.client.sendLoggedOut().catch(err =>
this.error("Failed to send logout notice to client:", err))