Inbound sync & sticker improvements

- Handle "decrypting" state of messages
- Handle lazy loading of emoji
- Better handle lazy loading of images/stickers
- Improve reliability of message sending, especially when sending
  several messages quickly
- Use m.sticker events for inbound stickers instead of m.image, and add
  a config to optionally use m.image if desired
- Use proper sizing for emoji, and add config to scale them since they
  are somewhat small
- Deduplicate stickers as best as possible (works until they get a
  different blob URL)
- Add config to disable bridging stickers/emoji
- Send m.notice for inbound messages of unknown type
This commit is contained in:
Andrew Ferrazzutti 2021-06-06 18:15:38 -04:00
parent 712a256dee
commit 6d646e082b
8 changed files with 675 additions and 313 deletions

View File

@ -73,6 +73,9 @@ class Config(BaseBridgeConfig):
copy("bridge.delivery_receipts") copy("bridge.delivery_receipts")
copy("bridge.delivery_error_reports") copy("bridge.delivery_error_reports")
copy("bridge.resend_bridge_info") copy("bridge.resend_bridge_info")
copy("bridge.receive_stickers")
copy("bridge.use_sticker_events")
copy("bridge.emoji_scale_factor")
copy("bridge.command_prefix") copy("bridge.command_prefix")
copy("bridge.user") copy("bridge.user")

View File

@ -29,12 +29,14 @@ class Media:
media_id: str media_id: str
mxc: ContentURI mxc: ContentURI
# TODO Consider whether mime_type, file_name, and size are needed. mime_type: str
file_name: str
size: int
async def insert(self) -> None: async def insert(self) -> None:
q = ("INSERT INTO media (media_id, mxc) " q = ("INSERT INTO media (media_id, mxc, mime_type, file_name, size) "
"VALUES ($1, $2)") "VALUES ($1, $2, $3, $4, $5)")
await self.db.execute(q, self.media_id, self.mxc) await self.db.execute(q, self.media_id, self.mxc, self.mime_type, self.file_name, self.size)
async def update(self) -> None: async def update(self) -> None:
q = ("UPDATE media SET mxc=$2 " q = ("UPDATE media SET mxc=$2 "
@ -42,8 +44,8 @@ class Media:
await self.db.execute(q, self.media_id, self.mxc) await self.db.execute(q, self.media_id, self.mxc)
@classmethod @classmethod
async def get_by_id(cls, media_id: str) -> Optional[ContentURI]: async def get_by_id(cls, media_id: str) -> Optional['DBMedia']:
q = ("SELECT media_id, mxc " q = ("SELECT media_id, mxc, mime_type, file_name, size "
"FROM media WHERE media_id=$1") "FROM media WHERE media_id=$1")
row = await cls.db.fetchrow(q, media_id) row = await cls.db.fetchrow(q, media_id)
if not row: if not row:

View File

@ -104,3 +104,12 @@ async def upgrade_read_receipts(conn: Connection) -> None:
REFERENCES portal (mxid) REFERENCES portal (mxid)
ON DELETE CASCADE ON DELETE CASCADE
)""") )""")
@upgrade_table.register(description="Media metadata")
async def upgrade_deduplicate_blob(conn: Connection) -> None:
await conn.execute("""ALTER TABLE media
ADD COLUMN IF NOT EXISTS mime_type TEXT,
ADD COLUMN IF NOT EXISTS file_name TEXT,
ADD COLUMN IF NOT EXISTS size INTEGER
""")

View File

@ -131,6 +131,12 @@ bridge:
# This field will automatically be changed back to false after it, # This field will automatically be changed back to false after it,
# except if the config file is not writable. # except if the config file is not writable.
resend_bridge_info: false resend_bridge_info: false
# Set this to false to disable bridging stickers and emoji.
receive_stickers: true
# Set this to false to use m.image events for stickers instead of m.sticker.
use_sticker_events: true
# The scale by which to display emojis with. Must be a positive integer.
emoji_scale_factor: 1
# The prefix for commands. Only required in non-management rooms. # The prefix for commands. Only required in non-management rooms.
command_prefix: "!line" command_prefix: "!line"

View File

@ -48,9 +48,9 @@ except ImportError:
StateBridge = EventType.find("m.bridge", EventType.Class.STATE) StateBridge = EventType.find("m.bridge", EventType.Class.STATE)
StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE) StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE)
ReuploadedMediaInfo = NamedTuple('ReuploadedMediaInfo', mxc=Optional[ContentURI], MediaInfo = NamedTuple('MediaInfo', mxc=Optional[ContentURI],
decryption_info=Optional[EncryptedFile], decryption_info=Optional[EncryptedFile],
mime_type=str, file_name=str, size=int) mime_type=str, file_name=str, size=int)
class Portal(DBPortal, BasePortal): class Portal(DBPortal, BasePortal):
@ -112,6 +112,7 @@ class Portal(DBPortal, BasePortal):
cls.loop = bridge.loop cls.loop = bridge.loop
cls.bridge = bridge cls.bridge = bridge
cls.invite_own_puppet_to_pm = cls.config["bridge.invite_own_puppet_to_pm"] cls.invite_own_puppet_to_pm = cls.config["bridge.invite_own_puppet_to_pm"]
cls.emoji_scale_factor = max(int(cls.config["bridge.emoji_scale_factor"]), 1)
NotificationDisabler.puppet_cls = p.Puppet NotificationDisabler.puppet_cls = p.Puppet
NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"] NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"]
@ -228,15 +229,38 @@ class Portal(DBPortal, BasePortal):
self.log.debug(f"Ignoring duplicate message {evt.id}") self.log.debug(f"Ignoring duplicate message {evt.id}")
return return
event_id = None if evt.image and evt.image.url:
if evt.image_url: if not evt.image.is_sticker or self.config["bridge.receive_stickers"]:
# TODO Deduplicate stickers, but only if encryption is disabled media_info = await self._handle_remote_media(
content = await self._handle_remote_photo(source, intent, evt) source, intent, evt.image.url,
if not content: deduplicate=not self.encrypted and evt.image.is_sticker)
content = TextMessageEventContent( image_info = ImageInfo(
msgtype=MessageType.NOTICE, # Element Web doesn't animate PNGs, but setting the mimetype to GIF works.
body="<unbridgeable media>") # (PNG stickers never animate, and PNG images only animate after being clicked on.)
event_id = await self._send_message(intent, content, timestamp=evt.timestamp) # Making this exception since E.W. seems to be the only client that supports inline animated stickers & images.
# TODO Open an E.W. issue for this
# TODO Test Element Android
# TODO Find & test other non-GIF formats for animated images
mimetype="image/gif" if evt.image.is_animated and media_info.mime_type == "image/png" else media_info.mime_type,
size=media_info.size) if media_info else None
else:
media_info = None
send_sticker = self.config["bridge.use_sticker_events"] and evt.image.is_sticker and not self.encrypted and media_info
if send_sticker:
event_id = await intent.send_sticker(
self.mxid, media_info.mxc, image_info, "<sticker>", timestamp=evt.timestamp)
else:
if media_info:
content = MediaMessageEventContent(
url=media_info.mxc, file=media_info.decryption_info,
msgtype=MessageType.IMAGE,
body=media_info.file_name,
info=image_info)
else:
content = TextMessageEventContent(
msgtype=MessageType.NOTICE,
body=f"<{'sticker' if evt.image.is_sticker else 'image'}>")
event_id = await self._send_message(intent, content, timestamp=evt.timestamp)
elif evt.html and not evt.html.isspace(): elif evt.html and not evt.html.isspace():
chunks = [] chunks = []
@ -267,6 +291,7 @@ class Portal(DBPortal, BasePortal):
if msg_html: if msg_html:
msg_html += chunk["data"] msg_html += chunk["data"]
elif ctype == "img": elif ctype == "img":
height = int(chunk.get("height", 19)) * self.emoji_scale_factor
cclass = chunk["class"] cclass = chunk["class"]
if cclass == "emojione": if cclass == "emojione":
alt = chunk["alt"] alt = chunk["alt"]
@ -277,11 +302,11 @@ class Portal(DBPortal, BasePortal):
media_id = f'{chunk.get("data-stickon-pkg-cd", 0)}/{chunk.get("data-stickon-stk-cd", 0)}' media_id = f'{chunk.get("data-stickon-pkg-cd", 0)}/{chunk.get("data-stickon-stk-cd", 0)}'
# NOTE Not encrypting content linked to by HTML tags # NOTE Not encrypting content linked to by HTML tags
if not self.encrypted: if not self.encrypted and self.config["bridge.receive_stickers"]:
media_mxc = await self._get_mxc_for_remote_media(source, intent, chunk["src"], media_id) media_info = await self._handle_remote_media(source, intent, chunk["src"], media_id, deduplicate=True)
if not msg_html: if not msg_html:
msg_html = msg_text msg_html = msg_text
msg_html += f'<img data-mx-emoticon src="{media_mxc}" alt="{alt}" title="{alt}" height="32">' msg_html += f'<img data-mx-emoticon src="{media_info.mxc}" alt="{alt}" title="{alt}" height="{height}">'
msg_text += alt msg_text += alt
content = TextMessageEventContent( content = TextMessageEventContent(
@ -289,16 +314,21 @@ class Portal(DBPortal, BasePortal):
format=Format.HTML if msg_html else None, format=Format.HTML if msg_html else None,
body=msg_text, formatted_body=msg_html) body=msg_text, formatted_body=msg_html)
event_id = await self._send_message(intent, content, timestamp=evt.timestamp) event_id = await self._send_message(intent, content, timestamp=evt.timestamp)
if event_id: else:
if evt.is_outgoing and evt.receipt_count: content = TextMessageEventContent(
await self._handle_receipt(event_id, evt.receipt_count) msgtype=MessageType.NOTICE,
msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id) body="<Unbridgeable message>")
try: event_id = await self._send_message(intent, content, timestamp=evt.timestamp)
await msg.insert()
await self._send_delivery_receipt(event_id) if evt.is_outgoing and evt.receipt_count:
self.log.debug(f"Handled remote message {evt.id} -> {event_id}") await self._handle_receipt(event_id, evt.receipt_count)
except UniqueViolationError as e: msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id)
self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}") try:
await msg.insert()
await self._send_delivery_receipt(event_id)
self.log.debug(f"Handled remote message {evt.id} -> {event_id}")
except UniqueViolationError as e:
self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}")
async def handle_remote_receipt(self, receipt: Receipt) -> None: async def handle_remote_receipt(self, receipt: Receipt) -> None:
msg = await DBMessage.get_by_mid(receipt.id) msg = await DBMessage.get_by_mid(receipt.id)
@ -324,37 +354,34 @@ class Portal(DBPortal, BasePortal):
reaction_mxid = await self.main_intent.react(self.mxid, event_id, f"(Read by {receipt_count})") reaction_mxid = await self.main_intent.react(self.mxid, event_id, f"(Read by {receipt_count})")
await DBReceiptReaction(reaction_mxid, self.mxid, event_id, receipt_count).insert() await DBReceiptReaction(reaction_mxid, self.mxid, event_id, receipt_count).insert()
async def _handle_remote_photo(self, source: 'u.User', intent: IntentAPI, message: Message async def _handle_remote_media(self, source: 'u.User', intent: IntentAPI,
) -> Optional[MediaMessageEventContent]: media_url: str, media_id: Optional[str] = None,
try: deduplicate: bool = False) -> MediaInfo:
resp = await source.client.read_image(message.image_url)
except (RPCError, TypeError) as e:
self.log.warning(f"Failed to download remote photo from chat {self.chat_id}: {e}")
return None
media_info = await self._reupload_remote_media(resp.data, intent, resp.mime)
return MediaMessageEventContent(url=media_info.mxc, file=media_info.decryption_info,
msgtype=MessageType.IMAGE, body=media_info.file_name,
info=ImageInfo(mimetype=media_info.mime_type, size=media_info.size))
async def _get_mxc_for_remote_media(self, source: 'u.User', intent: IntentAPI,
media_url: str, media_id: Optional[str] = None
) -> ContentURI:
if not media_id: if not media_id:
media_id = media_url media_id = media_url
media_info = await DBMedia.get_by_id(media_id) db_media_info = await DBMedia.get_by_id(media_id) if deduplicate else None
if not media_info: if not db_media_info:
self.log.debug(f"Did not find existing mxc URL for {media_id}, uploading media now") # NOTE Blob URL of stickers only persists for a single session...still better than nothing.
resp = await source.client.read_image(media_url) self.log.debug(f"{'Did not find existing mxc URL for' if deduplicate else 'Not deduplicating'} {media_id}, uploading media now")
media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=True) try:
await DBMedia(media_id=media_id, mxc=media_info.mxc).insert() resp = await source.client.read_image(media_url)
self.log.debug(f"Uploaded media as {media_info.mxc}") except (RPCError, TypeError) as e:
self.log.warning(f"Failed to download remote media from chat {self.chat_id}: {e}")
return None
media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=deduplicate)
if deduplicate:
await DBMedia(
media_id=media_id, mxc=media_info.mxc,
size=media_info.size, mime_type=media_info.mime_type, file_name=media_info.file_name
).insert()
return media_info
else: else:
self.log.debug(f"Found existing mxc URL for {media_id}: {media_info.mxc}") self.log.debug(f"Found existing mxc URL for {media_id}: {db_media_info.mxc}")
return media_info.mxc return MediaInfo(db_media_info.mxc, None, db_media_info.mime_type, db_media_info.file_name, db_media_info.size)
async def _reupload_remote_media(self, data: bytes, intent: IntentAPI, async def _reupload_remote_media(self, data: bytes, intent: IntentAPI,
mime_type: str = None, file_name: str = None, mime_type: str = None, file_name: str = None,
disable_encryption: bool = True) -> ReuploadedMediaInfo: disable_encryption: bool = True) -> MediaInfo:
if not mime_type: if not mime_type:
mime_type = magic.from_buffer(data, mime=True) mime_type = magic.from_buffer(data, mime=True)
upload_mime_type = mime_type upload_mime_type = mime_type
@ -372,10 +399,13 @@ class Portal(DBPortal, BasePortal):
filename=upload_file_name) filename=upload_file_name)
if decryption_info: if decryption_info:
self.log.debug(f"Uploaded encrypted media as {mxc}")
decryption_info.url = mxc decryption_info.url = mxc
mxc = None mxc = None
else:
self.log.debug(f"Uploaded media as {mxc}")
return ReuploadedMediaInfo(mxc, decryption_info, mime_type, file_name, len(data)) return MediaInfo(mxc, decryption_info, mime_type, file_name, len(data))
async def update_info(self, conv: ChatInfo, client: Optional[Client]) -> None: async def update_info(self, conv: ChatInfo, client: Optional[Client]) -> None:
if self.is_direct: if self.is_direct:

View File

@ -51,6 +51,13 @@ class ChatInfo(ChatListInfo, SerializableAttrs['ChatInfo']):
participants: List[Participant] participants: List[Participant]
@dataclass
class MessageImage(SerializableAttrs['MessageImage']):
url: str
is_sticker: bool
is_animated: bool
@dataclass @dataclass
class Message(SerializableAttrs['Message']): class Message(SerializableAttrs['Message']):
id: int id: int
@ -59,7 +66,7 @@ class Message(SerializableAttrs['Message']):
sender: Optional[Participant] sender: Optional[Participant]
timestamp: int = None timestamp: int = None
html: Optional[str] = None html: Optional[str] = None
image_url: Optional[str] = None image: Optional[MessageImage] = None
receipt_count: Optional[int] = None receipt_count: Optional[int] = None

View File

@ -69,17 +69,19 @@ window.__mautrixExpiry = function (button) {}
* @return {Promise<void>} * @return {Promise<void>}
*/ */
window.__mautrixReceiveMessageID = function(id) {} window.__mautrixReceiveMessageID = function(id) {}
/**
* @return {Promise<Element>}
*/
window.__mautrixShowParticipantsList = function() {}
/**
* typedef ChatTypeEnum
*/
const ChatTypeEnum = Object.freeze({ const ChatTypeEnum = Object.freeze({
DIRECT: 1, DIRECT: 1,
GROUP: 2, GROUP: 2,
ROOM: 3, ROOM: 3,
}) })
const MSG_DECRYPTING = "ⓘ Decrypting..."
// TODO consts for common selectors
class MautrixController { class MautrixController {
constructor() { constructor() {
this.chatListObserver = null this.chatListObserver = null
@ -92,7 +94,6 @@ class MautrixController {
this.pinAppearObserver = null this.pinAppearObserver = null
this.ownID = null this.ownID = null
this.ownMsgPromise = Promise.resolve(-1)
this._promiseOwnMsgReset() this._promiseOwnMsgReset()
} }
@ -172,28 +173,45 @@ class MautrixController {
* @typedef MessageData * @typedef MessageData
* @type {object} * @type {object}
* @property {number} id - The ID of the message. Seems to be sequential. * @property {number} id - The ID of the message. Seems to be sequential.
* @property {number} timestamp - The unix timestamp of the message. Not very accurate. * @property {number} timestamp - The unix timestamp of the message. Accurate to the minute.
* @property {boolean} is_outgoing - Whether or not this user sent the message. * @property {boolean} is_outgoing - Whether or not this user sent the message.
* @property {?Participant} sender - Full data of the participant who sent the message, if needed and available. * @property {?Participant} sender - Full data of the participant who sent the message, if needed and available.
* @property {?string} html - The HTML format of the message, if necessary. * @property {?string} html - The HTML format of the message, if necessary.
* @property {?string} image_url - The URL to the image in the message, if it's an image-only message. * @property {?ImageInfo} image - Information of the image in the message, if it's an image-only message.
* @property {?int} receipt_count - The number of users who have read the message. * @property {?int} receipt_count - The number of users who have read the message.
*/ */
/**
* @typedef ImageInfo
* @type {object}
* @property {string} url - The URL of the image's location.
* @property {boolean} is_sticker - Whether the sent image is a sticker.
* @property {boolean} animated - Whether the sent image is animated. Only used for stickers (for now...?).
*/
/**
* Return whether a URL points to a loaded image or not.
*
* @param {string} src
* @return boolean
* @private
*/
_isLoadedImageURL(src) { _isLoadedImageURL(src) {
return src && (src.startsWith("blob:") || src.startsWith("res/")) return src && (
src.startsWith(`blob:`) ||
src.startsWith(`${document.location.origin}/res/`) && !src.startsWith(`${document.location.origin}/res/img/noimg/`))
} }
/** /**
* Parse a message element (mws-message-wrapper) * Parse a message element.
* *
* @param {Date} date - The most recent date indicator.
* @param {Element} element - The message element. * @param {Element} element - The message element.
* @param {int} chatType - What kind of chat this message is part of. * @param {Number} chatType - What kind of chat this message is part of.
* @return {MessageData} * @param {Date} refDate - The most recent date indicator. If undefined, do not retrieve the timestamp of this message.
* @return {Promise<MessageData>}
* @private * @private
*/ */
async _parseMessage(date, element, chatType) { async _parseMessage(element, chatType, refDate) {
const is_outgoing = element.classList.contains("mdRGT07Own") const is_outgoing = element.classList.contains("mdRGT07Own")
let sender = {} let sender = {}
@ -208,6 +226,7 @@ class MautrixController {
sender = null sender = null
receipt_count = is_outgoing ? (receipt ? 1 : 0) : null receipt_count = is_outgoing ? (receipt ? 1 : 0) : null
} else if (!is_outgoing) { } else if (!is_outgoing) {
let imgElement
sender.name = element.querySelector(".mdRGT07Body > .mdRGT07Ttl").innerText sender.name = element.querySelector(".mdRGT07Body > .mdRGT07Ttl").innerText
// Room members are always friends (right?), // Room members are always friends (right?),
// so search the friend list for the sender's name // so search the friend list for the sender's name
@ -216,23 +235,23 @@ class MautrixController {
// Group members aren't necessarily friends, // Group members aren't necessarily friends,
// but the participant list includes their ID. // but the participant list includes their ID.
if (!sender.id) { if (!sender.id) {
await window.__mautrixShowParticipantsList()
const participantsList = document.querySelector(participantsListSelector) const participantsList = document.querySelector(participantsListSelector)
sender.id = participantsList.querySelector(`img[alt='${senderName}'`).parentElement.parentElement.getAttribute("data-mid") imgElement = participantsList.querySelector(`img[alt='${sender.name}'`)
sender.id = imgElement.parentElement.parentElement.getAttribute("data-mid")
} else {
imgElement = element.querySelector(".mdRGT07Img > img")
} }
sender.avatar = this.getParticipantListItemAvatar(element) sender.avatar = this._getPathImage(imgElement)
receipt_count = null receipt_count = null
} else { } else {
// TODO Get own ID and store it somewhere appropriate. // TODO Get own ID and store it somewhere appropriate.
// Unable to get own ID from a room chat... // Unable to get own ID from a room chat...
// if (chatType == ChatTypeEnum.GROUP) { // if (chatType == ChatTypeEnum.GROUP) {
// await window.__mautrixShowParticipantsList()
// const participantsList = document.querySelector("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // const participantsList = document.querySelector("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul")
// // TODO The first member is always yourself, right? // // TODO The first member is always yourself, right?
// // TODO Cache this so own ID can be used later // // TODO Cache this so own ID can be used later
// sender = participantsList.children[0].getAttribute("data-mid") // sender = participantsList.children[0].getAttribute("data-mid")
// } // }
await window.__mautrixShowParticipantsList()
const participantsList = document.querySelector(participantsListSelector) const participantsList = document.querySelector(participantsListSelector)
sender.name = this.getParticipantListItemName(participantsList.children[0]) sender.name = this.getParticipantListItemName(participantsList.children[0])
sender.avatar = this.getParticipantListItemAvatar(participantsList.children[0]) sender.avatar = this.getParticipantListItemAvatar(participantsList.children[0])
@ -243,56 +262,175 @@ class MautrixController {
const messageData = { const messageData = {
id: +element.getAttribute("data-local-id"), id: +element.getAttribute("data-local-id"),
timestamp: date ? date.getTime() : null, timestamp:
refDate !== undefined
? (await this._tryParseDate(element.querySelector("time")?.innerText, refDate))?.getTime()
: null,
is_outgoing: is_outgoing, is_outgoing: is_outgoing,
sender: sender, sender: sender,
receipt_count: receipt_count receipt_count: receipt_count,
} }
const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg")
if (messageElement.classList.contains("mdRGT07Text")) {
messageData.html = messageElement.querySelector(".mdRGT07MsgTextInner")?.innerHTML
} else if (
messageElement.classList.contains("mdRGT07Image") ||
messageElement.classList.contains("mdRGT07Sticker")
) {
const img = messageElement.querySelector(".mdRGT07MsgImg > img")
if (img) {
let imgResolve
// TODO Should reject on "#_chat_message_image_failure"
let observer = new MutationObserver(changes => {
for (const change of changes) {
if (this._isLoadedImageURL(change.target.src) && observer) {
observer.disconnect()
observer = null
imgResolve(change.target.src)
return
}
}
})
observer.observe(img, { attributes: true, attributeFilter: ["src"] })
if (this._isLoadedImageURL(img.src)) { const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg")
// Check for this AFTER attaching the observer, in case const is_sticker = messageElement.classList.contains("mdRGT07Sticker")
// the image loaded after the img element was found but if (messageElement.classList.contains("mdRGT07Text")) {
// before the observer was attached. let msgSpan = messageElement.querySelector(".mdRGT07MsgTextInner")
messageData.image_url = img.src try {
observer.disconnect() if (msgSpan.innerHTML == MSG_DECRYPTING) {
} else { msgSpan = await this._waitForDecryptedMessage(element, msgSpan, 5000)
messageData.image_url = await new Promise(resolve => {
imgResolve = resolve
setTimeout(() => {
if (observer) {
observer.disconnect()
resolve(img.src)
}
}, 10000) // Longer timeout for image downloads
})
} }
messageData.html = await this._parseMessageHTML(msgSpan)
} catch {
// Throw to reject, but return what was parsed so far
throw messageData
}
} else if (is_sticker || messageElement.classList.contains("mdRGT07Image")) {
// TODO Animated non-sticker images require clicking its img element, which is just a thumbnail
// Real image: "#wrap_single_image img"
// Close button: "#wrap_single_image button"
// Viewer is open/closed based on "#wrap_single_image.MdNonDisp" / "#wrap_single_image:not(.MdNonDisp)"
let img = messageElement.querySelector(".mdRGT07MsgImg > img")
if (!this._isLoadedImageURL(img.src)) {
try {
img = await this._waitForLoadedImage(img, 10000)
} catch {
// Throw to reject, but return what was parsed so far
throw messageData
}
}
messageData.image = {
url: img.src,
is_sticker: is_sticker,
is_animated: is_sticker && img.parentElement.classList.contains("animationSticker"),
} }
} }
return messageData return messageData
} }
/**
* @param {Element} msgSpan
* @return Promise<DOMString>
* @private
*/
async _parseMessageHTML(msgSpan) {
const msgSpanImgs = msgSpan.getElementsByTagName("img")
if (msgSpanImgs.length == 0) {
return msgSpan.innerHTML
} else {
const unloadedImgs = Array.from(msgSpanImgs).filter(img => !this._isLoadedImageURL(img.src))
if (unloadedImgs.length > 0) {
// NOTE Use allSettled to not throw if any images time out
await Promise.allSettled(
unloadedImgs.map(img => this._waitForLoadedImage(img, 2000))
)
}
// Hack to put sticon dimensions in HTML (which are excluded by default)
// in such a way that doesn't alter the elements that are in the DOM
const msgSpanCopy = msgSpan.cloneNode(true)
const msgSpanCopyImgs = msgSpanCopy.getElementsByTagName("img")
for (let i = 0, n = msgSpanImgs.length; i < n; i++) {
msgSpanCopyImgs[i].height = msgSpanImgs[i].height
msgSpanCopyImgs[i].width = msgSpanImgs[i].width
}
return msgSpanCopy.innerHTML
}
}
/**
* @param {Element} element
* @param {Element} msgSpan
* @param {Number} timeoutLimitMillis
* @return {Promise<Element>}
* @private
*/
_waitForDecryptedMessage(element, msgSpan, timeoutLimitMillis) {
console.debug("Wait for message element to finish decrypting")
console.debug(element)
return new Promise((resolve, reject) => {
let observer = new MutationObserver(changes => {
for (const change of changes) {
const isTextUpdate = change.type == "characterData"
const target = isTextUpdate ? msgSpan : element.querySelector(".mdRGT07MsgTextInner")
if (target && target.innerHTML != MSG_DECRYPTING) {
if (isTextUpdate) {
console.debug("UNLIKELY(?) EVENT -- Found decrypted message from text update")
} else {
// TODO Looks like it's div.mdRGT07Body that gets always replaced. If so, watch only for that
console.debug("Found decrypted message from element replacement")
console.debug(target)
console.debug("Added:")
for (const change of changes) {
console.debug(change.removedNodes)
}
console.debug("Removed:")
for (const change of changes) {
console.debug(change.addedNodes)
}
}
observer.disconnect()
observer = null
resolve(target)
return
}
if (target && target != msgSpan) {
console.debug("UNLIKELY EVENT -- Somehow added a new \"decrypting\" span, it's the one to watch now")
console.debug(target)
msgSpan = target
observer.observe(msgSpan, { characterData: true })
}
}
})
// Either the span element or one of its ancestors is replaced,
// or the span element's content is updated.
// Not exactly sure which of these happens, or if the same kind
// of mutation always happens, so just look for them all...
observer.observe(element, { childList: true, subtree: true })
observer.observe(msgSpan, { characterData: true })
setTimeout(() => {
if (observer) {
observer.disconnect()
// Don't print log message, as this may be a safe timeout
reject()
}
}, timeoutLimitMillis)
})
}
/**
* @param {Element} img
* @param {Number} timeoutLimitMillis
* @return {Promise<Element>}
* @private
*/
_waitForLoadedImage(img, timeoutLimitMillis) {
console.debug("Wait for image element to finish loading")
console.debug(img)
// TODO Should reject on "#_chat_message_image_failure"
return new Promise((resolve, reject) => {
let observer = new MutationObserver(changes => {
for (const change of changes) {
if (this._isLoadedImageURL(change.target.src)) {
console.debug("Image element finished loading")
console.debug(change.target)
observer.disconnect()
observer = null
resolve(change.target)
return
}
}
})
observer.observe(img, { attributes: true, attributeFilter: ["src"] })
setTimeout(() => {
if (observer) {
observer.disconnect()
// Don't print log message, as this may be a safe timeout
reject()
}
}, timeoutLimitMillis)
})
}
/** /**
* Find the number in the "Read #" receipt message. * Find the number in the "Read #" receipt message.
* Don't look for "Read" specifically, to support multiple languages. * Don't look for "Read" specifically, to support multiple languages.
@ -313,8 +451,8 @@ class MautrixController {
* has succeeded or failed to be sent. * has succeeded or failed to be sent.
* *
* @param {int} timeoutLimitMillis - The maximum amount of time to wait for the message to be sent. * @param {int} timeoutLimitMillis - The maximum amount of time to wait for the message to be sent.
* @param {str} successSelector - The selector for the element that indicates the message was sent. * @param {string} successSelector - The selector for the element that indicates the message was sent.
* @param {str} failureSelector - The selector for the element that indicates the message failed to be sent. * @param {?string} failureSelector - The selector for the element that indicates the message failed to be sent.
*/ */
promiseOwnMessage(timeoutLimitMillis, successSelector, failureSelector=null) { promiseOwnMessage(timeoutLimitMillis, successSelector, failureSelector=null) {
this.promiseOwnMsgSuccessSelector = successSelector this.promiseOwnMsgSuccessSelector = successSelector
@ -323,13 +461,13 @@ class MautrixController {
this.ownMsgPromise = new Promise((resolve, reject) => { this.ownMsgPromise = new Promise((resolve, reject) => {
this.promiseOwnMsgResolve = resolve this.promiseOwnMsgResolve = resolve
this.promiseOwnMsgReject = reject this.promiseOwnMsgReject = reject
setTimeout(() => {
if (this.promiseOwnMsgReject) {
console.log("Timeout!")
this._rejectOwnMessage()
}
}, timeoutLimitMillis)
}) })
this.promiseOwnMsgTimeoutID = setTimeout(() => {
if (this.promiseOwnMsgReject) {
console.error("Timed out waiting for own message to be sent")
this._rejectOwnMessage()
}
}, timeoutLimitMillis)
} }
/** /**
@ -338,40 +476,39 @@ class MautrixController {
* @return {Promise<int>} - The ID of the sent message. * @return {Promise<int>} - The ID of the sent message.
*/ */
async waitForOwnMessage() { async waitForOwnMessage() {
return await this.ownMsgPromise return this.ownMsgPromise ? await this.ownMsgPromise : -1
}
async _tryParseMessages(msgList, chatType) {
const messages = []
let refDate = null
for (const child of msgList) {
if (child.classList.contains("mdRGT10Date")) {
refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText)
} else if (child.classList.contains("MdRGT07Cont")) {
// TODO :not(.MdNonDisp) to exclude not-yet-posted messages,
// but that is unlikely to be a problem here.
// Also, offscreen times may have .MdNonDisp on them
// TODO Explicitly look for the most recent date element,
// as it might not have been one of the new items in msgList
const timeElement = child.querySelector("time")
if (timeElement) {
const messageDate = await this._tryParseDate(timeElement.innerText, refDate)
messages.push(await this._parseMessage(messageDate, child, chatType))
}
}
}
return messages
} }
/** /**
* Parse the message list of whatever the currently-viewed chat is. * Parse the message list of whatever the currently-viewed chat is.
* *
* @return {[MessageData]} - A list of messages. * @param {int} minID - The minimum message ID to consider.
* @return {Promise<[MessageData]>} - A list of messages.
*/ */
async parseMessageList() { async parseMessageList(minID = 0) {
const msgList = Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]")) console.debug(`minID for full refresh: ${minID}`)
msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id")) const msgList =
return await this._tryParseMessages(msgList, this.getChatType(this.getCurrentChatID())) Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]"))
.filter(msg =>
msg.hasAttribute("data-local-id") &&
(!msg.classList.contains("MdRGT07Cont") || msg.getAttribute("data-local-id") > minID))
if (msgList.length == 0) {
return []
}
const messagePromises = []
const chatType = this.getChatType(this.getCurrentChatID())
let refDate = null
for (const child of msgList) {
if (child.classList.contains("mdRGT10Date")) {
refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText)
} else if (child.classList.contains("MdRGT07Cont")) {
messagePromises.push(this._parseMessage(child, chatType, refDate))
}
}
// NOTE No message should ever time out, but use allSettled to not throw if any do
return (await Promise.allSettled(messagePromises))
.filter(value => value.status == "fulfilled")
.map(value => value.value)
} }
/** /**
@ -439,7 +576,7 @@ class MautrixController {
const name = this.getParticipantListItemName(child) const name = this.getParticipantListItemName(child)
const id = this.getParticipantListItemID(child) || this.getUserIdFromFriendsList(name) const id = this.getParticipantListItemID(child) || this.getUserIdFromFriendsList(name)
return { return {
id: id, // NOTE Don't want non-own user's ID to ever be null. id: id,
avatar: this.getParticipantListItemAvatar(child), avatar: this.getParticipantListItemAvatar(child),
name: name, name: name,
} }
@ -497,6 +634,7 @@ class MautrixController {
/** /**
* Parse the list of recent/saved chats. * Parse the list of recent/saved chats.
*
* @return {[ChatListInfo]} - The list of chats. * @return {[ChatListInfo]} - The list of chats.
*/ */
parseChatList() { parseChatList() {
@ -505,20 +643,6 @@ class MautrixController {
child => this.parseChatListItem(child.firstElementChild)) child => this.parseChatListItem(child.firstElementChild))
} }
/**
* TODO
* Check if an image has been downloaded.
*
* @param {number} id - The ID of the message whose image to check.
* @return {boolean} - Whether or not the image has been downloaded
*/
imageExists(id) {
const imageElement = document.querySelector(
`mws-message-wrapper[msg-id="${id}"] mws-image-message-part .image-msg`)
return !imageElement.classList.contains("not-rendered")
&& imageElement.getAttribute("src") !== ""
}
/** /**
* Download an image at a given URL and return it as a data URL. * Download an image at a given URL and return it as a data URL.
* *
@ -544,7 +668,6 @@ class MautrixController {
// TODO Observe *added/removed* chats, not just new messages // TODO Observe *added/removed* chats, not just new messages
const changedChatIDs = new Set() const changedChatIDs = new Set()
for (const change of mutations) { for (const change of mutations) {
console.debug("Chat list mutation:", change)
if (change.target.id == "_chat_list_body") { if (change.target.id == "_chat_list_body") {
// TODO // TODO
// These could be new chats, or they're // These could be new chats, or they're
@ -555,16 +678,16 @@ class MautrixController {
*/ */
} else if (change.target.tagName == "LI") { } else if (change.target.tagName == "LI") {
if (change.target.classList.contains("ExSelected")) { if (change.target.classList.contains("ExSelected")) {
console.log("Not using chat list mutation response for currently-active chat") console.debug("Not using chat list mutation response for currently-active chat")
continue continue
} }
for (const node of change.addedNodes) { for (const node of change.addedNodes) {
const chat = this.parseChatListItem(node) const chat = this.parseChatListItem(node)
if (chat) { if (chat) {
console.log("Changed chat list item:", chat) console.log("Added chat list item:", chat)
changedChatIDs.add(chat.id) changedChatIDs.add(chat.id)
} else { } else {
console.debug("Could not parse node as a chat list item:", node) console.debug("Could not parse added node as a chat list item:", node)
} }
} }
} }
@ -584,10 +707,12 @@ class MautrixController {
addChatListObserver() { addChatListObserver() {
this.removeChatListObserver() this.removeChatListObserver()
this.chatListObserver = new MutationObserver(async (mutations) => { this.chatListObserver = new MutationObserver(async (mutations) => {
// Wait for pending sent messages to be resolved before responding to mutations if (this.ownMsgPromise) {
try { // Wait for pending sent messages to be resolved before responding to mutations
await this.ownMsgPromise try {
} catch (e) {} await this.ownMsgPromise
} catch (e) {}
}
try { try {
this._observeChatListMutations(mutations) this._observeChatListMutations(mutations)
@ -598,7 +723,7 @@ class MautrixController {
this.chatListObserver.observe( this.chatListObserver.observe(
document.querySelector("#_chat_list_body"), document.querySelector("#_chat_list_body"),
{ childList: true, subtree: true }) { childList: true, subtree: true })
console.debug("Started chat list observer") console.log("Started chat list observer")
} }
/** /**
@ -608,13 +733,13 @@ class MautrixController {
if (this.chatListObserver !== null) { if (this.chatListObserver !== null) {
this.chatListObserver.disconnect() this.chatListObserver.disconnect()
this.chatListObserver = null this.chatListObserver = null
console.debug("Disconnected chat list observer") console.log("Disconnected chat list observer")
} }
} }
/** /**
* @param {[MutationRecord]} mutations - The mutation records that occurred * @param {[MutationRecord]} mutations - The mutation records that occurred
* @param {str} chatID - The ID of the chat being observed. * @param {string} chatID - The ID of the chat being observed.
* @private * @private
*/ */
_observeReceiptsDirect(mutations, chatID) { _observeReceiptsDirect(mutations, chatID) {
@ -641,7 +766,7 @@ class MautrixController {
/** /**
* @param {[MutationRecord]} mutations - The mutation records that occurred * @param {[MutationRecord]} mutations - The mutation records that occurred
* @param {str} chatID - The ID of the chat being observed. * @param {string} chatID - The ID of the chat being observed.
* @private * @private
*/ */
_observeReceiptsMulti(mutations, chatID) { _observeReceiptsMulti(mutations, chatID) {
@ -649,17 +774,17 @@ class MautrixController {
const receipts = [] const receipts = []
for (const change of mutations) { for (const change of mutations) {
const target = change.type == "characterData" ? change.target.parentElement : change.target const target = change.type == "characterData" ? change.target.parentElement : change.target
if ( change.target.classList.contains("mdRGT07Read") && if ( target.classList.contains("mdRGT07Read") &&
!change.target.classList.contains("MdNonDisp")) !target.classList.contains("MdNonDisp"))
{ {
const msgElement = change.target.closest(".mdRGT07Own") const msgElement = target.closest(".mdRGT07Own")
if (msgElement) { if (msgElement) {
const id = +msgElement.getAttribute("data-local-id") const id = +msgElement.getAttribute("data-local-id")
if (!ids.has(id)) { if (!ids.has(id)) {
ids.add(id) ids.add(id)
receipts.push({ receipts.push({
id: id, id: id,
count: this._getReceiptCount(change.target), count: this._getReceiptCount(target),
}) })
} }
} }
@ -673,11 +798,90 @@ class MautrixController {
} }
} }
/**
* @typedef PendingMessage
* @type object
*
* @property {Promise<MessageData>} promise
* @property {Number} id
*/
/**
* @typedef SameIDMsgs
* @type object
*
* @property {Number} id
* @property {PendingMessage[]} msgs
* @property {Function} resolve
* @property {Number} numRejected
*/
/**
* Binary search for the array of messages with the provided ID.
*
* @param {SameIDMsgs[]} sortedSameIDMsgs
* @param {Number} id
* @param {boolean} returnClosest - If true, return the index of the nearest result on miss instead of -1.
* @return {Number} The index of the matched element, or -1 if not found.
*/
_findMsgsForID(
sortedSameIDMsgs, id, returnClosest = false,
lowerBound = 0, upperBound = sortedSameIDMsgs.length - 1)
{
if (lowerBound > upperBound) {
return -1
}
if (returnClosest && lowerBound == upperBound) {
// Caller must check if the result has a matching ID or not
return sortedSameIDMsgs[lowerBound].id <= id ? lowerBound : lowerBound-1
}
const i = lowerBound + Math.floor((upperBound - lowerBound)/2)
const val = sortedSameIDMsgs[i]
if (val.id == id) {
return i
} else if (val.id < id) {
return this._findMsgsForID(
sortedSameIDMsgs, id, returnClosest,
i+1, upperBound)
} else {
return this._findMsgsForID(
sortedSameIDMsgs, id, returnClosest,
lowerBound, i-1)
}
}
/**
* Insert the given message to the proper inner array.
* In no inner array exists, insert a new one, preserving sort order.
* Return the wrapper of which inner array was added to or created.
*
* @param {SameIDMsgs[]} sortedSameIDMsgs
* @param {PendingMessage} msg
* @return {SameIDMsgs}
*/
_insertMsgByID(sortedSameIDMsgs, msg) {
let i = this._findMsgsForID(sortedSameIDMsgs, msg.id, true)
if (i != -1 && sortedSameIDMsgs[i].id == msg.id) {
sortedSameIDMsgs[i].msgs.push(msg)
console.debug("UNLIKELY(?) EVENT -- Found two new message elements with the same ID, so tracking both of them")
} else {
sortedSameIDMsgs.splice(++i, 0, {
id: msg.id,
msgs: [msg],
numRejected: 0,
resolve: null,
})
}
return sortedSameIDMsgs[i]
}
/** /**
* Add a mutation observer to the message list of the current chat. * Add a mutation observer to the message list of the current chat.
* Used for observing new messages & read receipts. * Used for observing new messages & read receipts.
*
* @param {int} minID - The minimum message ID to consider.
*/ */
addMsgListObserver() { addMsgListObserver(minID = 0) {
const chat_room_msg_list = document.querySelector("#_chat_room_msg_list") const chat_room_msg_list = document.querySelector("#_chat_room_msg_list")
if (!chat_room_msg_list) { if (!chat_room_msg_list) {
console.debug("Could not start msg list observer: no msg list available!") console.debug("Could not start msg list observer: no msg list available!")
@ -688,34 +892,133 @@ class MautrixController {
const chatID = this.getCurrentChatID() const chatID = this.getCurrentChatID()
const chatType = this.getChatType(chatID) const chatType = this.getChatType(chatID)
let orderedPromises = [Promise.resolve()] // NEED TO HANDLE:
// * message elements arriving in any order
// * messages being potentially pending (i.e. decrypting or loading),
// and resolving in a potentially different order than they arrived in
// * pending messages potentially having multiple elements associated with
// them, where only one of them resolves
// * message elements being added/removed any number of times, which may
// or may not ever resolve
// * outgoing messages (i.e. sent by the bridge)
// And must send resolved messages to the bridge *in order*!
// BUT: Assuming that incoming messages will never be younger than a resolved one.
const sortedSameIDMsgs = []
const pendingMsgElements = new Set()
this.msgListObserver = new MutationObserver(changes => { this.msgListObserver = new MutationObserver(changes => {
let msgList = [] console.debug(`MESSAGE LIST CHANGES: check since ${minID}`)
const remoteMsgs = []
for (const change of changes) { for (const change of changes) {
change.addedNodes.forEach(child => { console.debug("---new change set---")
if (child.tagName == "DIV" && child.hasAttribute("data-local-id")) { for (const child of change.addedNodes) {
msgList.push(child) if (!pendingMsgElements.has(child) &&
child.tagName == "DIV" &&
child.hasAttribute("data-local-id") &&
// Skip timestamps, as these are always current
child.classList.contains("MdRGT07Cont"))
{
const msgID = child.getAttribute("data-local-id")
if (msgID > minID) {
pendingMsgElements.add(child)
// TODO Maybe handle own messages somewhere else...?
const ownMsg = this._observeOwnMessage(child)
if (ownMsg) {
console.log("Found own bridge-sent message, will wait for it to resolve")
console.debug(child)
this.ownMsgPromise
.then(msgID => {
console.log("Resolved own bridge-sent message")
console.debug(ownMsg)
pendingMsgElements.delete(ownMsg)
if (minID < msgID) {
minID = msgID
}
})
.catch(() => {
console.log("Rejected own bridge-sent message")
console.debug(ownMsg)
pendingMsgElements.delete(ownMsg)
})
} else {
console.log("Found remote message")
console.debug(child)
remoteMsgs.push({
id: msgID,
element: child
})
}
}
} }
}) }
// NOTE Ignoring removedNodes because an element can always be added back.
// Will simply let permanently-removed nodes time out.
} }
if (msgList.length == 0) { if (remoteMsgs.length == 0) {
console.debug("Found no new remote messages")
return return
} }
msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id"))
if (!this._observeOwnMessage(msgList)) { // No need to sort remoteMsgs, because sortedSameIDMsgs is enough
let prevPromise = orderedPromises.shift() for (const msg of remoteMsgs) {
orderedPromises.push(new Promise(resolve => prevPromise const messageElement = msg.element
.then(() => this._tryParseMessages(msgList, chatType)) const pendingMessage = {
.then(msgs => window.__mautrixReceiveMessages(chatID, msgs)) id: msg.id,
.then(() => resolve()) promise: this._parseMessage(messageElement, chatType)
)) }
const sameIDMsgs = this._insertMsgByID(sortedSameIDMsgs, pendingMessage)
const handleMessage = async (messageData) => {
minID = messageData.id
sortedSameIDMsgs.shift()
await window.__mautrixReceiveMessages(chatID, [messageData])
if (sortedSameIDMsgs.length > 0 && sortedSameIDMsgs[0].resolve) {
console.debug("Allowing queued resolved message to be sent")
console.debug(sortedSameIDMsgs[0])
sortedSameIDMsgs[0].resolve()
}
}
pendingMessage.promise.then(
async (messageData) => {
const i = this._findMsgsForID(sortedSameIDMsgs, messageData.id)
if (i == -1) {
console.debug(`Got resolved message for already-handled ID ${messageData.id}, ignore it`)
pendingMsgElements.delete(messageElement)
return
}
if (i != 0) {
console.debug(`Got resolved message for later ID ${messageData.id}, wait for earlier messages`)
await new Promise(resolve => sameIDMsgs.resolve = resolve)
console.debug(`Message before ID ${messageData.id} finished, can now send this one`)
} else {
console.debug(`Got resolved message for earliest ID ${messageData.id}, send it`)
}
console.debug(messageElement)
pendingMsgElements.delete(messageElement)
handleMessage(messageData)
},
// error case
async (messageData) => {
console.debug("Message element rejected")
console.debug(messageElement)
pendingMsgElements.delete(messageElement)
if (++sameIDMsgs.numRejected == sameIDMsgs.msgs.length) {
// Note that if another message element with this ID somehow comes later, it'll be ignored.
console.debug(`All messages for ID ${sameIDMsgs.id} rejected, abandoning this ID and sending dummy message`)
// Choice of which message to send should be arbitrary
handleMessage(messageData)
}
})
} }
}) })
this.msgListObserver.observe( this.msgListObserver.observe(
chat_room_msg_list, chat_room_msg_list,
{ childList: true }) { childList: true })
console.debug("Started msg list observer") console.debug(`Started msg list observer with minID = ${minID}`)
const observeReadReceipts = ( const observeReadReceipts = (
@ -736,74 +1039,65 @@ class MautrixController {
subtree: true, subtree: true,
attributes: true, attributes: true,
attributeFilter: ["class"], attributeFilter: ["class"],
// TODO Consider using the same observer to watch for "ⓘ Decrypting..."
characterData: chatType != ChatTypeEnum.DIRECT, characterData: chatType != ChatTypeEnum.DIRECT,
}) })
console.debug("Started receipt observer") console.debug("Started receipt observer")
} }
_observeOwnMessage(msgList) { _observeOwnMessage(ownMsg) {
if (!this.promiseOwnMsgSuccessSelector) { if (!this.ownMsgPromise) {
// Not waiting for a pending sent message // Not waiting for a pending sent message
return false return null
}
if (this.visibleSuccessObserver) {
// Already found a element that we're waiting on becoming visible
return true
} }
for (const ownMsg of msgList.filter(msg => msg.classList.contains("mdRGT07Own"))) { const successElement =
const successElement = ownMsg.querySelector(this.promiseOwnMsgSuccessSelector)
ownMsg.querySelector(this.promiseOwnMsgSuccessSelector) if (successElement) {
if (successElement) { if (successElement.classList.contains("MdNonDisp")) {
if (successElement.classList.contains("MdNonDisp")) { console.log("Invisible success for own bridge-sent message, will wait for it to resolve")
console.log("Invisible success") console.log(successElement)
console.log(successElement)
} else {
console.debug("Already visible success, must not be it")
console.debug(successElement)
continue
}
} else { } else {
continue console.debug("Already visible success, must not be it")
console.debug(successElement)
return null
} }
} else {
const failureElement = return null
this.promiseOwnMsgFailureSelector &&
ownMsg.querySelector(this.promiseOwnMsgFailureSelector)
if (failureElement) {
if (failureElement.classList.contains("MdNonDisp")) {
console.log("Invisible failure")
console.log(failureElement)
} else {
console.debug("Already visible failure, must not be it")
console.log(failureElement)
continue
}
} else if (this.promiseOwnMsgFailureSelector) {
continue
}
console.log("Found invisible element, wait")
const msgID = +ownMsg.getAttribute("data-local-id")
this.visibleSuccessObserver = new MutationObserver(
this._getOwnVisibleCallback(msgID))
this.visibleSuccessObserver.observe(
successElement,
{ attributes: true, attributeFilter: ["class"] })
if (this.promiseOwnMsgFailureSelector) {
this.visibleFailureObserver = new MutationObserver(
this._getOwnVisibleCallback())
this.visibleFailureObserver.observe(
failureElement,
{ attributes: true, attributeFilter: ["class"] })
}
return true
} }
return false
const failureElement =
this.promiseOwnMsgFailureSelector &&
ownMsg.querySelector(this.promiseOwnMsgFailureSelector)
if (failureElement) {
if (failureElement.classList.contains("MdNonDisp")) {
console.log("Invisible failure for own bridge-sent message, will wait for it (or success) to resolve")
console.log(failureElement)
} else {
console.debug("Already visible failure, must not be it")
console.log(failureElement)
return null
}
} else if (this.promiseOwnMsgFailureSelector) {
return null
}
const msgID = +ownMsg.getAttribute("data-local-id")
this.visibleSuccessObserver = new MutationObserver(
this._getOwnVisibleCallback(msgID))
this.visibleSuccessObserver.observe(
successElement,
{ attributes: true, attributeFilter: ["class"] })
if (this.promiseOwnMsgFailureSelector) {
this.visibleFailureObserver = new MutationObserver(
this._getOwnVisibleCallback())
this.visibleFailureObserver.observe(
failureElement,
{ attributes: true, attributeFilter: ["class"] })
}
return ownMsg
} }
_getOwnVisibleCallback(msgID=null) { _getOwnVisibleCallback(msgID=null) {
@ -811,7 +1105,7 @@ class MautrixController {
return changes => { return changes => {
for (const change of changes) { for (const change of changes) {
if (!change.target.classList.contains("MdNonDisp")) { if (!change.target.classList.contains("MdNonDisp")) {
console.log(`Waited for visible ${isSuccess ? "success" : "failure"}`) console.log(`Resolved ${isSuccess ? "success" : "failure"} for own bridge-sent message`)
console.log(change.target) console.log(change.target)
isSuccess ? this._resolveOwnMessage(msgID) : this._rejectOwnMessage(change.target) isSuccess ? this._resolveOwnMessage(msgID) : this._rejectOwnMessage(change.target)
return return
@ -822,6 +1116,7 @@ class MautrixController {
_resolveOwnMessage(msgID) { _resolveOwnMessage(msgID) {
if (!this.promiseOwnMsgResolve) return if (!this.promiseOwnMsgResolve) return
clearTimeout(this.promiseOwnMsgTimeoutID)
const resolve = this.promiseOwnMsgResolve const resolve = this.promiseOwnMsgResolve
this._promiseOwnMsgReset() this._promiseOwnMsgReset()
@ -838,10 +1133,12 @@ class MautrixController {
} }
_promiseOwnMsgReset() { _promiseOwnMsgReset() {
this.ownMsgPromise = null
this.promiseOwnMsgSuccessSelector = null this.promiseOwnMsgSuccessSelector = null
this.promiseOwnMsgFailureSelector = null this.promiseOwnMsgFailureSelector = null
this.promiseOwnMsgResolve = null this.promiseOwnMsgResolve = null
this.promiseOwnMsgReject = null this.promiseOwnMsgReject = null
this.promiseOwnMsgTimeoutID = null
if (this.visibleSuccessObserver) { if (this.visibleSuccessObserver) {
this.visibleSuccessObserver.disconnect() this.visibleSuccessObserver.disconnect()

View File

@ -107,7 +107,6 @@ export default class MessagesPuppeteer {
this._receiveReceiptDirectLatest.bind(this)) this._receiveReceiptDirectLatest.bind(this))
await this.page.exposeFunction("__mautrixReceiveReceiptMulti", await this.page.exposeFunction("__mautrixReceiveReceiptMulti",
this._receiveReceiptMulti.bind(this)) this._receiveReceiptMulti.bind(this))
await this.page.exposeFunction("__mautrixShowParticipantsList", this._showParticipantList.bind(this))
await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate)
// NOTE Must *always* re-login on a browser session, so no need to check if already logged in // NOTE Must *always* re-login on a browser session, so no need to check if already logged in
@ -376,15 +375,7 @@ export default class MessagesPuppeteer {
* @return {Promise<[MessageData]>} - The messages visible in the chat. * @return {Promise<[MessageData]>} - The messages visible in the chat.
*/ */
async getMessages(chatID) { async getMessages(chatID) {
return await this.taskQueue.push(async () => { return await this.taskQueue.push(async () => this._getMessagesUnsafe(chatID))
const messages = await this._getMessagesUnsafe(chatID)
if (messages.length > 0) {
for (const message of messages) {
message.chat_id = chatID
}
}
return messages
})
} }
setLastMessageIDs(ids) { setLastMessageIDs(ids) {
@ -392,7 +383,8 @@ export default class MessagesPuppeteer {
for (const [chatID, messageID] of Object.entries(ids)) { for (const [chatID, messageID] of Object.entries(ids)) {
this.mostRecentMessages.set(chatID, messageID) this.mostRecentMessages.set(chatID, messageID)
} }
this.log("Updated most recent message ID map:", this.mostRecentMessages) this.log("Updated most recent message ID map:")
this.log(this.mostRecentMessages)
} }
async readImage(imageUrl) { async readImage(imageUrl) {
@ -407,11 +399,15 @@ export default class MessagesPuppeteer {
} }
async startObserving() { async startObserving() {
this.log("Adding observers") const chatID = await this.page.evaluate(() => window.__mautrixController.getCurrentChatID())
this.log(`Adding observers for ${chatID || "empty chat"}`)
await this.page.evaluate( await this.page.evaluate(
() => window.__mautrixController.addChatListObserver()) () => window.__mautrixController.addChatListObserver())
await this.page.evaluate( if (chatID) {
() => window.__mautrixController.addMsgListObserver()) await this.page.evaluate(
(mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage),
this.mostRecentMessages.get(chatID))
}
} }
async stopObserving() { async stopObserving() {
@ -444,9 +440,10 @@ export default class MessagesPuppeteer {
if (await this.page.evaluate(isCorrectChatVisible, chatName)) { if (await this.page.evaluate(isCorrectChatVisible, chatName)) {
this.log("Already viewing chat, no need to switch") this.log("Already viewing chat, no need to switch")
} else { } else {
this.log("Switching chat, so remove msg list observer") this.log("Ensuring msg list observer is removed")
const hadMsgListObserver = await this.page.evaluate( const hadMsgListObserver = await this.page.evaluate(
() => window.__mautrixController.removeMsgListObserver()) () => window.__mautrixController.removeMsgListObserver())
this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer")
await chatListItem.click() await chatListItem.click()
this.log(`Waiting for chat header title to be "${chatName}"`) this.log(`Waiting for chat header title to be "${chatName}"`)
@ -455,39 +452,28 @@ export default class MessagesPuppeteer {
{polling: "mutation"}, {polling: "mutation"},
chatName) chatName)
// For consistent behaviour later, wait for the chat details sidebar to be hidden // Always show the chat details sidebar, as this makes life easier
this.log("Waiting for detail area to be auto-hidden upon entering chat")
await this.page.waitForFunction( await this.page.waitForFunction(
detailArea => detailArea.childElementCount == 0, detailArea => detailArea.childElementCount == 0,
{}, {},
await this.page.$("#_chat_detail_area")) await this.page.$("#_chat_detail_area"))
this.log("Clicking chat header to show detail area")
await this.page.click("#_chat_header_area > .mdRGT04Link")
this.log("Waiting for detail area")
await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info")
if (hadMsgListObserver) { if (hadMsgListObserver) {
this.log("Restoring msg list observer") this.log("Restoring msg list observer")
await this.page.evaluate( await this.page.evaluate(
() => window.__mautrixController.addMsgListObserver()) (mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage),
this.mostRecentMessages.get(chatID))
} else { } else {
this.log("Not restoring msg list observer, as there never was one") this.log("Not restoring msg list observer, as there never was one")
} }
} }
} }
// TODO Commonize
async _getParticipantList() {
await this._showParticipantList()
return await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul")
}
async _showParticipantList() {
const selector = "#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul"
let participantList = await this.page.$(selector)
if (!participantList) {
this.log("Participant list hidden, so clicking chat header to show it")
await this.page.click("#_chat_header_area > .mdRGT04Link")
// Use no timeout since the browser itself is using this
await this.page.waitForSelector(selector, {timeout: 0})
}
}
async _getChatInfoUnsafe(chatID) { async _getChatInfoUnsafe(chatID) {
const chatListItem = await this.page.$(this._listItemSelector(chatID)) const chatListItem = await this.page.$(this._listItemSelector(chatID))
const chatListInfo = await chatListItem.evaluate( const chatListInfo = await chatListItem.evaluate(
@ -512,7 +498,7 @@ export default class MessagesPuppeteer {
this.log("Found multi-user chat, so clicking chat header to get participants") this.log("Found multi-user chat, so clicking chat header to get participants")
// TODO This will mark the chat as "read"! // TODO This will mark the chat as "read"!
await this._switchChat(chatID) await this._switchChat(chatID)
const participantList = await this._getParticipantList() const participantList = await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul")
// TODO Is a group not actually created until a message is sent(?) // TODO Is a group not actually created until a message is sent(?)
// If so, maybe don't create a portal until there is a message. // If so, maybe don't create a portal until there is a message.
participants = await participantList.evaluate( participants = await participantList.evaluate(
@ -542,6 +528,7 @@ export default class MessagesPuppeteer {
async _sendMessageUnsafe(chatID, text) { async _sendMessageUnsafe(chatID, text) {
await this._switchChat(chatID) await this._switchChat(chatID)
// TODO Initiate the promise in the content script
await this.page.evaluate( await this.page.evaluate(
() => window.__mautrixController.promiseOwnMessage(5000, "time")) () => window.__mautrixController.promiseOwnMessage(5000, "time"))
@ -593,15 +580,12 @@ export default class MessagesPuppeteer {
} }
} }
_receiveMessages(chatID, messages) { async _receiveMessages(chatID, messages) {
if (this.client) { if (this.client) {
messages = this._filterMessages(chatID, messages) messages = await this._processMessages(chatID, messages)
if (messages.length > 0) { for (const message of messages) {
for (const message of messages) { this.client.sendMessage(message).catch(err =>
message.chat_id = chatID this.error("Failed to send message", message.id, "to client:", err))
this.client.sendMessage(message).catch(err =>
this.error("Failed to send message", message.id, "to client:", err))
}
} }
} else { } else {
this.log("No client connected, not sending messages") this.log("No client connected, not sending messages")
@ -609,27 +593,52 @@ export default class MessagesPuppeteer {
} }
async _getMessagesUnsafe(chatID) { async _getMessagesUnsafe(chatID) {
// TODO Also handle "decrypting" state // TODO Consider making a wrapper for pausing/resuming the msg list observers
this.log("Ensuring msg list observer is removed")
const hadMsgListObserver = await this.page.evaluate(
() => window.__mautrixController.removeMsgListObserver())
this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer")
// TODO Handle unloaded messages. Maybe scroll up // TODO Handle unloaded messages. Maybe scroll up
// TODO This will mark the chat as "read"! // TODO This will mark the chat as "read"!
await this._switchChat(chatID) await this._switchChat(chatID)
const messages = await this.page.evaluate(() => const messages = await this.page.evaluate(
window.__mautrixController.parseMessageList()) mostRecentMessage => window.__mautrixController.parseMessageList(mostRecentMessage),
return this._filterMessages(chatID, messages) this.mostRecentMessages.get(chatID))
// Doing this before restoring the observer since it updates minID
const filteredMessages = await this._processMessages(chatID, messages)
if (hadMsgListObserver) {
this.log("Restoring msg list observer")
await this.page.evaluate(
mostRecentMessage => window.__mautrixController.addMsgListObserver(mostRecentMessage),
this.mostRecentMessages.get(chatID))
} else {
this.log("Not restoring msg list observer, as there never was one")
}
return filteredMessages
} }
_filterMessages(chatID, messages) { async _processMessages(chatID, messages) {
// TODO Probably don't need minID filtering if Puppeteer context handles it now
const minID = this.mostRecentMessages.get(chatID) || 0 const minID = this.mostRecentMessages.get(chatID) || 0
const filtered_messages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id)) const filteredMessages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id))
if (filtered_messages.length > 0) { if (filteredMessages.length > 0) {
const newFirstID = filtered_messages[0].id const newFirstID = filteredMessages[0].id
const newLastID = filtered_messages[filtered_messages.length - 1].id const newLastID = filteredMessages[filteredMessages.length - 1].id
this.mostRecentMessages.set(chatID, newLastID) this.mostRecentMessages.set(chatID, newLastID)
const range = newFirstID === newLastID ? newFirstID : `${newFirstID}-${newLastID}` const range = newFirstID === newLastID ? newFirstID : `${newFirstID}-${newLastID}`
this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filtered_messages.length} newer than ${minID} (${range})`) this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filteredMessages.length} newer than ${minID} (${range})`)
for (const message of filteredMessages) {
message.chat_id = chatID
}
return filteredMessages
} else {
return []
} }
return filtered_messages
} }
async _processChatListChangeUnsafe(chatID) { async _processChatListChangeUnsafe(chatID) {
@ -643,7 +652,6 @@ export default class MessagesPuppeteer {
if (this.client) { if (this.client) {
for (const message of messages) { for (const message of messages) {
message.chat_id = chatID
await this.client.sendMessage(message).catch(err => await this.client.sendMessage(message).catch(err =>
this.error("Failed to send message", message.id, "to client:", err)) this.error("Failed to send message", message.id, "to client:", err))
} }