From 6d646e082b46f6dc7f7b11eff456af87a95c75ca Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:15:38 -0400 Subject: [PATCH] Inbound sync & sticker improvements - Handle "decrypting" state of messages - Handle lazy loading of emoji - Better handle lazy loading of images/stickers - Improve reliability of message sending, especially when sending several messages quickly - Use m.sticker events for inbound stickers instead of m.image, and add a config to optionally use m.image if desired - Use proper sizing for emoji, and add config to scale them since they are somewhat small - Deduplicate stickers as best as possible (works until they get a different blob URL) - Add config to disable bridging stickers/emoji - Send m.notice for inbound messages of unknown type --- matrix_puppeteer_line/config.py | 3 + matrix_puppeteer_line/db/media.py | 14 +- matrix_puppeteer_line/db/upgrade.py | 11 +- matrix_puppeteer_line/example-config.yaml | 6 + matrix_puppeteer_line/portal.py | 132 ++-- matrix_puppeteer_line/rpc/types.py | 9 +- puppet/src/contentscript.js | 695 +++++++++++++++------- puppet/src/puppet.js | 118 ++-- 8 files changed, 675 insertions(+), 313 deletions(-) diff --git a/matrix_puppeteer_line/config.py b/matrix_puppeteer_line/config.py index 7ab0b7d..b791561 100644 --- a/matrix_puppeteer_line/config.py +++ b/matrix_puppeteer_line/config.py @@ -73,6 +73,9 @@ class Config(BaseBridgeConfig): copy("bridge.delivery_receipts") copy("bridge.delivery_error_reports") copy("bridge.resend_bridge_info") + copy("bridge.receive_stickers") + copy("bridge.use_sticker_events") + copy("bridge.emoji_scale_factor") copy("bridge.command_prefix") copy("bridge.user") diff --git a/matrix_puppeteer_line/db/media.py b/matrix_puppeteer_line/db/media.py index 24c71b3..cab5d90 100644 --- a/matrix_puppeteer_line/db/media.py +++ b/matrix_puppeteer_line/db/media.py @@ -29,12 +29,14 @@ class Media: media_id: str mxc: ContentURI - # TODO Consider whether mime_type, file_name, and size are needed. + mime_type: str + file_name: str + size: int async def insert(self) -> None: - q = ("INSERT INTO media (media_id, mxc) " - "VALUES ($1, $2)") - await self.db.execute(q, self.media_id, self.mxc) + q = ("INSERT INTO media (media_id, mxc, mime_type, file_name, size) " + "VALUES ($1, $2, $3, $4, $5)") + await self.db.execute(q, self.media_id, self.mxc, self.mime_type, self.file_name, self.size) async def update(self) -> None: q = ("UPDATE media SET mxc=$2 " @@ -42,8 +44,8 @@ class Media: await self.db.execute(q, self.media_id, self.mxc) @classmethod - async def get_by_id(cls, media_id: str) -> Optional[ContentURI]: - q = ("SELECT media_id, mxc " + async def get_by_id(cls, media_id: str) -> Optional['DBMedia']: + q = ("SELECT media_id, mxc, mime_type, file_name, size " "FROM media WHERE media_id=$1") row = await cls.db.fetchrow(q, media_id) if not row: diff --git a/matrix_puppeteer_line/db/upgrade.py b/matrix_puppeteer_line/db/upgrade.py index 9a3b5da..d863009 100644 --- a/matrix_puppeteer_line/db/upgrade.py +++ b/matrix_puppeteer_line/db/upgrade.py @@ -103,4 +103,13 @@ async def upgrade_read_receipts(conn: Connection) -> None: FOREIGN KEY (mx_room) REFERENCES portal (mxid) ON DELETE CASCADE - )""") \ No newline at end of file + )""") + + +@upgrade_table.register(description="Media metadata") +async def upgrade_deduplicate_blob(conn: Connection) -> None: + await conn.execute("""ALTER TABLE media + ADD COLUMN IF NOT EXISTS mime_type TEXT, + ADD COLUMN IF NOT EXISTS file_name TEXT, + ADD COLUMN IF NOT EXISTS size INTEGER + """) \ No newline at end of file diff --git a/matrix_puppeteer_line/example-config.yaml b/matrix_puppeteer_line/example-config.yaml index 3c922e2..b519223 100644 --- a/matrix_puppeteer_line/example-config.yaml +++ b/matrix_puppeteer_line/example-config.yaml @@ -131,6 +131,12 @@ bridge: # This field will automatically be changed back to false after it, # except if the config file is not writable. resend_bridge_info: false + # Set this to false to disable bridging stickers and emoji. + receive_stickers: true + # Set this to false to use m.image events for stickers instead of m.sticker. + use_sticker_events: true + # The scale by which to display emojis with. Must be a positive integer. + emoji_scale_factor: 1 # The prefix for commands. Only required in non-management rooms. command_prefix: "!line" diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 276aa5f..b577dee 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -48,9 +48,9 @@ except ImportError: StateBridge = EventType.find("m.bridge", EventType.Class.STATE) StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE) -ReuploadedMediaInfo = NamedTuple('ReuploadedMediaInfo', mxc=Optional[ContentURI], - decryption_info=Optional[EncryptedFile], - mime_type=str, file_name=str, size=int) +MediaInfo = NamedTuple('MediaInfo', mxc=Optional[ContentURI], + decryption_info=Optional[EncryptedFile], + mime_type=str, file_name=str, size=int) class Portal(DBPortal, BasePortal): @@ -112,6 +112,7 @@ class Portal(DBPortal, BasePortal): cls.loop = bridge.loop cls.bridge = bridge cls.invite_own_puppet_to_pm = cls.config["bridge.invite_own_puppet_to_pm"] + cls.emoji_scale_factor = max(int(cls.config["bridge.emoji_scale_factor"]), 1) NotificationDisabler.puppet_cls = p.Puppet NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"] @@ -228,15 +229,38 @@ class Portal(DBPortal, BasePortal): self.log.debug(f"Ignoring duplicate message {evt.id}") return - event_id = None - if evt.image_url: - # TODO Deduplicate stickers, but only if encryption is disabled - content = await self._handle_remote_photo(source, intent, evt) - if not content: - content = TextMessageEventContent( - msgtype=MessageType.NOTICE, - body="") - event_id = await self._send_message(intent, content, timestamp=evt.timestamp) + if evt.image and evt.image.url: + if not evt.image.is_sticker or self.config["bridge.receive_stickers"]: + media_info = await self._handle_remote_media( + source, intent, evt.image.url, + deduplicate=not self.encrypted and evt.image.is_sticker) + image_info = ImageInfo( + # Element Web doesn't animate PNGs, but setting the mimetype to GIF works. + # (PNG stickers never animate, and PNG images only animate after being clicked on.) + # Making this exception since E.W. seems to be the only client that supports inline animated stickers & images. + # TODO Open an E.W. issue for this + # TODO Test Element Android + # TODO Find & test other non-GIF formats for animated images + mimetype="image/gif" if evt.image.is_animated and media_info.mime_type == "image/png" else media_info.mime_type, + size=media_info.size) if media_info else None + else: + media_info = None + send_sticker = self.config["bridge.use_sticker_events"] and evt.image.is_sticker and not self.encrypted and media_info + if send_sticker: + event_id = await intent.send_sticker( + self.mxid, media_info.mxc, image_info, "", timestamp=evt.timestamp) + else: + if media_info: + content = MediaMessageEventContent( + url=media_info.mxc, file=media_info.decryption_info, + msgtype=MessageType.IMAGE, + body=media_info.file_name, + info=image_info) + else: + content = TextMessageEventContent( + msgtype=MessageType.NOTICE, + body=f"<{'sticker' if evt.image.is_sticker else 'image'}>") + event_id = await self._send_message(intent, content, timestamp=evt.timestamp) elif evt.html and not evt.html.isspace(): chunks = [] @@ -267,6 +291,7 @@ class Portal(DBPortal, BasePortal): if msg_html: msg_html += chunk["data"] elif ctype == "img": + height = int(chunk.get("height", 19)) * self.emoji_scale_factor cclass = chunk["class"] if cclass == "emojione": alt = chunk["alt"] @@ -277,11 +302,11 @@ class Portal(DBPortal, BasePortal): media_id = f'{chunk.get("data-stickon-pkg-cd", 0)}/{chunk.get("data-stickon-stk-cd", 0)}' # NOTE Not encrypting content linked to by HTML tags - if not self.encrypted: - media_mxc = await self._get_mxc_for_remote_media(source, intent, chunk["src"], media_id) + if not self.encrypted and self.config["bridge.receive_stickers"]: + media_info = await self._handle_remote_media(source, intent, chunk["src"], media_id, deduplicate=True) if not msg_html: msg_html = msg_text - msg_html += f'{alt}' + msg_html += f'{alt}' msg_text += alt content = TextMessageEventContent( @@ -289,16 +314,21 @@ class Portal(DBPortal, BasePortal): format=Format.HTML if msg_html else None, body=msg_text, formatted_body=msg_html) event_id = await self._send_message(intent, content, timestamp=evt.timestamp) - if event_id: - if evt.is_outgoing and evt.receipt_count: - await self._handle_receipt(event_id, evt.receipt_count) - msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id) - try: - await msg.insert() - await self._send_delivery_receipt(event_id) - self.log.debug(f"Handled remote message {evt.id} -> {event_id}") - except UniqueViolationError as e: - self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}") + else: + content = TextMessageEventContent( + msgtype=MessageType.NOTICE, + body="") + event_id = await self._send_message(intent, content, timestamp=evt.timestamp) + + if evt.is_outgoing and evt.receipt_count: + await self._handle_receipt(event_id, evt.receipt_count) + msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id) + try: + await msg.insert() + await self._send_delivery_receipt(event_id) + self.log.debug(f"Handled remote message {evt.id} -> {event_id}") + except UniqueViolationError as e: + self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}") async def handle_remote_receipt(self, receipt: Receipt) -> None: msg = await DBMessage.get_by_mid(receipt.id) @@ -324,37 +354,34 @@ class Portal(DBPortal, BasePortal): reaction_mxid = await self.main_intent.react(self.mxid, event_id, f"(Read by {receipt_count})") await DBReceiptReaction(reaction_mxid, self.mxid, event_id, receipt_count).insert() - async def _handle_remote_photo(self, source: 'u.User', intent: IntentAPI, message: Message - ) -> Optional[MediaMessageEventContent]: - try: - resp = await source.client.read_image(message.image_url) - except (RPCError, TypeError) as e: - self.log.warning(f"Failed to download remote photo from chat {self.chat_id}: {e}") - return None - media_info = await self._reupload_remote_media(resp.data, intent, resp.mime) - return MediaMessageEventContent(url=media_info.mxc, file=media_info.decryption_info, - msgtype=MessageType.IMAGE, body=media_info.file_name, - info=ImageInfo(mimetype=media_info.mime_type, size=media_info.size)) - - async def _get_mxc_for_remote_media(self, source: 'u.User', intent: IntentAPI, - media_url: str, media_id: Optional[str] = None - ) -> ContentURI: + async def _handle_remote_media(self, source: 'u.User', intent: IntentAPI, + media_url: str, media_id: Optional[str] = None, + deduplicate: bool = False) -> MediaInfo: if not media_id: media_id = media_url - media_info = await DBMedia.get_by_id(media_id) - if not media_info: - self.log.debug(f"Did not find existing mxc URL for {media_id}, uploading media now") - resp = await source.client.read_image(media_url) - media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=True) - await DBMedia(media_id=media_id, mxc=media_info.mxc).insert() - self.log.debug(f"Uploaded media as {media_info.mxc}") + db_media_info = await DBMedia.get_by_id(media_id) if deduplicate else None + if not db_media_info: + # NOTE Blob URL of stickers only persists for a single session...still better than nothing. + self.log.debug(f"{'Did not find existing mxc URL for' if deduplicate else 'Not deduplicating'} {media_id}, uploading media now") + try: + resp = await source.client.read_image(media_url) + except (RPCError, TypeError) as e: + self.log.warning(f"Failed to download remote media from chat {self.chat_id}: {e}") + return None + media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=deduplicate) + if deduplicate: + await DBMedia( + media_id=media_id, mxc=media_info.mxc, + size=media_info.size, mime_type=media_info.mime_type, file_name=media_info.file_name + ).insert() + return media_info else: - self.log.debug(f"Found existing mxc URL for {media_id}: {media_info.mxc}") - return media_info.mxc + self.log.debug(f"Found existing mxc URL for {media_id}: {db_media_info.mxc}") + return MediaInfo(db_media_info.mxc, None, db_media_info.mime_type, db_media_info.file_name, db_media_info.size) async def _reupload_remote_media(self, data: bytes, intent: IntentAPI, mime_type: str = None, file_name: str = None, - disable_encryption: bool = True) -> ReuploadedMediaInfo: + disable_encryption: bool = True) -> MediaInfo: if not mime_type: mime_type = magic.from_buffer(data, mime=True) upload_mime_type = mime_type @@ -372,10 +399,13 @@ class Portal(DBPortal, BasePortal): filename=upload_file_name) if decryption_info: + self.log.debug(f"Uploaded encrypted media as {mxc}") decryption_info.url = mxc mxc = None + else: + self.log.debug(f"Uploaded media as {mxc}") - return ReuploadedMediaInfo(mxc, decryption_info, mime_type, file_name, len(data)) + return MediaInfo(mxc, decryption_info, mime_type, file_name, len(data)) async def update_info(self, conv: ChatInfo, client: Optional[Client]) -> None: if self.is_direct: diff --git a/matrix_puppeteer_line/rpc/types.py b/matrix_puppeteer_line/rpc/types.py index 3ddd302..9fdcebf 100644 --- a/matrix_puppeteer_line/rpc/types.py +++ b/matrix_puppeteer_line/rpc/types.py @@ -51,6 +51,13 @@ class ChatInfo(ChatListInfo, SerializableAttrs['ChatInfo']): participants: List[Participant] +@dataclass +class MessageImage(SerializableAttrs['MessageImage']): + url: str + is_sticker: bool + is_animated: bool + + @dataclass class Message(SerializableAttrs['Message']): id: int @@ -59,7 +66,7 @@ class Message(SerializableAttrs['Message']): sender: Optional[Participant] timestamp: int = None html: Optional[str] = None - image_url: Optional[str] = None + image: Optional[MessageImage] = None receipt_count: Optional[int] = None diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index 3de2b02..2fcefdf 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -69,17 +69,19 @@ window.__mautrixExpiry = function (button) {} * @return {Promise} */ window.__mautrixReceiveMessageID = function(id) {} -/** - * @return {Promise} - */ -window.__mautrixShowParticipantsList = function() {} +/** + * typedef ChatTypeEnum + */ const ChatTypeEnum = Object.freeze({ DIRECT: 1, GROUP: 2, ROOM: 3, }) +const MSG_DECRYPTING = "ⓘ Decrypting..." +// TODO consts for common selectors + class MautrixController { constructor() { this.chatListObserver = null @@ -92,7 +94,6 @@ class MautrixController { this.pinAppearObserver = null this.ownID = null - this.ownMsgPromise = Promise.resolve(-1) this._promiseOwnMsgReset() } @@ -172,28 +173,45 @@ class MautrixController { * @typedef MessageData * @type {object} * @property {number} id - The ID of the message. Seems to be sequential. - * @property {number} timestamp - The unix timestamp of the message. Not very accurate. + * @property {number} timestamp - The unix timestamp of the message. Accurate to the minute. * @property {boolean} is_outgoing - Whether or not this user sent the message. * @property {?Participant} sender - Full data of the participant who sent the message, if needed and available. * @property {?string} html - The HTML format of the message, if necessary. - * @property {?string} image_url - The URL to the image in the message, if it's an image-only message. + * @property {?ImageInfo} image - Information of the image in the message, if it's an image-only message. * @property {?int} receipt_count - The number of users who have read the message. */ + /** + * @typedef ImageInfo + * @type {object} + * @property {string} url - The URL of the image's location. + * @property {boolean} is_sticker - Whether the sent image is a sticker. + * @property {boolean} animated - Whether the sent image is animated. Only used for stickers (for now...?). + */ + + /** + * Return whether a URL points to a loaded image or not. + * + * @param {string} src + * @return boolean + * @private + */ _isLoadedImageURL(src) { - return src && (src.startsWith("blob:") || src.startsWith("res/")) + return src && ( + src.startsWith(`blob:`) || + src.startsWith(`${document.location.origin}/res/`) && !src.startsWith(`${document.location.origin}/res/img/noimg/`)) } /** - * Parse a message element (mws-message-wrapper) + * Parse a message element. * - * @param {Date} date - The most recent date indicator. * @param {Element} element - The message element. - * @param {int} chatType - What kind of chat this message is part of. - * @return {MessageData} + * @param {Number} chatType - What kind of chat this message is part of. + * @param {Date} refDate - The most recent date indicator. If undefined, do not retrieve the timestamp of this message. + * @return {Promise} * @private */ - async _parseMessage(date, element, chatType) { + async _parseMessage(element, chatType, refDate) { const is_outgoing = element.classList.contains("mdRGT07Own") let sender = {} @@ -208,6 +226,7 @@ class MautrixController { sender = null receipt_count = is_outgoing ? (receipt ? 1 : 0) : null } else if (!is_outgoing) { + let imgElement sender.name = element.querySelector(".mdRGT07Body > .mdRGT07Ttl").innerText // Room members are always friends (right?), // so search the friend list for the sender's name @@ -216,23 +235,23 @@ class MautrixController { // Group members aren't necessarily friends, // but the participant list includes their ID. if (!sender.id) { - await window.__mautrixShowParticipantsList() const participantsList = document.querySelector(participantsListSelector) - sender.id = participantsList.querySelector(`img[alt='${senderName}'`).parentElement.parentElement.getAttribute("data-mid") + imgElement = participantsList.querySelector(`img[alt='${sender.name}'`) + sender.id = imgElement.parentElement.parentElement.getAttribute("data-mid") + } else { + imgElement = element.querySelector(".mdRGT07Img > img") } - sender.avatar = this.getParticipantListItemAvatar(element) + sender.avatar = this._getPathImage(imgElement) receipt_count = null } else { // TODO Get own ID and store it somewhere appropriate. // Unable to get own ID from a room chat... // if (chatType == ChatTypeEnum.GROUP) { - // await window.__mautrixShowParticipantsList() // const participantsList = document.querySelector("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // // TODO The first member is always yourself, right? // // TODO Cache this so own ID can be used later // sender = participantsList.children[0].getAttribute("data-mid") // } - await window.__mautrixShowParticipantsList() const participantsList = document.querySelector(participantsListSelector) sender.name = this.getParticipantListItemName(participantsList.children[0]) sender.avatar = this.getParticipantListItemAvatar(participantsList.children[0]) @@ -243,56 +262,175 @@ class MautrixController { const messageData = { id: +element.getAttribute("data-local-id"), - timestamp: date ? date.getTime() : null, + timestamp: + refDate !== undefined + ? (await this._tryParseDate(element.querySelector("time")?.innerText, refDate))?.getTime() + : null, is_outgoing: is_outgoing, sender: sender, - receipt_count: receipt_count + receipt_count: receipt_count, } - const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg") - if (messageElement.classList.contains("mdRGT07Text")) { - messageData.html = messageElement.querySelector(".mdRGT07MsgTextInner")?.innerHTML - } else if ( - messageElement.classList.contains("mdRGT07Image") || - messageElement.classList.contains("mdRGT07Sticker") - ) { - const img = messageElement.querySelector(".mdRGT07MsgImg > img") - if (img) { - let imgResolve - // TODO Should reject on "#_chat_message_image_failure" - let observer = new MutationObserver(changes => { - for (const change of changes) { - if (this._isLoadedImageURL(change.target.src) && observer) { - observer.disconnect() - observer = null - imgResolve(change.target.src) - return - } - } - }) - observer.observe(img, { attributes: true, attributeFilter: ["src"] }) - if (this._isLoadedImageURL(img.src)) { - // Check for this AFTER attaching the observer, in case - // the image loaded after the img element was found but - // before the observer was attached. - messageData.image_url = img.src - observer.disconnect() - } else { - messageData.image_url = await new Promise(resolve => { - imgResolve = resolve - setTimeout(() => { - if (observer) { - observer.disconnect() - resolve(img.src) - } - }, 10000) // Longer timeout for image downloads - }) + const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg") + const is_sticker = messageElement.classList.contains("mdRGT07Sticker") + if (messageElement.classList.contains("mdRGT07Text")) { + let msgSpan = messageElement.querySelector(".mdRGT07MsgTextInner") + try { + if (msgSpan.innerHTML == MSG_DECRYPTING) { + msgSpan = await this._waitForDecryptedMessage(element, msgSpan, 5000) } + messageData.html = await this._parseMessageHTML(msgSpan) + } catch { + // Throw to reject, but return what was parsed so far + throw messageData + } + } else if (is_sticker || messageElement.classList.contains("mdRGT07Image")) { + // TODO Animated non-sticker images require clicking its img element, which is just a thumbnail + // Real image: "#wrap_single_image img" + // Close button: "#wrap_single_image button" + // Viewer is open/closed based on "#wrap_single_image.MdNonDisp" / "#wrap_single_image:not(.MdNonDisp)" + let img = messageElement.querySelector(".mdRGT07MsgImg > img") + if (!this._isLoadedImageURL(img.src)) { + try { + img = await this._waitForLoadedImage(img, 10000) + } catch { + // Throw to reject, but return what was parsed so far + throw messageData + } + } + messageData.image = { + url: img.src, + is_sticker: is_sticker, + is_animated: is_sticker && img.parentElement.classList.contains("animationSticker"), } } return messageData } + /** + * @param {Element} msgSpan + * @return Promise + * @private + */ + async _parseMessageHTML(msgSpan) { + const msgSpanImgs = msgSpan.getElementsByTagName("img") + if (msgSpanImgs.length == 0) { + return msgSpan.innerHTML + } else { + const unloadedImgs = Array.from(msgSpanImgs).filter(img => !this._isLoadedImageURL(img.src)) + if (unloadedImgs.length > 0) { + // NOTE Use allSettled to not throw if any images time out + await Promise.allSettled( + unloadedImgs.map(img => this._waitForLoadedImage(img, 2000)) + ) + } + + // Hack to put sticon dimensions in HTML (which are excluded by default) + // in such a way that doesn't alter the elements that are in the DOM + const msgSpanCopy = msgSpan.cloneNode(true) + const msgSpanCopyImgs = msgSpanCopy.getElementsByTagName("img") + for (let i = 0, n = msgSpanImgs.length; i < n; i++) { + msgSpanCopyImgs[i].height = msgSpanImgs[i].height + msgSpanCopyImgs[i].width = msgSpanImgs[i].width + } + return msgSpanCopy.innerHTML + } + } + + /** + * @param {Element} element + * @param {Element} msgSpan + * @param {Number} timeoutLimitMillis + * @return {Promise} + * @private + */ + _waitForDecryptedMessage(element, msgSpan, timeoutLimitMillis) { + console.debug("Wait for message element to finish decrypting") + console.debug(element) + return new Promise((resolve, reject) => { + let observer = new MutationObserver(changes => { + for (const change of changes) { + const isTextUpdate = change.type == "characterData" + const target = isTextUpdate ? msgSpan : element.querySelector(".mdRGT07MsgTextInner") + if (target && target.innerHTML != MSG_DECRYPTING) { + if (isTextUpdate) { + console.debug("UNLIKELY(?) EVENT -- Found decrypted message from text update") + } else { + // TODO Looks like it's div.mdRGT07Body that gets always replaced. If so, watch only for that + console.debug("Found decrypted message from element replacement") + console.debug(target) + console.debug("Added:") + for (const change of changes) { + console.debug(change.removedNodes) + } + console.debug("Removed:") + for (const change of changes) { + console.debug(change.addedNodes) + } + } + observer.disconnect() + observer = null + resolve(target) + return + } + if (target && target != msgSpan) { + console.debug("UNLIKELY EVENT -- Somehow added a new \"decrypting\" span, it's the one to watch now") + console.debug(target) + msgSpan = target + observer.observe(msgSpan, { characterData: true }) + } + } + }) + // Either the span element or one of its ancestors is replaced, + // or the span element's content is updated. + // Not exactly sure which of these happens, or if the same kind + // of mutation always happens, so just look for them all... + observer.observe(element, { childList: true, subtree: true }) + observer.observe(msgSpan, { characterData: true }) + setTimeout(() => { + if (observer) { + observer.disconnect() + // Don't print log message, as this may be a safe timeout + reject() + } + }, timeoutLimitMillis) + }) + } + + /** + * @param {Element} img + * @param {Number} timeoutLimitMillis + * @return {Promise} + * @private + */ + _waitForLoadedImage(img, timeoutLimitMillis) { + console.debug("Wait for image element to finish loading") + console.debug(img) + // TODO Should reject on "#_chat_message_image_failure" + return new Promise((resolve, reject) => { + let observer = new MutationObserver(changes => { + for (const change of changes) { + if (this._isLoadedImageURL(change.target.src)) { + console.debug("Image element finished loading") + console.debug(change.target) + observer.disconnect() + observer = null + resolve(change.target) + return + } + } + }) + observer.observe(img, { attributes: true, attributeFilter: ["src"] }) + setTimeout(() => { + if (observer) { + observer.disconnect() + // Don't print log message, as this may be a safe timeout + reject() + } + }, timeoutLimitMillis) + }) + } + /** * Find the number in the "Read #" receipt message. * Don't look for "Read" specifically, to support multiple languages. @@ -313,8 +451,8 @@ class MautrixController { * has succeeded or failed to be sent. * * @param {int} timeoutLimitMillis - The maximum amount of time to wait for the message to be sent. - * @param {str} successSelector - The selector for the element that indicates the message was sent. - * @param {str} failureSelector - The selector for the element that indicates the message failed to be sent. + * @param {string} successSelector - The selector for the element that indicates the message was sent. + * @param {?string} failureSelector - The selector for the element that indicates the message failed to be sent. */ promiseOwnMessage(timeoutLimitMillis, successSelector, failureSelector=null) { this.promiseOwnMsgSuccessSelector = successSelector @@ -323,13 +461,13 @@ class MautrixController { this.ownMsgPromise = new Promise((resolve, reject) => { this.promiseOwnMsgResolve = resolve this.promiseOwnMsgReject = reject - setTimeout(() => { - if (this.promiseOwnMsgReject) { - console.log("Timeout!") - this._rejectOwnMessage() - } - }, timeoutLimitMillis) }) + this.promiseOwnMsgTimeoutID = setTimeout(() => { + if (this.promiseOwnMsgReject) { + console.error("Timed out waiting for own message to be sent") + this._rejectOwnMessage() + } + }, timeoutLimitMillis) } /** @@ -338,40 +476,39 @@ class MautrixController { * @return {Promise} - The ID of the sent message. */ async waitForOwnMessage() { - return await this.ownMsgPromise - } - - async _tryParseMessages(msgList, chatType) { - const messages = [] - let refDate = null - for (const child of msgList) { - if (child.classList.contains("mdRGT10Date")) { - refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText) - } else if (child.classList.contains("MdRGT07Cont")) { - // TODO :not(.MdNonDisp) to exclude not-yet-posted messages, - // but that is unlikely to be a problem here. - // Also, offscreen times may have .MdNonDisp on them - // TODO Explicitly look for the most recent date element, - // as it might not have been one of the new items in msgList - const timeElement = child.querySelector("time") - if (timeElement) { - const messageDate = await this._tryParseDate(timeElement.innerText, refDate) - messages.push(await this._parseMessage(messageDate, child, chatType)) - } - } - } - return messages + return this.ownMsgPromise ? await this.ownMsgPromise : -1 } /** * Parse the message list of whatever the currently-viewed chat is. * - * @return {[MessageData]} - A list of messages. + * @param {int} minID - The minimum message ID to consider. + * @return {Promise<[MessageData]>} - A list of messages. */ - async parseMessageList() { - const msgList = Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]")) - msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id")) - return await this._tryParseMessages(msgList, this.getChatType(this.getCurrentChatID())) + async parseMessageList(minID = 0) { + console.debug(`minID for full refresh: ${minID}`) + const msgList = + Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]")) + .filter(msg => + msg.hasAttribute("data-local-id") && + (!msg.classList.contains("MdRGT07Cont") || msg.getAttribute("data-local-id") > minID)) + if (msgList.length == 0) { + return [] + } + const messagePromises = [] + const chatType = this.getChatType(this.getCurrentChatID()) + let refDate = null + for (const child of msgList) { + if (child.classList.contains("mdRGT10Date")) { + refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText) + } else if (child.classList.contains("MdRGT07Cont")) { + messagePromises.push(this._parseMessage(child, chatType, refDate)) + } + } + // NOTE No message should ever time out, but use allSettled to not throw if any do + return (await Promise.allSettled(messagePromises)) + .filter(value => value.status == "fulfilled") + .map(value => value.value) } /** @@ -439,7 +576,7 @@ class MautrixController { const name = this.getParticipantListItemName(child) const id = this.getParticipantListItemID(child) || this.getUserIdFromFriendsList(name) return { - id: id, // NOTE Don't want non-own user's ID to ever be null. + id: id, avatar: this.getParticipantListItemAvatar(child), name: name, } @@ -497,6 +634,7 @@ class MautrixController { /** * Parse the list of recent/saved chats. + * * @return {[ChatListInfo]} - The list of chats. */ parseChatList() { @@ -505,20 +643,6 @@ class MautrixController { child => this.parseChatListItem(child.firstElementChild)) } - /** - * TODO - * Check if an image has been downloaded. - * - * @param {number} id - The ID of the message whose image to check. - * @return {boolean} - Whether or not the image has been downloaded - */ - imageExists(id) { - const imageElement = document.querySelector( - `mws-message-wrapper[msg-id="${id}"] mws-image-message-part .image-msg`) - return !imageElement.classList.contains("not-rendered") - && imageElement.getAttribute("src") !== "" - } - /** * Download an image at a given URL and return it as a data URL. * @@ -544,7 +668,6 @@ class MautrixController { // TODO Observe *added/removed* chats, not just new messages const changedChatIDs = new Set() for (const change of mutations) { - console.debug("Chat list mutation:", change) if (change.target.id == "_chat_list_body") { // TODO // These could be new chats, or they're @@ -555,16 +678,16 @@ class MautrixController { */ } else if (change.target.tagName == "LI") { if (change.target.classList.contains("ExSelected")) { - console.log("Not using chat list mutation response for currently-active chat") + console.debug("Not using chat list mutation response for currently-active chat") continue } for (const node of change.addedNodes) { const chat = this.parseChatListItem(node) if (chat) { - console.log("Changed chat list item:", chat) + console.log("Added chat list item:", chat) changedChatIDs.add(chat.id) } else { - console.debug("Could not parse node as a chat list item:", node) + console.debug("Could not parse added node as a chat list item:", node) } } } @@ -584,10 +707,12 @@ class MautrixController { addChatListObserver() { this.removeChatListObserver() this.chatListObserver = new MutationObserver(async (mutations) => { - // Wait for pending sent messages to be resolved before responding to mutations - try { - await this.ownMsgPromise - } catch (e) {} + if (this.ownMsgPromise) { + // Wait for pending sent messages to be resolved before responding to mutations + try { + await this.ownMsgPromise + } catch (e) {} + } try { this._observeChatListMutations(mutations) @@ -598,7 +723,7 @@ class MautrixController { this.chatListObserver.observe( document.querySelector("#_chat_list_body"), { childList: true, subtree: true }) - console.debug("Started chat list observer") + console.log("Started chat list observer") } /** @@ -608,13 +733,13 @@ class MautrixController { if (this.chatListObserver !== null) { this.chatListObserver.disconnect() this.chatListObserver = null - console.debug("Disconnected chat list observer") + console.log("Disconnected chat list observer") } } /** * @param {[MutationRecord]} mutations - The mutation records that occurred - * @param {str} chatID - The ID of the chat being observed. + * @param {string} chatID - The ID of the chat being observed. * @private */ _observeReceiptsDirect(mutations, chatID) { @@ -641,7 +766,7 @@ class MautrixController { /** * @param {[MutationRecord]} mutations - The mutation records that occurred - * @param {str} chatID - The ID of the chat being observed. + * @param {string} chatID - The ID of the chat being observed. * @private */ _observeReceiptsMulti(mutations, chatID) { @@ -649,17 +774,17 @@ class MautrixController { const receipts = [] for (const change of mutations) { const target = change.type == "characterData" ? change.target.parentElement : change.target - if ( change.target.classList.contains("mdRGT07Read") && - !change.target.classList.contains("MdNonDisp")) + if ( target.classList.contains("mdRGT07Read") && + !target.classList.contains("MdNonDisp")) { - const msgElement = change.target.closest(".mdRGT07Own") + const msgElement = target.closest(".mdRGT07Own") if (msgElement) { const id = +msgElement.getAttribute("data-local-id") if (!ids.has(id)) { ids.add(id) receipts.push({ id: id, - count: this._getReceiptCount(change.target), + count: this._getReceiptCount(target), }) } } @@ -673,11 +798,90 @@ class MautrixController { } } + /** + * @typedef PendingMessage + * @type object + * + * @property {Promise} promise + * @property {Number} id + */ + + /** + * @typedef SameIDMsgs + * @type object + * + * @property {Number} id + * @property {PendingMessage[]} msgs + * @property {Function} resolve + * @property {Number} numRejected + */ + + /** + * Binary search for the array of messages with the provided ID. + * + * @param {SameIDMsgs[]} sortedSameIDMsgs + * @param {Number} id + * @param {boolean} returnClosest - If true, return the index of the nearest result on miss instead of -1. + * @return {Number} The index of the matched element, or -1 if not found. + */ + _findMsgsForID( + sortedSameIDMsgs, id, returnClosest = false, + lowerBound = 0, upperBound = sortedSameIDMsgs.length - 1) + { + if (lowerBound > upperBound) { + return -1 + } + if (returnClosest && lowerBound == upperBound) { + // Caller must check if the result has a matching ID or not + return sortedSameIDMsgs[lowerBound].id <= id ? lowerBound : lowerBound-1 + } + const i = lowerBound + Math.floor((upperBound - lowerBound)/2) + const val = sortedSameIDMsgs[i] + if (val.id == id) { + return i + } else if (val.id < id) { + return this._findMsgsForID( + sortedSameIDMsgs, id, returnClosest, + i+1, upperBound) + } else { + return this._findMsgsForID( + sortedSameIDMsgs, id, returnClosest, + lowerBound, i-1) + } + } + + /** + * Insert the given message to the proper inner array. + * In no inner array exists, insert a new one, preserving sort order. + * Return the wrapper of which inner array was added to or created. + * + * @param {SameIDMsgs[]} sortedSameIDMsgs + * @param {PendingMessage} msg + * @return {SameIDMsgs} + */ + _insertMsgByID(sortedSameIDMsgs, msg) { + let i = this._findMsgsForID(sortedSameIDMsgs, msg.id, true) + if (i != -1 && sortedSameIDMsgs[i].id == msg.id) { + sortedSameIDMsgs[i].msgs.push(msg) + console.debug("UNLIKELY(?) EVENT -- Found two new message elements with the same ID, so tracking both of them") + } else { + sortedSameIDMsgs.splice(++i, 0, { + id: msg.id, + msgs: [msg], + numRejected: 0, + resolve: null, + }) + } + return sortedSameIDMsgs[i] + } + /** * Add a mutation observer to the message list of the current chat. * Used for observing new messages & read receipts. + * + * @param {int} minID - The minimum message ID to consider. */ - addMsgListObserver() { + addMsgListObserver(minID = 0) { const chat_room_msg_list = document.querySelector("#_chat_room_msg_list") if (!chat_room_msg_list) { console.debug("Could not start msg list observer: no msg list available!") @@ -688,34 +892,133 @@ class MautrixController { const chatID = this.getCurrentChatID() const chatType = this.getChatType(chatID) - let orderedPromises = [Promise.resolve()] + // NEED TO HANDLE: + // * message elements arriving in any order + // * messages being potentially pending (i.e. decrypting or loading), + // and resolving in a potentially different order than they arrived in + // * pending messages potentially having multiple elements associated with + // them, where only one of them resolves + // * message elements being added/removed any number of times, which may + // or may not ever resolve + // * outgoing messages (i.e. sent by the bridge) + // And must send resolved messages to the bridge *in order*! + // BUT: Assuming that incoming messages will never be younger than a resolved one. + + const sortedSameIDMsgs = [] + const pendingMsgElements = new Set() + this.msgListObserver = new MutationObserver(changes => { - let msgList = [] + console.debug(`MESSAGE LIST CHANGES: check since ${minID}`) + const remoteMsgs = [] for (const change of changes) { - change.addedNodes.forEach(child => { - if (child.tagName == "DIV" && child.hasAttribute("data-local-id")) { - msgList.push(child) + console.debug("---new change set---") + for (const child of change.addedNodes) { + if (!pendingMsgElements.has(child) && + child.tagName == "DIV" && + child.hasAttribute("data-local-id") && + // Skip timestamps, as these are always current + child.classList.contains("MdRGT07Cont")) + { + const msgID = child.getAttribute("data-local-id") + if (msgID > minID) { + pendingMsgElements.add(child) + + // TODO Maybe handle own messages somewhere else...? + const ownMsg = this._observeOwnMessage(child) + if (ownMsg) { + console.log("Found own bridge-sent message, will wait for it to resolve") + console.debug(child) + this.ownMsgPromise + .then(msgID => { + console.log("Resolved own bridge-sent message") + console.debug(ownMsg) + pendingMsgElements.delete(ownMsg) + if (minID < msgID) { + minID = msgID + } + }) + .catch(() => { + console.log("Rejected own bridge-sent message") + console.debug(ownMsg) + pendingMsgElements.delete(ownMsg) + }) + } else { + console.log("Found remote message") + console.debug(child) + remoteMsgs.push({ + id: msgID, + element: child + }) + } + } } - }) + } + // NOTE Ignoring removedNodes because an element can always be added back. + // Will simply let permanently-removed nodes time out. } - if (msgList.length == 0) { + if (remoteMsgs.length == 0) { + console.debug("Found no new remote messages") return } - msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id")) - if (!this._observeOwnMessage(msgList)) { - let prevPromise = orderedPromises.shift() - orderedPromises.push(new Promise(resolve => prevPromise - .then(() => this._tryParseMessages(msgList, chatType)) - .then(msgs => window.__mautrixReceiveMessages(chatID, msgs)) - .then(() => resolve()) - )) + + // No need to sort remoteMsgs, because sortedSameIDMsgs is enough + for (const msg of remoteMsgs) { + const messageElement = msg.element + const pendingMessage = { + id: msg.id, + promise: this._parseMessage(messageElement, chatType) + } + const sameIDMsgs = this._insertMsgByID(sortedSameIDMsgs, pendingMessage) + + const handleMessage = async (messageData) => { + minID = messageData.id + sortedSameIDMsgs.shift() + await window.__mautrixReceiveMessages(chatID, [messageData]) + if (sortedSameIDMsgs.length > 0 && sortedSameIDMsgs[0].resolve) { + console.debug("Allowing queued resolved message to be sent") + console.debug(sortedSameIDMsgs[0]) + sortedSameIDMsgs[0].resolve() + } + } + + pendingMessage.promise.then( + async (messageData) => { + const i = this._findMsgsForID(sortedSameIDMsgs, messageData.id) + if (i == -1) { + console.debug(`Got resolved message for already-handled ID ${messageData.id}, ignore it`) + pendingMsgElements.delete(messageElement) + return + } + if (i != 0) { + console.debug(`Got resolved message for later ID ${messageData.id}, wait for earlier messages`) + await new Promise(resolve => sameIDMsgs.resolve = resolve) + console.debug(`Message before ID ${messageData.id} finished, can now send this one`) + } else { + console.debug(`Got resolved message for earliest ID ${messageData.id}, send it`) + } + console.debug(messageElement) + pendingMsgElements.delete(messageElement) + handleMessage(messageData) + }, + // error case + async (messageData) => { + console.debug("Message element rejected") + console.debug(messageElement) + pendingMsgElements.delete(messageElement) + if (++sameIDMsgs.numRejected == sameIDMsgs.msgs.length) { + // Note that if another message element with this ID somehow comes later, it'll be ignored. + console.debug(`All messages for ID ${sameIDMsgs.id} rejected, abandoning this ID and sending dummy message`) + // Choice of which message to send should be arbitrary + handleMessage(messageData) + } + }) } }) this.msgListObserver.observe( chat_room_msg_list, { childList: true }) - console.debug("Started msg list observer") + console.debug(`Started msg list observer with minID = ${minID}`) const observeReadReceipts = ( @@ -736,74 +1039,65 @@ class MautrixController { subtree: true, attributes: true, attributeFilter: ["class"], - // TODO Consider using the same observer to watch for "ⓘ Decrypting..." characterData: chatType != ChatTypeEnum.DIRECT, }) console.debug("Started receipt observer") } - _observeOwnMessage(msgList) { - if (!this.promiseOwnMsgSuccessSelector) { + _observeOwnMessage(ownMsg) { + if (!this.ownMsgPromise) { // Not waiting for a pending sent message - return false - } - if (this.visibleSuccessObserver) { - // Already found a element that we're waiting on becoming visible - return true + return null } - for (const ownMsg of msgList.filter(msg => msg.classList.contains("mdRGT07Own"))) { - const successElement = - ownMsg.querySelector(this.promiseOwnMsgSuccessSelector) - if (successElement) { - if (successElement.classList.contains("MdNonDisp")) { - console.log("Invisible success") - console.log(successElement) - } else { - console.debug("Already visible success, must not be it") - console.debug(successElement) - continue - } + const successElement = + ownMsg.querySelector(this.promiseOwnMsgSuccessSelector) + if (successElement) { + if (successElement.classList.contains("MdNonDisp")) { + console.log("Invisible success for own bridge-sent message, will wait for it to resolve") + console.log(successElement) } else { - continue + console.debug("Already visible success, must not be it") + console.debug(successElement) + return null } - - const failureElement = - this.promiseOwnMsgFailureSelector && - ownMsg.querySelector(this.promiseOwnMsgFailureSelector) - if (failureElement) { - if (failureElement.classList.contains("MdNonDisp")) { - console.log("Invisible failure") - console.log(failureElement) - } else { - console.debug("Already visible failure, must not be it") - console.log(failureElement) - continue - } - } else if (this.promiseOwnMsgFailureSelector) { - continue - } - - console.log("Found invisible element, wait") - const msgID = +ownMsg.getAttribute("data-local-id") - this.visibleSuccessObserver = new MutationObserver( - this._getOwnVisibleCallback(msgID)) - this.visibleSuccessObserver.observe( - successElement, - { attributes: true, attributeFilter: ["class"] }) - - if (this.promiseOwnMsgFailureSelector) { - this.visibleFailureObserver = new MutationObserver( - this._getOwnVisibleCallback()) - this.visibleFailureObserver.observe( - failureElement, - { attributes: true, attributeFilter: ["class"] }) - } - - return true + } else { + return null } - return false + + const failureElement = + this.promiseOwnMsgFailureSelector && + ownMsg.querySelector(this.promiseOwnMsgFailureSelector) + if (failureElement) { + if (failureElement.classList.contains("MdNonDisp")) { + console.log("Invisible failure for own bridge-sent message, will wait for it (or success) to resolve") + console.log(failureElement) + } else { + console.debug("Already visible failure, must not be it") + console.log(failureElement) + return null + } + } else if (this.promiseOwnMsgFailureSelector) { + return null + } + + const msgID = +ownMsg.getAttribute("data-local-id") + this.visibleSuccessObserver = new MutationObserver( + this._getOwnVisibleCallback(msgID)) + this.visibleSuccessObserver.observe( + successElement, + { attributes: true, attributeFilter: ["class"] }) + + if (this.promiseOwnMsgFailureSelector) { + this.visibleFailureObserver = new MutationObserver( + this._getOwnVisibleCallback()) + this.visibleFailureObserver.observe( + failureElement, + { attributes: true, attributeFilter: ["class"] }) + } + + return ownMsg } _getOwnVisibleCallback(msgID=null) { @@ -811,7 +1105,7 @@ class MautrixController { return changes => { for (const change of changes) { if (!change.target.classList.contains("MdNonDisp")) { - console.log(`Waited for visible ${isSuccess ? "success" : "failure"}`) + console.log(`Resolved ${isSuccess ? "success" : "failure"} for own bridge-sent message`) console.log(change.target) isSuccess ? this._resolveOwnMessage(msgID) : this._rejectOwnMessage(change.target) return @@ -822,6 +1116,7 @@ class MautrixController { _resolveOwnMessage(msgID) { if (!this.promiseOwnMsgResolve) return + clearTimeout(this.promiseOwnMsgTimeoutID) const resolve = this.promiseOwnMsgResolve this._promiseOwnMsgReset() @@ -838,10 +1133,12 @@ class MautrixController { } _promiseOwnMsgReset() { + this.ownMsgPromise = null this.promiseOwnMsgSuccessSelector = null this.promiseOwnMsgFailureSelector = null this.promiseOwnMsgResolve = null this.promiseOwnMsgReject = null + this.promiseOwnMsgTimeoutID = null if (this.visibleSuccessObserver) { this.visibleSuccessObserver.disconnect() diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 0f07af9..0238b70 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -107,7 +107,6 @@ export default class MessagesPuppeteer { this._receiveReceiptDirectLatest.bind(this)) await this.page.exposeFunction("__mautrixReceiveReceiptMulti", this._receiveReceiptMulti.bind(this)) - await this.page.exposeFunction("__mautrixShowParticipantsList", this._showParticipantList.bind(this)) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) // NOTE Must *always* re-login on a browser session, so no need to check if already logged in @@ -376,15 +375,7 @@ export default class MessagesPuppeteer { * @return {Promise<[MessageData]>} - The messages visible in the chat. */ async getMessages(chatID) { - return await this.taskQueue.push(async () => { - const messages = await this._getMessagesUnsafe(chatID) - if (messages.length > 0) { - for (const message of messages) { - message.chat_id = chatID - } - } - return messages - }) + return await this.taskQueue.push(async () => this._getMessagesUnsafe(chatID)) } setLastMessageIDs(ids) { @@ -392,7 +383,8 @@ export default class MessagesPuppeteer { for (const [chatID, messageID] of Object.entries(ids)) { this.mostRecentMessages.set(chatID, messageID) } - this.log("Updated most recent message ID map:", this.mostRecentMessages) + this.log("Updated most recent message ID map:") + this.log(this.mostRecentMessages) } async readImage(imageUrl) { @@ -407,11 +399,15 @@ export default class MessagesPuppeteer { } async startObserving() { - this.log("Adding observers") + const chatID = await this.page.evaluate(() => window.__mautrixController.getCurrentChatID()) + this.log(`Adding observers for ${chatID || "empty chat"}`) await this.page.evaluate( () => window.__mautrixController.addChatListObserver()) - await this.page.evaluate( - () => window.__mautrixController.addMsgListObserver()) + if (chatID) { + await this.page.evaluate( + (mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + } } async stopObserving() { @@ -444,9 +440,10 @@ export default class MessagesPuppeteer { if (await this.page.evaluate(isCorrectChatVisible, chatName)) { this.log("Already viewing chat, no need to switch") } else { - this.log("Switching chat, so remove msg list observer") + this.log("Ensuring msg list observer is removed") const hadMsgListObserver = await this.page.evaluate( () => window.__mautrixController.removeMsgListObserver()) + this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer") await chatListItem.click() this.log(`Waiting for chat header title to be "${chatName}"`) @@ -455,39 +452,28 @@ export default class MessagesPuppeteer { {polling: "mutation"}, chatName) - // For consistent behaviour later, wait for the chat details sidebar to be hidden + // Always show the chat details sidebar, as this makes life easier + this.log("Waiting for detail area to be auto-hidden upon entering chat") await this.page.waitForFunction( detailArea => detailArea.childElementCount == 0, {}, await this.page.$("#_chat_detail_area")) + this.log("Clicking chat header to show detail area") + await this.page.click("#_chat_header_area > .mdRGT04Link") + this.log("Waiting for detail area") + await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info") if (hadMsgListObserver) { this.log("Restoring msg list observer") await this.page.evaluate( - () => window.__mautrixController.addMsgListObserver()) + (mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) } else { this.log("Not restoring msg list observer, as there never was one") } } } - // TODO Commonize - async _getParticipantList() { - await this._showParticipantList() - return await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") - } - - async _showParticipantList() { - const selector = "#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul" - let participantList = await this.page.$(selector) - if (!participantList) { - this.log("Participant list hidden, so clicking chat header to show it") - await this.page.click("#_chat_header_area > .mdRGT04Link") - // Use no timeout since the browser itself is using this - await this.page.waitForSelector(selector, {timeout: 0}) - } - } - async _getChatInfoUnsafe(chatID) { const chatListItem = await this.page.$(this._listItemSelector(chatID)) const chatListInfo = await chatListItem.evaluate( @@ -512,7 +498,7 @@ export default class MessagesPuppeteer { this.log("Found multi-user chat, so clicking chat header to get participants") // TODO This will mark the chat as "read"! await this._switchChat(chatID) - const participantList = await this._getParticipantList() + const participantList = await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // TODO Is a group not actually created until a message is sent(?) // If so, maybe don't create a portal until there is a message. participants = await participantList.evaluate( @@ -542,6 +528,7 @@ export default class MessagesPuppeteer { async _sendMessageUnsafe(chatID, text) { await this._switchChat(chatID) + // TODO Initiate the promise in the content script await this.page.evaluate( () => window.__mautrixController.promiseOwnMessage(5000, "time")) @@ -593,15 +580,12 @@ export default class MessagesPuppeteer { } } - _receiveMessages(chatID, messages) { + async _receiveMessages(chatID, messages) { if (this.client) { - messages = this._filterMessages(chatID, messages) - if (messages.length > 0) { - for (const message of messages) { - message.chat_id = chatID - this.client.sendMessage(message).catch(err => - this.error("Failed to send message", message.id, "to client:", err)) - } + messages = await this._processMessages(chatID, messages) + for (const message of messages) { + this.client.sendMessage(message).catch(err => + this.error("Failed to send message", message.id, "to client:", err)) } } else { this.log("No client connected, not sending messages") @@ -609,27 +593,52 @@ export default class MessagesPuppeteer { } async _getMessagesUnsafe(chatID) { - // TODO Also handle "decrypting" state + // TODO Consider making a wrapper for pausing/resuming the msg list observers + this.log("Ensuring msg list observer is removed") + const hadMsgListObserver = await this.page.evaluate( + () => window.__mautrixController.removeMsgListObserver()) + this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer") + // TODO Handle unloaded messages. Maybe scroll up // TODO This will mark the chat as "read"! await this._switchChat(chatID) - const messages = await this.page.evaluate(() => - window.__mautrixController.parseMessageList()) - return this._filterMessages(chatID, messages) + const messages = await this.page.evaluate( + mostRecentMessage => window.__mautrixController.parseMessageList(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + // Doing this before restoring the observer since it updates minID + const filteredMessages = await this._processMessages(chatID, messages) + + if (hadMsgListObserver) { + this.log("Restoring msg list observer") + await this.page.evaluate( + mostRecentMessage => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + } else { + this.log("Not restoring msg list observer, as there never was one") + } + + return filteredMessages } - _filterMessages(chatID, messages) { + async _processMessages(chatID, messages) { + // TODO Probably don't need minID filtering if Puppeteer context handles it now const minID = this.mostRecentMessages.get(chatID) || 0 - const filtered_messages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id)) + const filteredMessages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id)) - if (filtered_messages.length > 0) { - const newFirstID = filtered_messages[0].id - const newLastID = filtered_messages[filtered_messages.length - 1].id + if (filteredMessages.length > 0) { + const newFirstID = filteredMessages[0].id + const newLastID = filteredMessages[filteredMessages.length - 1].id this.mostRecentMessages.set(chatID, newLastID) const range = newFirstID === newLastID ? newFirstID : `${newFirstID}-${newLastID}` - this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filtered_messages.length} newer than ${minID} (${range})`) + this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filteredMessages.length} newer than ${minID} (${range})`) + + for (const message of filteredMessages) { + message.chat_id = chatID + } + return filteredMessages + } else { + return [] } - return filtered_messages } async _processChatListChangeUnsafe(chatID) { @@ -643,7 +652,6 @@ export default class MessagesPuppeteer { if (this.client) { for (const message of messages) { - message.chat_id = chatID await this.client.sendMessage(message).catch(err => this.error("Failed to send message", message.id, "to client:", err)) }