From 2fb9be2037ce89098fea53ff779054b71138fda6 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 4 Jun 2021 02:19:46 -0400 Subject: [PATCH 01/22] Small doc update --- ROADMAP.md | 2 +- SETUP.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index ac41525..1dfe532 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -52,7 +52,7 @@ * [x] Rooms (unnamed chats / "multi-user direct chats") * [ ] Membership actions * [x] Add member - * [x] Remove member + * [ ] Remove member * [ ] Block * Misc * [x] Automatic portal creation diff --git a/SETUP.md b/SETUP.md index 940c8ab..584d73f 100644 --- a/SETUP.md +++ b/SETUP.md @@ -38,3 +38,5 @@ An easy way to do so is to install `xvfb` from your distribution, and run the Pu # Upgrading Simply `git pull` or `git rebase` the latest changes, and rerun any installation commands (`yarn --production`, `pip install -Ur ...`). + +To upgrade the LINE extension used by Puppeteer, simply download and extract the latest .crx in the same location as for initial setup. From 94788a21dd68e18ab974a118c308971b8333b048 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sat, 5 Jun 2021 23:41:05 -0400 Subject: [PATCH 02/22] Various fixes and cleanups --- matrix_puppeteer_line/portal.py | 12 +++--------- matrix_puppeteer_line/user.py | 9 +++++---- puppet/src/contentscript.js | 10 +++++----- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 6bd2871..276aa5f 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -29,14 +29,14 @@ from mautrix.types import (EventID, MessageEventContent, RoomID, EventType, Mess TextMessageEventContent, MediaMessageEventContent, Membership, Format, ContentURI, EncryptedFile, ImageInfo, RelatesTo, RelationType) -from mautrix.errors import IntentError, MatrixError +from mautrix.errors import IntentError from mautrix.util.simple_lock import SimpleLock from .db import Portal as DBPortal, Message as DBMessage, ReceiptReaction as DBReceiptReaction, Media as DBMedia from .config import Config from .rpc import ChatInfo, Participant, Message, Receipt, Client, PathImage from .rpc.types import RPCError -from . import user as u, puppet as p, matrix as m +from . import user as u, puppet as p if TYPE_CHECKING: from .__main__ import MessagesBridge @@ -170,7 +170,7 @@ class Portal(DBPortal, BasePortal): self.log.debug(f"Handled Matrix message {event_id} -> {message_id}") except UniqueViolationError as e: self.log.warning(f"Failed to handle Matrix message {event_id} -> {message_id}: {e}") - if not msg: + if not msg and self.config["bridge.delivery_error_reports"]: await self.main_intent.send_notice( self.mxid, "Posting this message to LINE may have failed.", @@ -179,12 +179,6 @@ class Portal(DBPortal, BasePortal): async def handle_matrix_leave(self, user: 'u.User') -> None: self.log.info(f"{user.mxid} left portal to {self.chat_id}, " f"cleaning up and deleting...") - if self.invite_own_puppet_to_pm: - # TODO Use own puppet instead of bridge bot. Then cleanup_and_delete will handle it - try: - await self.az.intent.leave_room(self.mxid) - except MatrixError: - pass await self.cleanup_and_delete() async def _bridge_own_message_pm(self, source: 'u.User', sender: Optional['p.Puppet'], mid: str, diff --git a/matrix_puppeteer_line/user.py b/matrix_puppeteer_line/user.py index b2c9923..2d4ea36 100644 --- a/matrix_puppeteer_line/user.py +++ b/matrix_puppeteer_line/user.py @@ -64,8 +64,9 @@ class User(DBUser, BaseUser): cls.loop = bridge.loop Client.config = bridge.config - async def send_notice(self, text) -> None: + async def send_bridge_notice(self, text) -> None: if self.notice_room: + self.log.debug(f"Sending bridge notice: {text}") await self.az.intent.send_notice(self.notice_room, text) async def is_logged_in(self) -> bool: @@ -96,17 +97,17 @@ class User(DBUser, BaseUser): self.loop.create_task(self.connect_double_puppet()) self.client = Client(self.mxid) self.log.debug("Starting client") - await self.send_notice("Starting up...") + await self.send_bridge_notice("Starting up...") state = await self.client.start() await self.client.on_message(self.handle_message) await self.client.on_receipt(self.handle_receipt) if state.is_connected: self._track_metric(METRIC_CONNECTED, True) if state.is_logged_in: - await self.send_notice("Already logged in to LINE") + await self.send_bridge_notice("Already logged in to LINE") self.loop.create_task(self._try_sync()) else: - await self.send_notice("Ready to log in to LINE") + await self.send_bridge_notice("Ready to log in to LINE") async def _try_sync(self) -> None: try: diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index e47c887..dbee656 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -34,13 +34,13 @@ window.__mautrixReceiveChanges = function (changes) {} */ window.__mautrixReceiveMessages = function (chatID, messages) {} /** - * @param {str} chatID - The ID of the chat whose receipts are being processed. - * @param {str} receipt_id - The ID of the most recently-read message for the current chat. + * @param {string} chatID - The ID of the chat whose receipts are being processed. + * @param {string} receipt_id - The ID of the most recently-read message for the current chat. * @return {Promise} */ -window.__mautrixReceiveReceiptDirectLatest = function (chat_id, receipt_id) {} +window.__mautrixReceiveReceiptDirectLatest = function (chatID, receipt_id) {} /** - * @param {str} chatID - The ID of the chat whose receipts are being processed. + * @param {string} chatID - The ID of the chat whose receipts are being processed. * @param {[Receipt]} receipts - All newly-seen receipts for the current chat. * @return {Promise} */ @@ -162,7 +162,7 @@ class MautrixController { * Try to match a user against an entry in the friends list to get their ID. * * @param {Element} element - The display name of the user to find the ID for. - * @return {?str} - The user's ID if found. + * @return {?string} - The user's ID if found. */ getUserIdFromFriendsList(senderName) { return document.querySelector(`#contact_wrap_friends > ul > li[title='${senderName}']`)?.getAttribute("data-mid") From 65838153012525c23c08840fc5df8df58936308e Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sat, 5 Jun 2021 23:51:12 -0400 Subject: [PATCH 03/22] Dismiss error dialog for getting disconnected If the connection to LINE is lost, when it comes back, and error dialog appears. Detect that dialog and click it automatically. The same detection works for any error dialog. --- puppet/src/contentscript.js | 40 ++++++++++++++++++------------------- puppet/src/puppet.js | 10 ---------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index dbee656..faf4a73 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -90,7 +90,6 @@ class MautrixController { this.qrAppearObserver = null this.emailAppearObserver = null this.pinAppearObserver = null - this.expiryObserver = null this.ownID = null this.ownMsgPromise = Promise.resolve(-1) @@ -968,26 +967,25 @@ class MautrixController { } } - addExpiryObserver(element) { - this.removeExpiryObserver() - const button = element.querySelector("dialog button") - this.expiryObserver = new MutationObserver(changes => { - if (changes.length == 1 && !changes[0].target.classList.contains("MdNonDisp")) { - window.__mautrixExpiry(button) - } - }) - this.expiryObserver.observe(element, { - attributes: true, - attributeFilter: ["class"], - }) - } - - removeExpiryObserver() { - if (this.expiryObserver !== null) { - this.expiryObserver.disconnect() - this.expiryObserver = null - } - } } window.__mautrixController = new MautrixController() + +/** + * Watch for an error dialog / PIN expiry dialog to appear, and click its "OK" button. + * Must watch for both its parent appearing & it being added to its parent in the first place. + */ +const layer = document.querySelector("#layer_contents") +new MutationObserver(() => { + if (!layer.classList.contains("MdNonDisp")) { + const button = layer.querySelector("dialog button") + if (button) { + console.log("Something expired, clicking OK button to continue") + button.click() + } + } +}).observe(layer, { + attributes: true, + attributeFilter: ["class"], + childList: true, +}) diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index aa8baeb..0f07af9 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -97,7 +97,6 @@ export default class MessagesPuppeteer { await this.page.exposeFunction("__mautrixReceiveQR", this._receiveQRChange.bind(this)) await this.page.exposeFunction("__mautrixSendEmailCredentials", this._sendEmailCredentials.bind(this)) await this.page.exposeFunction("__mautrixReceivePIN", this._receivePIN.bind(this)) - await this.page.exposeFunction("__mautrixExpiry", this._receiveExpiry.bind(this)) await this.page.exposeFunction("__mautrixReceiveMessageID", id => this.sentMessageIDs.add(id)) await this.page.exposeFunction("__mautrixReceiveChanges", @@ -183,8 +182,6 @@ export default class MessagesPuppeteer { await this.page.evaluate( element => window.__mautrixController.addPINAppearObserver(element), loginContentArea) - await this.page.$eval("#layer_contents", - element => window.__mautrixController.addExpiryObserver(element)) this.log("Waiting for login response") let doneWaiting = false @@ -226,7 +223,6 @@ export default class MessagesPuppeteer { await this.page.evaluate(() => window.__mautrixController.removeQRAppearObserver()) await this.page.evaluate(() => window.__mautrixController.removeEmailAppearObserver()) await this.page.evaluate(() => window.__mautrixController.removePINAppearObserver()) - await this.page.evaluate(() => window.__mautrixController.removeExpiryObserver()) delete this.login_email delete this.login_password @@ -739,10 +735,4 @@ export default class MessagesPuppeteer { this.log("No client connected, not sending failure reason") } } - - async _receiveExpiry(button) { - this.log("Something expired, clicking OK button to continue") - this.page.click(button).catch(err => - this.error("Failed to dismiss expiry dialog:", err)) - } } From 712a256deebba89878dd27099e4e401ae202f4dc Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sat, 5 Jun 2021 23:56:54 -0400 Subject: [PATCH 04/22] Fix incorrect time parsing For some reason, string-interpolating the result of chrono.parseDate can set the time of day-only dates to noon, instead of midnight, which is much more useful as a baseline time. To get midnight, prepend "00:00" to all day-only date strings before parsing them with chrono.parseDate. --- puppet/src/contentscript.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index faf4a73..3de2b02 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -136,17 +136,18 @@ class MautrixController { } /** - * Parse a date separator (mws-relative-timestamp) + * Parse a date separator. * - * @param {string} text - The text in the mws-relative-timestamp element. - * @return {?Date} - The value in the date separator. + * @param {string} text - The text in the date saparator. + * @return {Promise} - The value of the date separator. * @private */ - async _tryParseDayDate(text) { + async _tryParseDateSeparator(text) { if (!text) { return null } - text = text.replace(/\. /, "/") + // Must prefix with midnight to prevent getting noon + text = "00:00 " + text.replace(/\. /, "/") const now = new Date() let newDate = await this._tryParseDate(text) if (!newDate || newDate > now) { @@ -345,7 +346,7 @@ class MautrixController { let refDate = null for (const child of msgList) { if (child.classList.contains("mdRGT10Date")) { - refDate = await this._tryParseDayDate(child.firstElementChild.innerText) + refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText) } else if (child.classList.contains("MdRGT07Cont")) { // TODO :not(.MdNonDisp) to exclude not-yet-posted messages, // but that is unlikely to be a problem here. From 6d646e082b46f6dc7f7b11eff456af87a95c75ca Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:15:38 -0400 Subject: [PATCH 05/22] Inbound sync & sticker improvements - Handle "decrypting" state of messages - Handle lazy loading of emoji - Better handle lazy loading of images/stickers - Improve reliability of message sending, especially when sending several messages quickly - Use m.sticker events for inbound stickers instead of m.image, and add a config to optionally use m.image if desired - Use proper sizing for emoji, and add config to scale them since they are somewhat small - Deduplicate stickers as best as possible (works until they get a different blob URL) - Add config to disable bridging stickers/emoji - Send m.notice for inbound messages of unknown type --- matrix_puppeteer_line/config.py | 3 + matrix_puppeteer_line/db/media.py | 14 +- matrix_puppeteer_line/db/upgrade.py | 11 +- matrix_puppeteer_line/example-config.yaml | 6 + matrix_puppeteer_line/portal.py | 132 ++-- matrix_puppeteer_line/rpc/types.py | 9 +- puppet/src/contentscript.js | 695 +++++++++++++++------- puppet/src/puppet.js | 118 ++-- 8 files changed, 675 insertions(+), 313 deletions(-) diff --git a/matrix_puppeteer_line/config.py b/matrix_puppeteer_line/config.py index 7ab0b7d..b791561 100644 --- a/matrix_puppeteer_line/config.py +++ b/matrix_puppeteer_line/config.py @@ -73,6 +73,9 @@ class Config(BaseBridgeConfig): copy("bridge.delivery_receipts") copy("bridge.delivery_error_reports") copy("bridge.resend_bridge_info") + copy("bridge.receive_stickers") + copy("bridge.use_sticker_events") + copy("bridge.emoji_scale_factor") copy("bridge.command_prefix") copy("bridge.user") diff --git a/matrix_puppeteer_line/db/media.py b/matrix_puppeteer_line/db/media.py index 24c71b3..cab5d90 100644 --- a/matrix_puppeteer_line/db/media.py +++ b/matrix_puppeteer_line/db/media.py @@ -29,12 +29,14 @@ class Media: media_id: str mxc: ContentURI - # TODO Consider whether mime_type, file_name, and size are needed. + mime_type: str + file_name: str + size: int async def insert(self) -> None: - q = ("INSERT INTO media (media_id, mxc) " - "VALUES ($1, $2)") - await self.db.execute(q, self.media_id, self.mxc) + q = ("INSERT INTO media (media_id, mxc, mime_type, file_name, size) " + "VALUES ($1, $2, $3, $4, $5)") + await self.db.execute(q, self.media_id, self.mxc, self.mime_type, self.file_name, self.size) async def update(self) -> None: q = ("UPDATE media SET mxc=$2 " @@ -42,8 +44,8 @@ class Media: await self.db.execute(q, self.media_id, self.mxc) @classmethod - async def get_by_id(cls, media_id: str) -> Optional[ContentURI]: - q = ("SELECT media_id, mxc " + async def get_by_id(cls, media_id: str) -> Optional['DBMedia']: + q = ("SELECT media_id, mxc, mime_type, file_name, size " "FROM media WHERE media_id=$1") row = await cls.db.fetchrow(q, media_id) if not row: diff --git a/matrix_puppeteer_line/db/upgrade.py b/matrix_puppeteer_line/db/upgrade.py index 9a3b5da..d863009 100644 --- a/matrix_puppeteer_line/db/upgrade.py +++ b/matrix_puppeteer_line/db/upgrade.py @@ -103,4 +103,13 @@ async def upgrade_read_receipts(conn: Connection) -> None: FOREIGN KEY (mx_room) REFERENCES portal (mxid) ON DELETE CASCADE - )""") \ No newline at end of file + )""") + + +@upgrade_table.register(description="Media metadata") +async def upgrade_deduplicate_blob(conn: Connection) -> None: + await conn.execute("""ALTER TABLE media + ADD COLUMN IF NOT EXISTS mime_type TEXT, + ADD COLUMN IF NOT EXISTS file_name TEXT, + ADD COLUMN IF NOT EXISTS size INTEGER + """) \ No newline at end of file diff --git a/matrix_puppeteer_line/example-config.yaml b/matrix_puppeteer_line/example-config.yaml index 3c922e2..b519223 100644 --- a/matrix_puppeteer_line/example-config.yaml +++ b/matrix_puppeteer_line/example-config.yaml @@ -131,6 +131,12 @@ bridge: # This field will automatically be changed back to false after it, # except if the config file is not writable. resend_bridge_info: false + # Set this to false to disable bridging stickers and emoji. + receive_stickers: true + # Set this to false to use m.image events for stickers instead of m.sticker. + use_sticker_events: true + # The scale by which to display emojis with. Must be a positive integer. + emoji_scale_factor: 1 # The prefix for commands. Only required in non-management rooms. command_prefix: "!line" diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 276aa5f..b577dee 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -48,9 +48,9 @@ except ImportError: StateBridge = EventType.find("m.bridge", EventType.Class.STATE) StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE) -ReuploadedMediaInfo = NamedTuple('ReuploadedMediaInfo', mxc=Optional[ContentURI], - decryption_info=Optional[EncryptedFile], - mime_type=str, file_name=str, size=int) +MediaInfo = NamedTuple('MediaInfo', mxc=Optional[ContentURI], + decryption_info=Optional[EncryptedFile], + mime_type=str, file_name=str, size=int) class Portal(DBPortal, BasePortal): @@ -112,6 +112,7 @@ class Portal(DBPortal, BasePortal): cls.loop = bridge.loop cls.bridge = bridge cls.invite_own_puppet_to_pm = cls.config["bridge.invite_own_puppet_to_pm"] + cls.emoji_scale_factor = max(int(cls.config["bridge.emoji_scale_factor"]), 1) NotificationDisabler.puppet_cls = p.Puppet NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"] @@ -228,15 +229,38 @@ class Portal(DBPortal, BasePortal): self.log.debug(f"Ignoring duplicate message {evt.id}") return - event_id = None - if evt.image_url: - # TODO Deduplicate stickers, but only if encryption is disabled - content = await self._handle_remote_photo(source, intent, evt) - if not content: - content = TextMessageEventContent( - msgtype=MessageType.NOTICE, - body="") - event_id = await self._send_message(intent, content, timestamp=evt.timestamp) + if evt.image and evt.image.url: + if not evt.image.is_sticker or self.config["bridge.receive_stickers"]: + media_info = await self._handle_remote_media( + source, intent, evt.image.url, + deduplicate=not self.encrypted and evt.image.is_sticker) + image_info = ImageInfo( + # Element Web doesn't animate PNGs, but setting the mimetype to GIF works. + # (PNG stickers never animate, and PNG images only animate after being clicked on.) + # Making this exception since E.W. seems to be the only client that supports inline animated stickers & images. + # TODO Open an E.W. issue for this + # TODO Test Element Android + # TODO Find & test other non-GIF formats for animated images + mimetype="image/gif" if evt.image.is_animated and media_info.mime_type == "image/png" else media_info.mime_type, + size=media_info.size) if media_info else None + else: + media_info = None + send_sticker = self.config["bridge.use_sticker_events"] and evt.image.is_sticker and not self.encrypted and media_info + if send_sticker: + event_id = await intent.send_sticker( + self.mxid, media_info.mxc, image_info, "", timestamp=evt.timestamp) + else: + if media_info: + content = MediaMessageEventContent( + url=media_info.mxc, file=media_info.decryption_info, + msgtype=MessageType.IMAGE, + body=media_info.file_name, + info=image_info) + else: + content = TextMessageEventContent( + msgtype=MessageType.NOTICE, + body=f"<{'sticker' if evt.image.is_sticker else 'image'}>") + event_id = await self._send_message(intent, content, timestamp=evt.timestamp) elif evt.html and not evt.html.isspace(): chunks = [] @@ -267,6 +291,7 @@ class Portal(DBPortal, BasePortal): if msg_html: msg_html += chunk["data"] elif ctype == "img": + height = int(chunk.get("height", 19)) * self.emoji_scale_factor cclass = chunk["class"] if cclass == "emojione": alt = chunk["alt"] @@ -277,11 +302,11 @@ class Portal(DBPortal, BasePortal): media_id = f'{chunk.get("data-stickon-pkg-cd", 0)}/{chunk.get("data-stickon-stk-cd", 0)}' # NOTE Not encrypting content linked to by HTML tags - if not self.encrypted: - media_mxc = await self._get_mxc_for_remote_media(source, intent, chunk["src"], media_id) + if not self.encrypted and self.config["bridge.receive_stickers"]: + media_info = await self._handle_remote_media(source, intent, chunk["src"], media_id, deduplicate=True) if not msg_html: msg_html = msg_text - msg_html += f'{alt}' + msg_html += f'{alt}' msg_text += alt content = TextMessageEventContent( @@ -289,16 +314,21 @@ class Portal(DBPortal, BasePortal): format=Format.HTML if msg_html else None, body=msg_text, formatted_body=msg_html) event_id = await self._send_message(intent, content, timestamp=evt.timestamp) - if event_id: - if evt.is_outgoing and evt.receipt_count: - await self._handle_receipt(event_id, evt.receipt_count) - msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id) - try: - await msg.insert() - await self._send_delivery_receipt(event_id) - self.log.debug(f"Handled remote message {evt.id} -> {event_id}") - except UniqueViolationError as e: - self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}") + else: + content = TextMessageEventContent( + msgtype=MessageType.NOTICE, + body="") + event_id = await self._send_message(intent, content, timestamp=evt.timestamp) + + if evt.is_outgoing and evt.receipt_count: + await self._handle_receipt(event_id, evt.receipt_count) + msg = DBMessage(mxid=event_id, mx_room=self.mxid, mid=evt.id, chat_id=self.chat_id) + try: + await msg.insert() + await self._send_delivery_receipt(event_id) + self.log.debug(f"Handled remote message {evt.id} -> {event_id}") + except UniqueViolationError as e: + self.log.debug(f"Failed to handle remote message {evt.id} -> {event_id}: {e}") async def handle_remote_receipt(self, receipt: Receipt) -> None: msg = await DBMessage.get_by_mid(receipt.id) @@ -324,37 +354,34 @@ class Portal(DBPortal, BasePortal): reaction_mxid = await self.main_intent.react(self.mxid, event_id, f"(Read by {receipt_count})") await DBReceiptReaction(reaction_mxid, self.mxid, event_id, receipt_count).insert() - async def _handle_remote_photo(self, source: 'u.User', intent: IntentAPI, message: Message - ) -> Optional[MediaMessageEventContent]: - try: - resp = await source.client.read_image(message.image_url) - except (RPCError, TypeError) as e: - self.log.warning(f"Failed to download remote photo from chat {self.chat_id}: {e}") - return None - media_info = await self._reupload_remote_media(resp.data, intent, resp.mime) - return MediaMessageEventContent(url=media_info.mxc, file=media_info.decryption_info, - msgtype=MessageType.IMAGE, body=media_info.file_name, - info=ImageInfo(mimetype=media_info.mime_type, size=media_info.size)) - - async def _get_mxc_for_remote_media(self, source: 'u.User', intent: IntentAPI, - media_url: str, media_id: Optional[str] = None - ) -> ContentURI: + async def _handle_remote_media(self, source: 'u.User', intent: IntentAPI, + media_url: str, media_id: Optional[str] = None, + deduplicate: bool = False) -> MediaInfo: if not media_id: media_id = media_url - media_info = await DBMedia.get_by_id(media_id) - if not media_info: - self.log.debug(f"Did not find existing mxc URL for {media_id}, uploading media now") - resp = await source.client.read_image(media_url) - media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=True) - await DBMedia(media_id=media_id, mxc=media_info.mxc).insert() - self.log.debug(f"Uploaded media as {media_info.mxc}") + db_media_info = await DBMedia.get_by_id(media_id) if deduplicate else None + if not db_media_info: + # NOTE Blob URL of stickers only persists for a single session...still better than nothing. + self.log.debug(f"{'Did not find existing mxc URL for' if deduplicate else 'Not deduplicating'} {media_id}, uploading media now") + try: + resp = await source.client.read_image(media_url) + except (RPCError, TypeError) as e: + self.log.warning(f"Failed to download remote media from chat {self.chat_id}: {e}") + return None + media_info = await self._reupload_remote_media(resp.data, intent, resp.mime, disable_encryption=deduplicate) + if deduplicate: + await DBMedia( + media_id=media_id, mxc=media_info.mxc, + size=media_info.size, mime_type=media_info.mime_type, file_name=media_info.file_name + ).insert() + return media_info else: - self.log.debug(f"Found existing mxc URL for {media_id}: {media_info.mxc}") - return media_info.mxc + self.log.debug(f"Found existing mxc URL for {media_id}: {db_media_info.mxc}") + return MediaInfo(db_media_info.mxc, None, db_media_info.mime_type, db_media_info.file_name, db_media_info.size) async def _reupload_remote_media(self, data: bytes, intent: IntentAPI, mime_type: str = None, file_name: str = None, - disable_encryption: bool = True) -> ReuploadedMediaInfo: + disable_encryption: bool = True) -> MediaInfo: if not mime_type: mime_type = magic.from_buffer(data, mime=True) upload_mime_type = mime_type @@ -372,10 +399,13 @@ class Portal(DBPortal, BasePortal): filename=upload_file_name) if decryption_info: + self.log.debug(f"Uploaded encrypted media as {mxc}") decryption_info.url = mxc mxc = None + else: + self.log.debug(f"Uploaded media as {mxc}") - return ReuploadedMediaInfo(mxc, decryption_info, mime_type, file_name, len(data)) + return MediaInfo(mxc, decryption_info, mime_type, file_name, len(data)) async def update_info(self, conv: ChatInfo, client: Optional[Client]) -> None: if self.is_direct: diff --git a/matrix_puppeteer_line/rpc/types.py b/matrix_puppeteer_line/rpc/types.py index 3ddd302..9fdcebf 100644 --- a/matrix_puppeteer_line/rpc/types.py +++ b/matrix_puppeteer_line/rpc/types.py @@ -51,6 +51,13 @@ class ChatInfo(ChatListInfo, SerializableAttrs['ChatInfo']): participants: List[Participant] +@dataclass +class MessageImage(SerializableAttrs['MessageImage']): + url: str + is_sticker: bool + is_animated: bool + + @dataclass class Message(SerializableAttrs['Message']): id: int @@ -59,7 +66,7 @@ class Message(SerializableAttrs['Message']): sender: Optional[Participant] timestamp: int = None html: Optional[str] = None - image_url: Optional[str] = None + image: Optional[MessageImage] = None receipt_count: Optional[int] = None diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index 3de2b02..2fcefdf 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -69,17 +69,19 @@ window.__mautrixExpiry = function (button) {} * @return {Promise} */ window.__mautrixReceiveMessageID = function(id) {} -/** - * @return {Promise} - */ -window.__mautrixShowParticipantsList = function() {} +/** + * typedef ChatTypeEnum + */ const ChatTypeEnum = Object.freeze({ DIRECT: 1, GROUP: 2, ROOM: 3, }) +const MSG_DECRYPTING = "ⓘ Decrypting..." +// TODO consts for common selectors + class MautrixController { constructor() { this.chatListObserver = null @@ -92,7 +94,6 @@ class MautrixController { this.pinAppearObserver = null this.ownID = null - this.ownMsgPromise = Promise.resolve(-1) this._promiseOwnMsgReset() } @@ -172,28 +173,45 @@ class MautrixController { * @typedef MessageData * @type {object} * @property {number} id - The ID of the message. Seems to be sequential. - * @property {number} timestamp - The unix timestamp of the message. Not very accurate. + * @property {number} timestamp - The unix timestamp of the message. Accurate to the minute. * @property {boolean} is_outgoing - Whether or not this user sent the message. * @property {?Participant} sender - Full data of the participant who sent the message, if needed and available. * @property {?string} html - The HTML format of the message, if necessary. - * @property {?string} image_url - The URL to the image in the message, if it's an image-only message. + * @property {?ImageInfo} image - Information of the image in the message, if it's an image-only message. * @property {?int} receipt_count - The number of users who have read the message. */ + /** + * @typedef ImageInfo + * @type {object} + * @property {string} url - The URL of the image's location. + * @property {boolean} is_sticker - Whether the sent image is a sticker. + * @property {boolean} animated - Whether the sent image is animated. Only used for stickers (for now...?). + */ + + /** + * Return whether a URL points to a loaded image or not. + * + * @param {string} src + * @return boolean + * @private + */ _isLoadedImageURL(src) { - return src && (src.startsWith("blob:") || src.startsWith("res/")) + return src && ( + src.startsWith(`blob:`) || + src.startsWith(`${document.location.origin}/res/`) && !src.startsWith(`${document.location.origin}/res/img/noimg/`)) } /** - * Parse a message element (mws-message-wrapper) + * Parse a message element. * - * @param {Date} date - The most recent date indicator. * @param {Element} element - The message element. - * @param {int} chatType - What kind of chat this message is part of. - * @return {MessageData} + * @param {Number} chatType - What kind of chat this message is part of. + * @param {Date} refDate - The most recent date indicator. If undefined, do not retrieve the timestamp of this message. + * @return {Promise} * @private */ - async _parseMessage(date, element, chatType) { + async _parseMessage(element, chatType, refDate) { const is_outgoing = element.classList.contains("mdRGT07Own") let sender = {} @@ -208,6 +226,7 @@ class MautrixController { sender = null receipt_count = is_outgoing ? (receipt ? 1 : 0) : null } else if (!is_outgoing) { + let imgElement sender.name = element.querySelector(".mdRGT07Body > .mdRGT07Ttl").innerText // Room members are always friends (right?), // so search the friend list for the sender's name @@ -216,23 +235,23 @@ class MautrixController { // Group members aren't necessarily friends, // but the participant list includes their ID. if (!sender.id) { - await window.__mautrixShowParticipantsList() const participantsList = document.querySelector(participantsListSelector) - sender.id = participantsList.querySelector(`img[alt='${senderName}'`).parentElement.parentElement.getAttribute("data-mid") + imgElement = participantsList.querySelector(`img[alt='${sender.name}'`) + sender.id = imgElement.parentElement.parentElement.getAttribute("data-mid") + } else { + imgElement = element.querySelector(".mdRGT07Img > img") } - sender.avatar = this.getParticipantListItemAvatar(element) + sender.avatar = this._getPathImage(imgElement) receipt_count = null } else { // TODO Get own ID and store it somewhere appropriate. // Unable to get own ID from a room chat... // if (chatType == ChatTypeEnum.GROUP) { - // await window.__mautrixShowParticipantsList() // const participantsList = document.querySelector("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // // TODO The first member is always yourself, right? // // TODO Cache this so own ID can be used later // sender = participantsList.children[0].getAttribute("data-mid") // } - await window.__mautrixShowParticipantsList() const participantsList = document.querySelector(participantsListSelector) sender.name = this.getParticipantListItemName(participantsList.children[0]) sender.avatar = this.getParticipantListItemAvatar(participantsList.children[0]) @@ -243,56 +262,175 @@ class MautrixController { const messageData = { id: +element.getAttribute("data-local-id"), - timestamp: date ? date.getTime() : null, + timestamp: + refDate !== undefined + ? (await this._tryParseDate(element.querySelector("time")?.innerText, refDate))?.getTime() + : null, is_outgoing: is_outgoing, sender: sender, - receipt_count: receipt_count + receipt_count: receipt_count, } - const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg") - if (messageElement.classList.contains("mdRGT07Text")) { - messageData.html = messageElement.querySelector(".mdRGT07MsgTextInner")?.innerHTML - } else if ( - messageElement.classList.contains("mdRGT07Image") || - messageElement.classList.contains("mdRGT07Sticker") - ) { - const img = messageElement.querySelector(".mdRGT07MsgImg > img") - if (img) { - let imgResolve - // TODO Should reject on "#_chat_message_image_failure" - let observer = new MutationObserver(changes => { - for (const change of changes) { - if (this._isLoadedImageURL(change.target.src) && observer) { - observer.disconnect() - observer = null - imgResolve(change.target.src) - return - } - } - }) - observer.observe(img, { attributes: true, attributeFilter: ["src"] }) - if (this._isLoadedImageURL(img.src)) { - // Check for this AFTER attaching the observer, in case - // the image loaded after the img element was found but - // before the observer was attached. - messageData.image_url = img.src - observer.disconnect() - } else { - messageData.image_url = await new Promise(resolve => { - imgResolve = resolve - setTimeout(() => { - if (observer) { - observer.disconnect() - resolve(img.src) - } - }, 10000) // Longer timeout for image downloads - }) + const messageElement = element.querySelector(".mdRGT07Body > .mdRGT07Msg") + const is_sticker = messageElement.classList.contains("mdRGT07Sticker") + if (messageElement.classList.contains("mdRGT07Text")) { + let msgSpan = messageElement.querySelector(".mdRGT07MsgTextInner") + try { + if (msgSpan.innerHTML == MSG_DECRYPTING) { + msgSpan = await this._waitForDecryptedMessage(element, msgSpan, 5000) } + messageData.html = await this._parseMessageHTML(msgSpan) + } catch { + // Throw to reject, but return what was parsed so far + throw messageData + } + } else if (is_sticker || messageElement.classList.contains("mdRGT07Image")) { + // TODO Animated non-sticker images require clicking its img element, which is just a thumbnail + // Real image: "#wrap_single_image img" + // Close button: "#wrap_single_image button" + // Viewer is open/closed based on "#wrap_single_image.MdNonDisp" / "#wrap_single_image:not(.MdNonDisp)" + let img = messageElement.querySelector(".mdRGT07MsgImg > img") + if (!this._isLoadedImageURL(img.src)) { + try { + img = await this._waitForLoadedImage(img, 10000) + } catch { + // Throw to reject, but return what was parsed so far + throw messageData + } + } + messageData.image = { + url: img.src, + is_sticker: is_sticker, + is_animated: is_sticker && img.parentElement.classList.contains("animationSticker"), } } return messageData } + /** + * @param {Element} msgSpan + * @return Promise + * @private + */ + async _parseMessageHTML(msgSpan) { + const msgSpanImgs = msgSpan.getElementsByTagName("img") + if (msgSpanImgs.length == 0) { + return msgSpan.innerHTML + } else { + const unloadedImgs = Array.from(msgSpanImgs).filter(img => !this._isLoadedImageURL(img.src)) + if (unloadedImgs.length > 0) { + // NOTE Use allSettled to not throw if any images time out + await Promise.allSettled( + unloadedImgs.map(img => this._waitForLoadedImage(img, 2000)) + ) + } + + // Hack to put sticon dimensions in HTML (which are excluded by default) + // in such a way that doesn't alter the elements that are in the DOM + const msgSpanCopy = msgSpan.cloneNode(true) + const msgSpanCopyImgs = msgSpanCopy.getElementsByTagName("img") + for (let i = 0, n = msgSpanImgs.length; i < n; i++) { + msgSpanCopyImgs[i].height = msgSpanImgs[i].height + msgSpanCopyImgs[i].width = msgSpanImgs[i].width + } + return msgSpanCopy.innerHTML + } + } + + /** + * @param {Element} element + * @param {Element} msgSpan + * @param {Number} timeoutLimitMillis + * @return {Promise} + * @private + */ + _waitForDecryptedMessage(element, msgSpan, timeoutLimitMillis) { + console.debug("Wait for message element to finish decrypting") + console.debug(element) + return new Promise((resolve, reject) => { + let observer = new MutationObserver(changes => { + for (const change of changes) { + const isTextUpdate = change.type == "characterData" + const target = isTextUpdate ? msgSpan : element.querySelector(".mdRGT07MsgTextInner") + if (target && target.innerHTML != MSG_DECRYPTING) { + if (isTextUpdate) { + console.debug("UNLIKELY(?) EVENT -- Found decrypted message from text update") + } else { + // TODO Looks like it's div.mdRGT07Body that gets always replaced. If so, watch only for that + console.debug("Found decrypted message from element replacement") + console.debug(target) + console.debug("Added:") + for (const change of changes) { + console.debug(change.removedNodes) + } + console.debug("Removed:") + for (const change of changes) { + console.debug(change.addedNodes) + } + } + observer.disconnect() + observer = null + resolve(target) + return + } + if (target && target != msgSpan) { + console.debug("UNLIKELY EVENT -- Somehow added a new \"decrypting\" span, it's the one to watch now") + console.debug(target) + msgSpan = target + observer.observe(msgSpan, { characterData: true }) + } + } + }) + // Either the span element or one of its ancestors is replaced, + // or the span element's content is updated. + // Not exactly sure which of these happens, or if the same kind + // of mutation always happens, so just look for them all... + observer.observe(element, { childList: true, subtree: true }) + observer.observe(msgSpan, { characterData: true }) + setTimeout(() => { + if (observer) { + observer.disconnect() + // Don't print log message, as this may be a safe timeout + reject() + } + }, timeoutLimitMillis) + }) + } + + /** + * @param {Element} img + * @param {Number} timeoutLimitMillis + * @return {Promise} + * @private + */ + _waitForLoadedImage(img, timeoutLimitMillis) { + console.debug("Wait for image element to finish loading") + console.debug(img) + // TODO Should reject on "#_chat_message_image_failure" + return new Promise((resolve, reject) => { + let observer = new MutationObserver(changes => { + for (const change of changes) { + if (this._isLoadedImageURL(change.target.src)) { + console.debug("Image element finished loading") + console.debug(change.target) + observer.disconnect() + observer = null + resolve(change.target) + return + } + } + }) + observer.observe(img, { attributes: true, attributeFilter: ["src"] }) + setTimeout(() => { + if (observer) { + observer.disconnect() + // Don't print log message, as this may be a safe timeout + reject() + } + }, timeoutLimitMillis) + }) + } + /** * Find the number in the "Read #" receipt message. * Don't look for "Read" specifically, to support multiple languages. @@ -313,8 +451,8 @@ class MautrixController { * has succeeded or failed to be sent. * * @param {int} timeoutLimitMillis - The maximum amount of time to wait for the message to be sent. - * @param {str} successSelector - The selector for the element that indicates the message was sent. - * @param {str} failureSelector - The selector for the element that indicates the message failed to be sent. + * @param {string} successSelector - The selector for the element that indicates the message was sent. + * @param {?string} failureSelector - The selector for the element that indicates the message failed to be sent. */ promiseOwnMessage(timeoutLimitMillis, successSelector, failureSelector=null) { this.promiseOwnMsgSuccessSelector = successSelector @@ -323,13 +461,13 @@ class MautrixController { this.ownMsgPromise = new Promise((resolve, reject) => { this.promiseOwnMsgResolve = resolve this.promiseOwnMsgReject = reject - setTimeout(() => { - if (this.promiseOwnMsgReject) { - console.log("Timeout!") - this._rejectOwnMessage() - } - }, timeoutLimitMillis) }) + this.promiseOwnMsgTimeoutID = setTimeout(() => { + if (this.promiseOwnMsgReject) { + console.error("Timed out waiting for own message to be sent") + this._rejectOwnMessage() + } + }, timeoutLimitMillis) } /** @@ -338,40 +476,39 @@ class MautrixController { * @return {Promise} - The ID of the sent message. */ async waitForOwnMessage() { - return await this.ownMsgPromise - } - - async _tryParseMessages(msgList, chatType) { - const messages = [] - let refDate = null - for (const child of msgList) { - if (child.classList.contains("mdRGT10Date")) { - refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText) - } else if (child.classList.contains("MdRGT07Cont")) { - // TODO :not(.MdNonDisp) to exclude not-yet-posted messages, - // but that is unlikely to be a problem here. - // Also, offscreen times may have .MdNonDisp on them - // TODO Explicitly look for the most recent date element, - // as it might not have been one of the new items in msgList - const timeElement = child.querySelector("time") - if (timeElement) { - const messageDate = await this._tryParseDate(timeElement.innerText, refDate) - messages.push(await this._parseMessage(messageDate, child, chatType)) - } - } - } - return messages + return this.ownMsgPromise ? await this.ownMsgPromise : -1 } /** * Parse the message list of whatever the currently-viewed chat is. * - * @return {[MessageData]} - A list of messages. + * @param {int} minID - The minimum message ID to consider. + * @return {Promise<[MessageData]>} - A list of messages. */ - async parseMessageList() { - const msgList = Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]")) - msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id")) - return await this._tryParseMessages(msgList, this.getChatType(this.getCurrentChatID())) + async parseMessageList(minID = 0) { + console.debug(`minID for full refresh: ${minID}`) + const msgList = + Array.from(document.querySelectorAll("#_chat_room_msg_list > div[data-local-id]")) + .filter(msg => + msg.hasAttribute("data-local-id") && + (!msg.classList.contains("MdRGT07Cont") || msg.getAttribute("data-local-id") > minID)) + if (msgList.length == 0) { + return [] + } + const messagePromises = [] + const chatType = this.getChatType(this.getCurrentChatID()) + let refDate = null + for (const child of msgList) { + if (child.classList.contains("mdRGT10Date")) { + refDate = await this._tryParseDateSeparator(child.firstElementChild.innerText) + } else if (child.classList.contains("MdRGT07Cont")) { + messagePromises.push(this._parseMessage(child, chatType, refDate)) + } + } + // NOTE No message should ever time out, but use allSettled to not throw if any do + return (await Promise.allSettled(messagePromises)) + .filter(value => value.status == "fulfilled") + .map(value => value.value) } /** @@ -439,7 +576,7 @@ class MautrixController { const name = this.getParticipantListItemName(child) const id = this.getParticipantListItemID(child) || this.getUserIdFromFriendsList(name) return { - id: id, // NOTE Don't want non-own user's ID to ever be null. + id: id, avatar: this.getParticipantListItemAvatar(child), name: name, } @@ -497,6 +634,7 @@ class MautrixController { /** * Parse the list of recent/saved chats. + * * @return {[ChatListInfo]} - The list of chats. */ parseChatList() { @@ -505,20 +643,6 @@ class MautrixController { child => this.parseChatListItem(child.firstElementChild)) } - /** - * TODO - * Check if an image has been downloaded. - * - * @param {number} id - The ID of the message whose image to check. - * @return {boolean} - Whether or not the image has been downloaded - */ - imageExists(id) { - const imageElement = document.querySelector( - `mws-message-wrapper[msg-id="${id}"] mws-image-message-part .image-msg`) - return !imageElement.classList.contains("not-rendered") - && imageElement.getAttribute("src") !== "" - } - /** * Download an image at a given URL and return it as a data URL. * @@ -544,7 +668,6 @@ class MautrixController { // TODO Observe *added/removed* chats, not just new messages const changedChatIDs = new Set() for (const change of mutations) { - console.debug("Chat list mutation:", change) if (change.target.id == "_chat_list_body") { // TODO // These could be new chats, or they're @@ -555,16 +678,16 @@ class MautrixController { */ } else if (change.target.tagName == "LI") { if (change.target.classList.contains("ExSelected")) { - console.log("Not using chat list mutation response for currently-active chat") + console.debug("Not using chat list mutation response for currently-active chat") continue } for (const node of change.addedNodes) { const chat = this.parseChatListItem(node) if (chat) { - console.log("Changed chat list item:", chat) + console.log("Added chat list item:", chat) changedChatIDs.add(chat.id) } else { - console.debug("Could not parse node as a chat list item:", node) + console.debug("Could not parse added node as a chat list item:", node) } } } @@ -584,10 +707,12 @@ class MautrixController { addChatListObserver() { this.removeChatListObserver() this.chatListObserver = new MutationObserver(async (mutations) => { - // Wait for pending sent messages to be resolved before responding to mutations - try { - await this.ownMsgPromise - } catch (e) {} + if (this.ownMsgPromise) { + // Wait for pending sent messages to be resolved before responding to mutations + try { + await this.ownMsgPromise + } catch (e) {} + } try { this._observeChatListMutations(mutations) @@ -598,7 +723,7 @@ class MautrixController { this.chatListObserver.observe( document.querySelector("#_chat_list_body"), { childList: true, subtree: true }) - console.debug("Started chat list observer") + console.log("Started chat list observer") } /** @@ -608,13 +733,13 @@ class MautrixController { if (this.chatListObserver !== null) { this.chatListObserver.disconnect() this.chatListObserver = null - console.debug("Disconnected chat list observer") + console.log("Disconnected chat list observer") } } /** * @param {[MutationRecord]} mutations - The mutation records that occurred - * @param {str} chatID - The ID of the chat being observed. + * @param {string} chatID - The ID of the chat being observed. * @private */ _observeReceiptsDirect(mutations, chatID) { @@ -641,7 +766,7 @@ class MautrixController { /** * @param {[MutationRecord]} mutations - The mutation records that occurred - * @param {str} chatID - The ID of the chat being observed. + * @param {string} chatID - The ID of the chat being observed. * @private */ _observeReceiptsMulti(mutations, chatID) { @@ -649,17 +774,17 @@ class MautrixController { const receipts = [] for (const change of mutations) { const target = change.type == "characterData" ? change.target.parentElement : change.target - if ( change.target.classList.contains("mdRGT07Read") && - !change.target.classList.contains("MdNonDisp")) + if ( target.classList.contains("mdRGT07Read") && + !target.classList.contains("MdNonDisp")) { - const msgElement = change.target.closest(".mdRGT07Own") + const msgElement = target.closest(".mdRGT07Own") if (msgElement) { const id = +msgElement.getAttribute("data-local-id") if (!ids.has(id)) { ids.add(id) receipts.push({ id: id, - count: this._getReceiptCount(change.target), + count: this._getReceiptCount(target), }) } } @@ -673,11 +798,90 @@ class MautrixController { } } + /** + * @typedef PendingMessage + * @type object + * + * @property {Promise} promise + * @property {Number} id + */ + + /** + * @typedef SameIDMsgs + * @type object + * + * @property {Number} id + * @property {PendingMessage[]} msgs + * @property {Function} resolve + * @property {Number} numRejected + */ + + /** + * Binary search for the array of messages with the provided ID. + * + * @param {SameIDMsgs[]} sortedSameIDMsgs + * @param {Number} id + * @param {boolean} returnClosest - If true, return the index of the nearest result on miss instead of -1. + * @return {Number} The index of the matched element, or -1 if not found. + */ + _findMsgsForID( + sortedSameIDMsgs, id, returnClosest = false, + lowerBound = 0, upperBound = sortedSameIDMsgs.length - 1) + { + if (lowerBound > upperBound) { + return -1 + } + if (returnClosest && lowerBound == upperBound) { + // Caller must check if the result has a matching ID or not + return sortedSameIDMsgs[lowerBound].id <= id ? lowerBound : lowerBound-1 + } + const i = lowerBound + Math.floor((upperBound - lowerBound)/2) + const val = sortedSameIDMsgs[i] + if (val.id == id) { + return i + } else if (val.id < id) { + return this._findMsgsForID( + sortedSameIDMsgs, id, returnClosest, + i+1, upperBound) + } else { + return this._findMsgsForID( + sortedSameIDMsgs, id, returnClosest, + lowerBound, i-1) + } + } + + /** + * Insert the given message to the proper inner array. + * In no inner array exists, insert a new one, preserving sort order. + * Return the wrapper of which inner array was added to or created. + * + * @param {SameIDMsgs[]} sortedSameIDMsgs + * @param {PendingMessage} msg + * @return {SameIDMsgs} + */ + _insertMsgByID(sortedSameIDMsgs, msg) { + let i = this._findMsgsForID(sortedSameIDMsgs, msg.id, true) + if (i != -1 && sortedSameIDMsgs[i].id == msg.id) { + sortedSameIDMsgs[i].msgs.push(msg) + console.debug("UNLIKELY(?) EVENT -- Found two new message elements with the same ID, so tracking both of them") + } else { + sortedSameIDMsgs.splice(++i, 0, { + id: msg.id, + msgs: [msg], + numRejected: 0, + resolve: null, + }) + } + return sortedSameIDMsgs[i] + } + /** * Add a mutation observer to the message list of the current chat. * Used for observing new messages & read receipts. + * + * @param {int} minID - The minimum message ID to consider. */ - addMsgListObserver() { + addMsgListObserver(minID = 0) { const chat_room_msg_list = document.querySelector("#_chat_room_msg_list") if (!chat_room_msg_list) { console.debug("Could not start msg list observer: no msg list available!") @@ -688,34 +892,133 @@ class MautrixController { const chatID = this.getCurrentChatID() const chatType = this.getChatType(chatID) - let orderedPromises = [Promise.resolve()] + // NEED TO HANDLE: + // * message elements arriving in any order + // * messages being potentially pending (i.e. decrypting or loading), + // and resolving in a potentially different order than they arrived in + // * pending messages potentially having multiple elements associated with + // them, where only one of them resolves + // * message elements being added/removed any number of times, which may + // or may not ever resolve + // * outgoing messages (i.e. sent by the bridge) + // And must send resolved messages to the bridge *in order*! + // BUT: Assuming that incoming messages will never be younger than a resolved one. + + const sortedSameIDMsgs = [] + const pendingMsgElements = new Set() + this.msgListObserver = new MutationObserver(changes => { - let msgList = [] + console.debug(`MESSAGE LIST CHANGES: check since ${minID}`) + const remoteMsgs = [] for (const change of changes) { - change.addedNodes.forEach(child => { - if (child.tagName == "DIV" && child.hasAttribute("data-local-id")) { - msgList.push(child) + console.debug("---new change set---") + for (const child of change.addedNodes) { + if (!pendingMsgElements.has(child) && + child.tagName == "DIV" && + child.hasAttribute("data-local-id") && + // Skip timestamps, as these are always current + child.classList.contains("MdRGT07Cont")) + { + const msgID = child.getAttribute("data-local-id") + if (msgID > minID) { + pendingMsgElements.add(child) + + // TODO Maybe handle own messages somewhere else...? + const ownMsg = this._observeOwnMessage(child) + if (ownMsg) { + console.log("Found own bridge-sent message, will wait for it to resolve") + console.debug(child) + this.ownMsgPromise + .then(msgID => { + console.log("Resolved own bridge-sent message") + console.debug(ownMsg) + pendingMsgElements.delete(ownMsg) + if (minID < msgID) { + minID = msgID + } + }) + .catch(() => { + console.log("Rejected own bridge-sent message") + console.debug(ownMsg) + pendingMsgElements.delete(ownMsg) + }) + } else { + console.log("Found remote message") + console.debug(child) + remoteMsgs.push({ + id: msgID, + element: child + }) + } + } } - }) + } + // NOTE Ignoring removedNodes because an element can always be added back. + // Will simply let permanently-removed nodes time out. } - if (msgList.length == 0) { + if (remoteMsgs.length == 0) { + console.debug("Found no new remote messages") return } - msgList.sort((a,b) => a.getAttribute("data-local-id") - b.getAttribute("data-local-id")) - if (!this._observeOwnMessage(msgList)) { - let prevPromise = orderedPromises.shift() - orderedPromises.push(new Promise(resolve => prevPromise - .then(() => this._tryParseMessages(msgList, chatType)) - .then(msgs => window.__mautrixReceiveMessages(chatID, msgs)) - .then(() => resolve()) - )) + + // No need to sort remoteMsgs, because sortedSameIDMsgs is enough + for (const msg of remoteMsgs) { + const messageElement = msg.element + const pendingMessage = { + id: msg.id, + promise: this._parseMessage(messageElement, chatType) + } + const sameIDMsgs = this._insertMsgByID(sortedSameIDMsgs, pendingMessage) + + const handleMessage = async (messageData) => { + minID = messageData.id + sortedSameIDMsgs.shift() + await window.__mautrixReceiveMessages(chatID, [messageData]) + if (sortedSameIDMsgs.length > 0 && sortedSameIDMsgs[0].resolve) { + console.debug("Allowing queued resolved message to be sent") + console.debug(sortedSameIDMsgs[0]) + sortedSameIDMsgs[0].resolve() + } + } + + pendingMessage.promise.then( + async (messageData) => { + const i = this._findMsgsForID(sortedSameIDMsgs, messageData.id) + if (i == -1) { + console.debug(`Got resolved message for already-handled ID ${messageData.id}, ignore it`) + pendingMsgElements.delete(messageElement) + return + } + if (i != 0) { + console.debug(`Got resolved message for later ID ${messageData.id}, wait for earlier messages`) + await new Promise(resolve => sameIDMsgs.resolve = resolve) + console.debug(`Message before ID ${messageData.id} finished, can now send this one`) + } else { + console.debug(`Got resolved message for earliest ID ${messageData.id}, send it`) + } + console.debug(messageElement) + pendingMsgElements.delete(messageElement) + handleMessage(messageData) + }, + // error case + async (messageData) => { + console.debug("Message element rejected") + console.debug(messageElement) + pendingMsgElements.delete(messageElement) + if (++sameIDMsgs.numRejected == sameIDMsgs.msgs.length) { + // Note that if another message element with this ID somehow comes later, it'll be ignored. + console.debug(`All messages for ID ${sameIDMsgs.id} rejected, abandoning this ID and sending dummy message`) + // Choice of which message to send should be arbitrary + handleMessage(messageData) + } + }) } }) this.msgListObserver.observe( chat_room_msg_list, { childList: true }) - console.debug("Started msg list observer") + console.debug(`Started msg list observer with minID = ${minID}`) const observeReadReceipts = ( @@ -736,74 +1039,65 @@ class MautrixController { subtree: true, attributes: true, attributeFilter: ["class"], - // TODO Consider using the same observer to watch for "ⓘ Decrypting..." characterData: chatType != ChatTypeEnum.DIRECT, }) console.debug("Started receipt observer") } - _observeOwnMessage(msgList) { - if (!this.promiseOwnMsgSuccessSelector) { + _observeOwnMessage(ownMsg) { + if (!this.ownMsgPromise) { // Not waiting for a pending sent message - return false - } - if (this.visibleSuccessObserver) { - // Already found a element that we're waiting on becoming visible - return true + return null } - for (const ownMsg of msgList.filter(msg => msg.classList.contains("mdRGT07Own"))) { - const successElement = - ownMsg.querySelector(this.promiseOwnMsgSuccessSelector) - if (successElement) { - if (successElement.classList.contains("MdNonDisp")) { - console.log("Invisible success") - console.log(successElement) - } else { - console.debug("Already visible success, must not be it") - console.debug(successElement) - continue - } + const successElement = + ownMsg.querySelector(this.promiseOwnMsgSuccessSelector) + if (successElement) { + if (successElement.classList.contains("MdNonDisp")) { + console.log("Invisible success for own bridge-sent message, will wait for it to resolve") + console.log(successElement) } else { - continue + console.debug("Already visible success, must not be it") + console.debug(successElement) + return null } - - const failureElement = - this.promiseOwnMsgFailureSelector && - ownMsg.querySelector(this.promiseOwnMsgFailureSelector) - if (failureElement) { - if (failureElement.classList.contains("MdNonDisp")) { - console.log("Invisible failure") - console.log(failureElement) - } else { - console.debug("Already visible failure, must not be it") - console.log(failureElement) - continue - } - } else if (this.promiseOwnMsgFailureSelector) { - continue - } - - console.log("Found invisible element, wait") - const msgID = +ownMsg.getAttribute("data-local-id") - this.visibleSuccessObserver = new MutationObserver( - this._getOwnVisibleCallback(msgID)) - this.visibleSuccessObserver.observe( - successElement, - { attributes: true, attributeFilter: ["class"] }) - - if (this.promiseOwnMsgFailureSelector) { - this.visibleFailureObserver = new MutationObserver( - this._getOwnVisibleCallback()) - this.visibleFailureObserver.observe( - failureElement, - { attributes: true, attributeFilter: ["class"] }) - } - - return true + } else { + return null } - return false + + const failureElement = + this.promiseOwnMsgFailureSelector && + ownMsg.querySelector(this.promiseOwnMsgFailureSelector) + if (failureElement) { + if (failureElement.classList.contains("MdNonDisp")) { + console.log("Invisible failure for own bridge-sent message, will wait for it (or success) to resolve") + console.log(failureElement) + } else { + console.debug("Already visible failure, must not be it") + console.log(failureElement) + return null + } + } else if (this.promiseOwnMsgFailureSelector) { + return null + } + + const msgID = +ownMsg.getAttribute("data-local-id") + this.visibleSuccessObserver = new MutationObserver( + this._getOwnVisibleCallback(msgID)) + this.visibleSuccessObserver.observe( + successElement, + { attributes: true, attributeFilter: ["class"] }) + + if (this.promiseOwnMsgFailureSelector) { + this.visibleFailureObserver = new MutationObserver( + this._getOwnVisibleCallback()) + this.visibleFailureObserver.observe( + failureElement, + { attributes: true, attributeFilter: ["class"] }) + } + + return ownMsg } _getOwnVisibleCallback(msgID=null) { @@ -811,7 +1105,7 @@ class MautrixController { return changes => { for (const change of changes) { if (!change.target.classList.contains("MdNonDisp")) { - console.log(`Waited for visible ${isSuccess ? "success" : "failure"}`) + console.log(`Resolved ${isSuccess ? "success" : "failure"} for own bridge-sent message`) console.log(change.target) isSuccess ? this._resolveOwnMessage(msgID) : this._rejectOwnMessage(change.target) return @@ -822,6 +1116,7 @@ class MautrixController { _resolveOwnMessage(msgID) { if (!this.promiseOwnMsgResolve) return + clearTimeout(this.promiseOwnMsgTimeoutID) const resolve = this.promiseOwnMsgResolve this._promiseOwnMsgReset() @@ -838,10 +1133,12 @@ class MautrixController { } _promiseOwnMsgReset() { + this.ownMsgPromise = null this.promiseOwnMsgSuccessSelector = null this.promiseOwnMsgFailureSelector = null this.promiseOwnMsgResolve = null this.promiseOwnMsgReject = null + this.promiseOwnMsgTimeoutID = null if (this.visibleSuccessObserver) { this.visibleSuccessObserver.disconnect() diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 0f07af9..0238b70 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -107,7 +107,6 @@ export default class MessagesPuppeteer { this._receiveReceiptDirectLatest.bind(this)) await this.page.exposeFunction("__mautrixReceiveReceiptMulti", this._receiveReceiptMulti.bind(this)) - await this.page.exposeFunction("__mautrixShowParticipantsList", this._showParticipantList.bind(this)) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) // NOTE Must *always* re-login on a browser session, so no need to check if already logged in @@ -376,15 +375,7 @@ export default class MessagesPuppeteer { * @return {Promise<[MessageData]>} - The messages visible in the chat. */ async getMessages(chatID) { - return await this.taskQueue.push(async () => { - const messages = await this._getMessagesUnsafe(chatID) - if (messages.length > 0) { - for (const message of messages) { - message.chat_id = chatID - } - } - return messages - }) + return await this.taskQueue.push(async () => this._getMessagesUnsafe(chatID)) } setLastMessageIDs(ids) { @@ -392,7 +383,8 @@ export default class MessagesPuppeteer { for (const [chatID, messageID] of Object.entries(ids)) { this.mostRecentMessages.set(chatID, messageID) } - this.log("Updated most recent message ID map:", this.mostRecentMessages) + this.log("Updated most recent message ID map:") + this.log(this.mostRecentMessages) } async readImage(imageUrl) { @@ -407,11 +399,15 @@ export default class MessagesPuppeteer { } async startObserving() { - this.log("Adding observers") + const chatID = await this.page.evaluate(() => window.__mautrixController.getCurrentChatID()) + this.log(`Adding observers for ${chatID || "empty chat"}`) await this.page.evaluate( () => window.__mautrixController.addChatListObserver()) - await this.page.evaluate( - () => window.__mautrixController.addMsgListObserver()) + if (chatID) { + await this.page.evaluate( + (mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + } } async stopObserving() { @@ -444,9 +440,10 @@ export default class MessagesPuppeteer { if (await this.page.evaluate(isCorrectChatVisible, chatName)) { this.log("Already viewing chat, no need to switch") } else { - this.log("Switching chat, so remove msg list observer") + this.log("Ensuring msg list observer is removed") const hadMsgListObserver = await this.page.evaluate( () => window.__mautrixController.removeMsgListObserver()) + this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer") await chatListItem.click() this.log(`Waiting for chat header title to be "${chatName}"`) @@ -455,39 +452,28 @@ export default class MessagesPuppeteer { {polling: "mutation"}, chatName) - // For consistent behaviour later, wait for the chat details sidebar to be hidden + // Always show the chat details sidebar, as this makes life easier + this.log("Waiting for detail area to be auto-hidden upon entering chat") await this.page.waitForFunction( detailArea => detailArea.childElementCount == 0, {}, await this.page.$("#_chat_detail_area")) + this.log("Clicking chat header to show detail area") + await this.page.click("#_chat_header_area > .mdRGT04Link") + this.log("Waiting for detail area") + await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info") if (hadMsgListObserver) { this.log("Restoring msg list observer") await this.page.evaluate( - () => window.__mautrixController.addMsgListObserver()) + (mostRecentMessage) => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) } else { this.log("Not restoring msg list observer, as there never was one") } } } - // TODO Commonize - async _getParticipantList() { - await this._showParticipantList() - return await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") - } - - async _showParticipantList() { - const selector = "#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul" - let participantList = await this.page.$(selector) - if (!participantList) { - this.log("Participant list hidden, so clicking chat header to show it") - await this.page.click("#_chat_header_area > .mdRGT04Link") - // Use no timeout since the browser itself is using this - await this.page.waitForSelector(selector, {timeout: 0}) - } - } - async _getChatInfoUnsafe(chatID) { const chatListItem = await this.page.$(this._listItemSelector(chatID)) const chatListInfo = await chatListItem.evaluate( @@ -512,7 +498,7 @@ export default class MessagesPuppeteer { this.log("Found multi-user chat, so clicking chat header to get participants") // TODO This will mark the chat as "read"! await this._switchChat(chatID) - const participantList = await this._getParticipantList() + const participantList = await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // TODO Is a group not actually created until a message is sent(?) // If so, maybe don't create a portal until there is a message. participants = await participantList.evaluate( @@ -542,6 +528,7 @@ export default class MessagesPuppeteer { async _sendMessageUnsafe(chatID, text) { await this._switchChat(chatID) + // TODO Initiate the promise in the content script await this.page.evaluate( () => window.__mautrixController.promiseOwnMessage(5000, "time")) @@ -593,15 +580,12 @@ export default class MessagesPuppeteer { } } - _receiveMessages(chatID, messages) { + async _receiveMessages(chatID, messages) { if (this.client) { - messages = this._filterMessages(chatID, messages) - if (messages.length > 0) { - for (const message of messages) { - message.chat_id = chatID - this.client.sendMessage(message).catch(err => - this.error("Failed to send message", message.id, "to client:", err)) - } + messages = await this._processMessages(chatID, messages) + for (const message of messages) { + this.client.sendMessage(message).catch(err => + this.error("Failed to send message", message.id, "to client:", err)) } } else { this.log("No client connected, not sending messages") @@ -609,27 +593,52 @@ export default class MessagesPuppeteer { } async _getMessagesUnsafe(chatID) { - // TODO Also handle "decrypting" state + // TODO Consider making a wrapper for pausing/resuming the msg list observers + this.log("Ensuring msg list observer is removed") + const hadMsgListObserver = await this.page.evaluate( + () => window.__mautrixController.removeMsgListObserver()) + this.log(hadMsgListObserver ? "Observer was already removed" : "Removed observer") + // TODO Handle unloaded messages. Maybe scroll up // TODO This will mark the chat as "read"! await this._switchChat(chatID) - const messages = await this.page.evaluate(() => - window.__mautrixController.parseMessageList()) - return this._filterMessages(chatID, messages) + const messages = await this.page.evaluate( + mostRecentMessage => window.__mautrixController.parseMessageList(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + // Doing this before restoring the observer since it updates minID + const filteredMessages = await this._processMessages(chatID, messages) + + if (hadMsgListObserver) { + this.log("Restoring msg list observer") + await this.page.evaluate( + mostRecentMessage => window.__mautrixController.addMsgListObserver(mostRecentMessage), + this.mostRecentMessages.get(chatID)) + } else { + this.log("Not restoring msg list observer, as there never was one") + } + + return filteredMessages } - _filterMessages(chatID, messages) { + async _processMessages(chatID, messages) { + // TODO Probably don't need minID filtering if Puppeteer context handles it now const minID = this.mostRecentMessages.get(chatID) || 0 - const filtered_messages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id)) + const filteredMessages = messages.filter(msg => msg.id > minID && !this.sentMessageIDs.has(msg.id)) - if (filtered_messages.length > 0) { - const newFirstID = filtered_messages[0].id - const newLastID = filtered_messages[filtered_messages.length - 1].id + if (filteredMessages.length > 0) { + const newFirstID = filteredMessages[0].id + const newLastID = filteredMessages[filteredMessages.length - 1].id this.mostRecentMessages.set(chatID, newLastID) const range = newFirstID === newLastID ? newFirstID : `${newFirstID}-${newLastID}` - this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filtered_messages.length} newer than ${minID} (${range})`) + this.log(`Loaded ${messages.length} messages in ${chatID}, got ${filteredMessages.length} newer than ${minID} (${range})`) + + for (const message of filteredMessages) { + message.chat_id = chatID + } + return filteredMessages + } else { + return [] } - return filtered_messages } async _processChatListChangeUnsafe(chatID) { @@ -643,7 +652,6 @@ export default class MessagesPuppeteer { if (this.client) { for (const message of messages) { - message.chat_id = chatID await this.client.sendMessage(message).catch(err => this.error("Failed to send message", message.id, "to client:", err)) } From c89c59e7fc098ffcd1db83264a2d5a0d2e899227 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:18:57 -0400 Subject: [PATCH 06/22] Handle incoming line breaks --- matrix_puppeteer_line/portal.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index b577dee..b46f0c7 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -269,12 +269,11 @@ class Portal(DBPortal, BasePortal): chunks.append({"type": "data", "data": data}) def handle_starttag(tag, attrs): - if tag == "img": - obj = {"type": tag} - for attr in attrs: - obj[attr[0]] = attr[1] - nonlocal chunks - chunks.append(obj) + obj = {"type": tag} + for attr in attrs: + obj[attr[0]] = attr[1] + nonlocal chunks + chunks.append(obj) parser = HTMLParser() parser.handle_data = handle_data @@ -286,7 +285,12 @@ class Portal(DBPortal, BasePortal): for chunk in chunks: ctype = chunk["type"] - if ctype == "data": + if ctype == "br": + msg_text += "\n" + if not msg_html: + msg_html = msg_text + msg_html += "
" + elif ctype == "data": msg_text += chunk["data"] if msg_html: msg_html += chunk["data"] From 11ba99e17b441d5c5b80e71c00f7a14edf9fbcd2 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:19:36 -0400 Subject: [PATCH 07/22] Fix mistake in receipt sending And possible mistake in member kicking --- matrix_puppeteer_line/portal.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index b46f0c7..899a112 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -349,9 +349,10 @@ class Portal(DBPortal, BasePortal): if reaction: await self.main_intent.redact(self.mxid, reaction.mxid) await reaction.delete() + # TODO Not just -1 if there are multiple _OWN_ puppets... if receipt_count == len(self._last_participant_update) - 1: - for participant in self._last_participant_update: - puppet = await p.Puppet.get_by_mid(participant.id) + for participant in filter(lambda participant: not p.Puppet.is_mid_for_own_puppet(participant), self._last_participant_update): + puppet = await p.Puppet.get_by_mid(participant) await puppet.intent.send_receipt(self.mxid, event_id) else: # TODO Translatable string for "Read by" @@ -477,10 +478,10 @@ class Portal(DBPortal, BasePortal): # Make sure puppets who should be here are here for participant in participants: - puppet = await p.Puppet.get_by_mid(participant.id) if forbid_own_puppets and p.Puppet.is_mid_for_own_puppet(participant.id): continue - await puppet.intent.ensure_joined(self.mxid) + intent = (await p.Puppet.get_by_mid(participant.id)).intent + await intent.ensure_joined(self.mxid) print(current_members) From 27fda19567e58d9bb5c08d76de5c9b0e72b2718a Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:20:17 -0400 Subject: [PATCH 08/22] Allow enabling encryption after room creation Also forbid reactions --- matrix_puppeteer_line/portal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 899a112..746084a 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -628,9 +628,11 @@ class Portal(DBPortal, BasePortal): "users": { self.az.bot_mxid: 100, self.main_intent.mxid: 100, + source.mxid: 25, }, "events": { - str(EventType.REACTION): 1 + str(EventType.REACTION): 100, + str(EventType.ROOM_ENCRYPTION): 25, } } }) From 34ea2021ecbb2bcf623883f75ec8e8bb02532758 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 6 Jun 2021 18:21:32 -0400 Subject: [PATCH 09/22] Attempt to respect bridge.initial_conversation_sync But probably doing it wrong --- matrix_puppeteer_line/user.py | 14 ++++++++------ puppet/src/puppet.js | 11 ++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/matrix_puppeteer_line/user.py b/matrix_puppeteer_line/user.py index 2d4ea36..d464063 100644 --- a/matrix_puppeteer_line/user.py +++ b/matrix_puppeteer_line/user.py @@ -124,22 +124,24 @@ class User(DBUser, BaseUser): if self._connection_check_task: self._connection_check_task.cancel() self._connection_check_task = self.loop.create_task(self._check_connection_loop()) - await self.client.set_last_message_ids(await DBMessage.get_max_mids()) - self.log.info("Syncing chats") - await self.send_notice("Synchronizing chats...") await self.client.pause() - chats = await self.client.get_chats() + await self.client.set_last_message_ids(await DBMessage.get_max_mids()) limit = self.config["bridge.initial_conversation_sync"] + self.log.info("Syncing chats") + await self.send_bridge_notice("Synchronizing chats...") + chats = await self.client.get_chats() + num_created = 0 for index, chat in enumerate(chats): portal = await po.Portal.get_by_chat_id(chat.id, create=True) - if portal.mxid or index < limit: + if portal.mxid or num_created < limit: chat = await self.client.get_chat(chat.id) if portal.mxid: await portal.update_matrix_room(self, chat) else: await portal.create_matrix_room(self, chat) + num_created += 1 + await self.send_bridge_notice("Synchronization complete") await self.client.resume() - await self.send_notice("Synchronization complete") async def stop(self) -> None: # TODO Notices for shutdown messages diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 0238b70..b63a38a 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -247,7 +247,16 @@ export default class MessagesPuppeteer { } this.loginRunning = false - await this.startObserving() + // Don't start observing yet, instead wait for explicit request. + // But at least view the most recent chat. + try { + let mostRecentChatID = await this.page.$eval("#_chat_list_body li", + element => window.getChatListItemID(element)) + await this._switchChat(mostRecentChatID) + this.log("Focused on most recent chat") + } catch (e) { + this.log("No chats available to focus on") + } this.log("Login complete") } From 54507f8aaf64fa2e6549eaef5219d36802db5c02 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 30 May 2021 17:41:28 -0400 Subject: [PATCH 10/22] Catch getting logged out Happens when logging into Line on Chrome somewhere else --- matrix_puppeteer_line/rpc/client.py | 6 ++++++ matrix_puppeteer_line/user.py | 7 +++++++ puppet/src/client.js | 8 ++++++++ puppet/src/contentscript.js | 17 +++++++++++++++++ puppet/src/puppet.js | 25 ++++++++++++++++++++++++- 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/matrix_puppeteer_line/rpc/client.py b/matrix_puppeteer_line/rpc/client.py index 94129fa..2762bff 100644 --- a/matrix_puppeteer_line/rpc/client.py +++ b/matrix_puppeteer_line/rpc/client.py @@ -98,6 +98,12 @@ class Client(RPCClient): self.add_event_handler("receipt", wrapper) + async def on_logged_out(self, func: Callable[[], Awaitable[None]]) -> None: + async def wrapper(data: Dict[str, Any]) -> None: + await func() + + self.add_event_handler("logged_out", wrapper) + # TODO Type hint for sender async def login(self, sender, **login_data) -> AsyncGenerator[Tuple[str, str], None]: login_data["login_type"] = sender.command_status["login_type"] diff --git a/matrix_puppeteer_line/user.py b/matrix_puppeteer_line/user.py index d464063..c7344ca 100644 --- a/matrix_puppeteer_line/user.py +++ b/matrix_puppeteer_line/user.py @@ -101,6 +101,7 @@ class User(DBUser, BaseUser): state = await self.client.start() await self.client.on_message(self.handle_message) await self.client.on_receipt(self.handle_receipt) + await self.client.on_logged_out(self.handle_logged_out) if state.is_connected: self._track_metric(METRIC_CONNECTED, True) if state.is_logged_in: @@ -176,6 +177,12 @@ class User(DBUser, BaseUser): await portal.create_matrix_room(self, chat_info) await portal.handle_remote_receipt(receipt) + async def handle_logged_out(self) -> None: + await self.send_bridge_notice("Logged out of LINE. Please run the \"login\" command to log back in.") + if self._connection_check_task: + self._connection_check_task.cancel() + self._connection_check_task = None + def _add_to_cache(self) -> None: self.by_mxid[self.mxid] = self diff --git a/puppet/src/client.js b/puppet/src/client.js index 2c5e623..9fbd4cc 100644 --- a/puppet/src/client.js +++ b/puppet/src/client.js @@ -144,6 +144,14 @@ export default class Client { }) } + sendLoggedOut() { + this.log("Sending logout notice to client") + return this._write({ + id: --this.notificationID, + command: "logged_out", + }) + } + handleStart = async (req) => { let started = false if (this.puppet === null) { diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index 2fcefdf..570caae 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -69,6 +69,10 @@ window.__mautrixExpiry = function (button) {} * @return {Promise} */ window.__mautrixReceiveMessageID = function(id) {} +/** + * @return {void} + */ +window.__mautrixLoggedOut = function() {} /** * typedef ChatTypeEnum @@ -1287,3 +1291,16 @@ new MutationObserver(() => { attributeFilter: ["class"], childList: true, }) + +/** + * Watch for being logged out. + */ +const mainApp = document.querySelector("#mainApp") +new MutationObserver(() => { + if (mainApp.classList.contains("MdNonDisp")) { + window.__mautrixLoggedOut() + } +}).observe(mainApp, { + attributes: true, + attributeFilter: ["class"], +}) diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index b63a38a..805f1f8 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -107,6 +107,8 @@ export default class MessagesPuppeteer { this._receiveReceiptDirectLatest.bind(this)) await this.page.exposeFunction("__mautrixReceiveReceiptMulti", this._receiveReceiptMulti.bind(this)) + await this.page.exposeFunction("__mautrixLoggedOut", + this._onLoggedOut.bind(this)) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) // NOTE Must *always* re-login on a browser session, so no need to check if already logged in @@ -302,7 +304,17 @@ export default class MessagesPuppeteer { * @return {Promise} - Whether or not the session is logged in. */ async isLoggedIn() { - return await this.page.$("#wrap_message_sync") !== null + const selectors = [ + "#mainApp:not(.MdNonDisp)", + "#wrap_message_sync", + "#_chat_list_body", + ] + for (const selector of selectors) { + if (await this.page.$(selector) == null) { + return false + } + } + return true } async isPermanentlyDisconnected() { @@ -752,4 +764,15 @@ export default class MessagesPuppeteer { this.log("No client connected, not sending failure reason") } } + + _onLoggedOut() { + this.log("Got logged out!") + this.stopObserving() + if (this.client) { + this.client.sendLoggedOut().catch(err => + this.error("Failed to send logout notice to client:", err)) + } else { + this.log("No client connected, not sending logout notice") + } + } } From 14de373787353cbc9f8aa67b50b31c937592789c Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 30 May 2021 19:10:52 -0400 Subject: [PATCH 11/22] Startup fixes --- puppet/src/puppet.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 805f1f8..5ae78be 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -246,16 +246,20 @@ export default class MessagesPuppeteer { } catch (err) { //this._sendLoginFailure(`Failed to sync: ${err}`) this.log("LINE's sync took too long, assume it's fine and carry on...") + } finally { + const syncText = await this.page.evaluate( + messageSyncElement => messageSyncElement.innerText, + result) + this.log(`Final sync text is: "${syncText}"`) } this.loginRunning = false // Don't start observing yet, instead wait for explicit request. // But at least view the most recent chat. try { - let mostRecentChatID = await this.page.$eval("#_chat_list_body li", - element => window.getChatListItemID(element)) + const mostRecentChatID = await this.page.$eval("#_chat_list_body li", + element => window.__mautrixController.getChatListItemID(element.firstElementChild)) await this._switchChat(mostRecentChatID) - this.log("Focused on most recent chat") } catch (e) { this.log("No chats available to focus on") } From ec14b907113ec7e304ae64bd6bff72d0bc4c8cc5 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sun, 30 May 2021 19:11:39 -0400 Subject: [PATCH 12/22] Allow syncing pathless avatar images But only if a path wasn't yet found for that image --- matrix_puppeteer_line/portal.py | 23 +++++++++++++++++---- matrix_puppeteer_line/puppet.py | 32 ++++++++++++++++++++---------- matrix_puppeteer_line/rpc/types.py | 2 +- puppet/src/contentscript.js | 18 ++++++++++------- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 746084a..be357db 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -447,18 +447,33 @@ class Portal(DBPortal, BasePortal): return False async def _update_icon(self, icon: Optional[PathImage], client: Client) -> bool: - icon_path = icon.path if icon else None + if icon: + if icon.url and not icon.path: + self.log.warn(f"Using URL as path for room icon of {self.name}") + icon_path = icon_url = icon.url + else: + icon_path = icon.path + icon_url = icon.url + else: + icon_path = icon_url = None + if icon_path != self.icon_path: + self.log.info(f"Updating room icon of {self.name}") self.icon_path = icon_path - if icon and icon.url: + if icon_url: resp = await client.read_image(icon.url) self.icon_mxc = await self.main_intent.upload_media(resp.data, mime_type=resp.mime) else: self.icon_mxc = ContentURI("") if self.mxid: - await self.main_intent.set_room_avatar(self.mxid, self.icon_mxc) + try: + await self.main_intent.set_room_avatar(self.mxid, self.icon_mxc) + except Exception as e: + self.log.exception(f"Failed to set room icon: {e}") return True - return False + else: + self.log.debug(f"No need to update room icon of {self.name}, new icon has same path as old one") + return False async def _update_participants(self, participants: List[Participant]) -> None: if not self.mxid: diff --git a/matrix_puppeteer_line/puppet.py b/matrix_puppeteer_line/puppet.py index 97c7976..9a3c626 100644 --- a/matrix_puppeteer_line/puppet.py +++ b/matrix_puppeteer_line/puppet.py @@ -85,14 +85,24 @@ class Puppet(DBPuppet, BasePuppet): return False async def _update_avatar(self, avatar: Optional[PathImage], client: Client) -> bool: - if avatar and avatar.url and not avatar.path: - # Avatar exists, but in a form that cannot be uniquely identified. - # Skip it for now. - return False - avatar_path = avatar.path if avatar else None - if avatar_path != self.avatar_path or not self.avatar_set: + if avatar: + if avatar.url and not avatar.path: + if self.avatar_set and self.avatar_path: + self.log.warn(f"Not updating user avatar of {self.name}: new avatar exists, but in a form that cannot be uniquely identified") + return False + else: + self.log.warn(f"Using URL as path for user avatar of {self.name}: no previous avatar exists") + avatar_path = avatar_url = avatar.url + else: + avatar_path = avatar.path + avatar_url = avatar.url + else: + avatar_path = avatar_url = None + + if not self.avatar_set or avatar_path != self.avatar_path: + self.log.info(f"Updating user avatar of {self.name}") self.avatar_path = avatar_path - if avatar and avatar.url: + if avatar_url: resp = await client.read_image(avatar.url) self.avatar_mxc = await self.intent.upload_media(resp.data, mime_type=resp.mime) else: @@ -100,11 +110,13 @@ class Puppet(DBPuppet, BasePuppet): try: await self.intent.set_avatar_url(self.avatar_mxc) self.avatar_set = True - except Exception: - self.log.exception("Failed to set user avatar") + except Exception as e: + self.log.exception(f"Failed to set user avatar: {e}") self.avatar_set = False return True - return False + else: + self.log.debug(f"No need to update user avatar of {self.name}, new avatar has same path as old one") + return False def _add_to_cache(self) -> None: self.by_mid[self.mid] = self diff --git a/matrix_puppeteer_line/rpc/types.py b/matrix_puppeteer_line/rpc/types.py index 9fdcebf..9b96413 100644 --- a/matrix_puppeteer_line/rpc/types.py +++ b/matrix_puppeteer_line/rpc/types.py @@ -26,7 +26,7 @@ class RPCError(Exception): @dataclass class PathImage(SerializableAttrs['PathImage']): - path: str + path: Optional[str] url: str diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index 570caae..394af5a 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -166,8 +166,8 @@ class MautrixController { /** * Try to match a user against an entry in the friends list to get their ID. * - * @param {Element} element - The display name of the user to find the ID for. - * @return {?string} - The user's ID if found. + * @param {string} senderName - The display name of the user to find the ID for. + * @return {?string} - The user's ID if found. */ getUserIdFromFriendsList(senderName) { return document.querySelector(`#contact_wrap_friends > ul > li[title='${senderName}']`)?.getAttribute("data-mid") @@ -518,10 +518,14 @@ class MautrixController { /** * @typedef PathImage * @type object - * @property {string} path - The virtual path of the image (behaves like an ID) - * @property {string} src - The URL of the image + * @property {?string} path - The virtual path of the image (behaves like an ID). Optional. + * @property {string} src - The URL of the image. Mandatory. */ + /** + * @param {Element} img - The image element to get the URL and path of. + * @return {?PathImage} - The image URL and its path, if found. + */ _getPathImage(img) { if (img && img.src.startsWith("blob:")) { // NOTE Having a blob but no path means the image exists, @@ -539,9 +543,9 @@ class MautrixController { /** * @typedef Participant * @type object - * @property {string} id - The member ID for the participant - * @property {PathImage} avatar - The path and blob URL of the participant's avatar - * @property {string} name - The contact list name of the participant + * @property {string} id - The member ID for the participant + * @property {?PathImage} avatar - The path and blob URL of the participant's avatar + * @property {string} name - The contact list name of the participant */ getParticipantListItemName(element) { From 9d1d6e379c41f259eebf402136ffff062f639d9c Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 3 Jun 2021 01:13:00 -0400 Subject: [PATCH 13/22] Support LINE users with no discoverable ID AKA "strangers". Should only happen to non-friends in rooms (not groups!) --- matrix_puppeteer_line/db/__init__.py | 5 +- matrix_puppeteer_line/db/stranger.py | 93 ++++++++++++++++++++++++++++ matrix_puppeteer_line/db/upgrade.py | 24 +++++-- matrix_puppeteer_line/portal.py | 73 ++++++++++++---------- matrix_puppeteer_line/puppet.py | 35 ++++++++++- matrix_puppeteer_line/rpc/types.py | 4 +- matrix_puppeteer_line/user.py | 3 +- puppet/src/contentscript.js | 12 +++- 8 files changed, 204 insertions(+), 45 deletions(-) create mode 100644 matrix_puppeteer_line/db/stranger.py diff --git a/matrix_puppeteer_line/db/__init__.py b/matrix_puppeteer_line/db/__init__.py index 666c340..62784dc 100644 --- a/matrix_puppeteer_line/db/__init__.py +++ b/matrix_puppeteer_line/db/__init__.py @@ -3,6 +3,7 @@ from mautrix.util.async_db import Database from .upgrade import upgrade_table from .user import User from .puppet import Puppet +from .stranger import Stranger from .portal import Portal from .message import Message from .media import Media @@ -10,8 +11,8 @@ from .receipt_reaction import ReceiptReaction def init(db: Database) -> None: - for table in (User, Puppet, Portal, Message, Media, ReceiptReaction): + for table in (User, Puppet, Stranger, Portal, Message, Media, ReceiptReaction): table.db = db -__all__ = ["upgrade_table", "User", "Puppet", "Portal", "Message", "Media", "ReceiptReaction"] +__all__ = ["upgrade_table", "User", "Puppet", "Stranger", "Portal", "Message", "Media", "ReceiptReaction"] diff --git a/matrix_puppeteer_line/db/stranger.py b/matrix_puppeteer_line/db/stranger.py new file mode 100644 index 0000000..5de8a2e --- /dev/null +++ b/matrix_puppeteer_line/db/stranger.py @@ -0,0 +1,93 @@ +# matrix-puppeteer-line - A very hacky Matrix-LINE bridge based on running LINE's Chrome extension in Puppeteer +# Copyright (C) 2020-2021 Tulir Asokan, Andrew Ferrazzutti +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from typing import Optional, ClassVar, TYPE_CHECKING + +from attr import dataclass +from random import randint, seed + +from mautrix.util.async_db import Database + +fake_db = Database("") if TYPE_CHECKING else None + + +@dataclass +class Stranger: + db: ClassVar[Database] = fake_db + + # Optional properties are ones that should be set by Puppet + fake_mid: str + name: Optional[str] = None + avatar_path: Optional[str] = None + available: bool = False + + async def insert(self) -> None: + q = ("INSERT INTO stranger (fake_mid, name, avatar_path, available) " + "VALUES ($1, $2, $3, $4)") + await self.db.execute(q, self.fake_mid, self.name, self.avatar_path, self.available) + + async def update_profile_info(self) -> None: + q = ("UPDATE stranger SET name=$2, avatar_path=$3 " + "WHERE fake_mid=$1") + await self.db.execute(q, self.fake_mid, self.name, self.avatar_path) + + async def make_available(self) -> None: + q = ("UPDATE stranger SET available=true " + "WHERE name=$1 AND avatar_path=$2") + await self.db.execute(q, self.name, self.avatar_path) + + @classmethod + async def get_by_mid(cls, mid: str) -> Optional['Stranger']: + q = ("SELECT fake_mid, name, avatar_path, available " + "FROM stranger WHERE fake_mid=$1") + row = await cls.db.fetchrow(q, mid) + if not row: + return None + return cls(**row) + + @classmethod + async def get_by_profile(cls, info: 'Participant') -> Optional['Stranger']: + q = ("SELECT fake_mid, name, avatar_path, available " + "FROM stranger WHERE name=$1 AND avatar_path=$2") + row = await cls.db.fetchrow(q, info.name, info.avatar.path if info.avatar else "") + if not row: + return None + return cls(**row) + + @classmethod + async def get_any_available(cls) -> Optional['Stranger']: + q = ("SELECT fake_mid, name, avatar_path, available " + "FROM stranger WHERE available=true") + row = await cls.db.fetchrow(q) + if not row: + return None + return cls(**row) + + @classmethod + async def init_available_or_new(cls) -> 'Stranger': + stranger = await cls.get_any_available() + if not stranger: + while True: + fake_mid = "_STRANGER_" + for _ in range(32): + fake_mid += f"{randint(0,15):x}" + if await cls.get_by_mid(fake_mid) != None: + # Extremely unlikely event of a randomly-generated ID colliding with another. + # If it happens, must be not that unlikely after all, so pick a new seed. + seed() + else: + stranger = cls(fake_mid) + break + return stranger \ No newline at end of file diff --git a/matrix_puppeteer_line/db/upgrade.py b/matrix_puppeteer_line/db/upgrade.py index d863009..f3e96f9 100644 --- a/matrix_puppeteer_line/db/upgrade.py +++ b/matrix_puppeteer_line/db/upgrade.py @@ -109,7 +109,23 @@ async def upgrade_read_receipts(conn: Connection) -> None: @upgrade_table.register(description="Media metadata") async def upgrade_deduplicate_blob(conn: Connection) -> None: await conn.execute("""ALTER TABLE media - ADD COLUMN IF NOT EXISTS mime_type TEXT, - ADD COLUMN IF NOT EXISTS file_name TEXT, - ADD COLUMN IF NOT EXISTS size INTEGER - """) \ No newline at end of file + ADD COLUMN IF NOT EXISTS mime_type TEXT, + ADD COLUMN IF NOT EXISTS file_name TEXT, + ADD COLUMN IF NOT EXISTS size INTEGER + """) + + +@upgrade_table.register(description="Strangers") +async def upgrade_strangers(conn: Connection) -> None: + await conn.execute(""" + CREATE TABLE IF NOT EXISTS stranger ( + fake_mid TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + avatar_path TEXT NOT NULL, + available BOOLEAN NOT NULL DEFAULT false, + + PRIMARY KEY (name, avatar_path), + FOREIGN KEY (fake_mid) + REFERENCES puppet (mid) + ON DELETE CASCADE + )""") \ No newline at end of file diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index be357db..f03d86c 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -205,29 +205,34 @@ class Portal(DBPortal, BasePortal): intent = None return intent - async def handle_remote_message(self, source: 'u.User', sender: Optional['p.Puppet'], - evt: Message) -> None: + async def handle_remote_message(self, source: 'u.User', evt: Message) -> None: + if await DBMessage.get_by_mid(evt.id): + self.log.debug(f"Ignoring duplicate message {evt.id}") + return + if evt.is_outgoing: if source.intent: + sender = None intent = source.intent else: if not self.invite_own_puppet_to_pm: self.log.warning(f"Ignoring message {evt.id}: double puppeting isn't enabled") return + sender = p.Puppet.get_by_mid(evt.sender.id) if not self.is_direct else None intent = await self._bridge_own_message_pm(source, sender, f"message {evt.id}") if not intent: return - elif self.other_user: - intent = (await p.Puppet.get_by_mid(self.other_user)).intent - elif sender: - intent = sender.intent else: - self.log.warning(f"Ignoring message {evt.id}: sender puppet is unavailable") - return - - if await DBMessage.get_by_mid(evt.id): - self.log.debug(f"Ignoring duplicate message {evt.id}") - return + sender = await p.Puppet.get_by_mid(self.other_user if self.is_direct else evt.sender.id) + # TODO Respond to name/avatar changes of users in a DM + if not self.is_direct: + if sender: + await sender.update_info(evt.sender, source.client) + else: + self.log.warning(f"Could not find ID of LINE user who sent event {evt.id}") + sender = await p.Puppet.get_by_profile(evt.sender, source.client) + intent = sender.intent + intent.ensure_joined(self.mxid) if evt.image and evt.image.url: if not evt.image.is_sticker or self.config["bridge.receive_stickers"]: @@ -318,6 +323,9 @@ class Portal(DBPortal, BasePortal): format=Format.HTML if msg_html else None, body=msg_text, formatted_body=msg_html) event_id = await self._send_message(intent, content, timestamp=evt.timestamp) + # TODO Joins/leaves/invites/rejects, which are sent as LINE message events after all! + # Also keep track of strangers who leave / get blocked / become friends + # (maybe not here for all of that) else: content = TextMessageEventContent( msgtype=MessageType.NOTICE, @@ -349,11 +357,13 @@ class Portal(DBPortal, BasePortal): if reaction: await self.main_intent.redact(self.mxid, reaction.mxid) await reaction.delete() + # If there are as many receipts as there are chat participants, then everyone + # must have read the message, so send real read receipts from each puppet. # TODO Not just -1 if there are multiple _OWN_ puppets... if receipt_count == len(self._last_participant_update) - 1: - for participant in filter(lambda participant: not p.Puppet.is_mid_for_own_puppet(participant), self._last_participant_update): - puppet = await p.Puppet.get_by_mid(participant) - await puppet.intent.send_receipt(self.mxid, event_id) + for mid in filter(lambda mid: not p.Puppet.is_mid_for_own_puppet(mid), self._last_participant_update): + intent = (await p.Puppet.get_by_mid(mid)).intent + await intent.send_receipt(self.mxid, event_id) else: # TODO Translatable string for "Read by" reaction_mxid = await self.main_intent.react(self.mxid, event_id, f"(Read by {receipt_count})") @@ -418,8 +428,13 @@ class Portal(DBPortal, BasePortal): if self._main_intent is self.az.intent: self._main_intent = (await p.Puppet.get_by_mid(self.other_user)).intent for participant in conv.participants: - puppet = await p.Puppet.get_by_mid(participant.id) - await puppet.update_info(participant, client) + # REMINDER: multi-user chats include your own LINE user in the participant list + if participant.id != None: + puppet = await p.Puppet.get_by_mid(participant.id, client) + await puppet.update_info(participant, client) + else: + self.log.warning(f"Could not find ID of LINE user {participant.name}") + puppet = await p.Puppet.get_by_profile(participant, client) # TODO Consider setting no room name for non-group chats. # But then the LINE bot itself may appear in the title... changed = await self._update_name(f"{conv.name} (LINE)") @@ -480,7 +495,12 @@ class Portal(DBPortal, BasePortal): return # Store the current member list to prevent unnecessary updates - current_members = {participant.id for participant in participants} + current_members = set() + for participant in participants: + current_members.add( + participant.id if participant.id != None else \ + (await p.Puppet.get_by_profile(participant)).mid) + if current_members == self._last_participant_update: self.log.trace("Not updating participants: list matches cached list") return @@ -495,7 +515,7 @@ class Portal(DBPortal, BasePortal): for participant in participants: if forbid_own_puppets and p.Puppet.is_mid_for_own_puppet(participant.id): continue - intent = (await p.Puppet.get_by_mid(participant.id)).intent + intent = (await p.Puppet.get_by_sender(participant)).intent await intent.ensure_joined(self.mxid) print(current_members) @@ -536,15 +556,8 @@ class Portal(DBPortal, BasePortal): self.log.debug("Got %d messages from server", len(messages)) async with NotificationDisabler(self.mxid, source): - # Member joins/leaves are not shown in chat history. - # Best we can do is have a puppet join if its user had sent a message. - members_known = set(await self.main_intent.get_room_members(self.mxid)) if not self.is_direct else None for evt in messages: - puppet = await p.Puppet.get_by_mid(evt.sender.id) if not self.is_direct else None - if puppet and evt.sender.id not in members_known: - await puppet.update_info(evt.sender, source.client) - members_known.add(evt.sender.id) - await self.handle_remote_message(source, puppet, evt) + await self.handle_remote_message(source, evt) self.log.info("Backfilled %d messages through %s", len(messages), source.mxid) @property @@ -680,10 +693,8 @@ class Portal(DBPortal, BasePortal): self.by_mxid[self.mxid] = self await self.backfill(source) if not self.is_direct: - # For multi-user chats, backfill before updating participants, - # to act as as a best guess of when users actually joined. - # No way to tell when a user actually left, so just check the - # participants list after backfilling. + # TODO Joins and leaves are (usually) shown after all, so track them properly. + # In the meantime, just check the participants list after backfilling. await self._update_participants(info.participants) return self.mxid diff --git a/matrix_puppeteer_line/puppet.py b/matrix_puppeteer_line/puppet.py index 9a3c626..8bf73c0 100644 --- a/matrix_puppeteer_line/puppet.py +++ b/matrix_puppeteer_line/puppet.py @@ -19,7 +19,7 @@ from mautrix.bridge import BasePuppet from mautrix.types import UserID, ContentURI from mautrix.util.simple_template import SimpleTemplate -from .db import Puppet as DBPuppet +from .db import Puppet as DBPuppet, Stranger from .config import Config from .rpc import Participant, Client, PathImage from . import user as u @@ -141,6 +141,9 @@ class Puppet(DBPuppet, BasePuppet): @classmethod async def get_by_mid(cls, mid: str, create: bool = True) -> Optional['Puppet']: + if mid is None: + return None + # TODO Might need to parse a real id from "_OWN" try: return cls.by_mid[mid] @@ -160,10 +163,38 @@ class Puppet(DBPuppet, BasePuppet): return None + @classmethod + async def get_by_profile(cls, info: Participant, client: Optional[Client] = None) -> 'Puppet': + stranger = await Stranger.get_by_profile(info) + if not stranger: + stranger = await Stranger.init_available_or_new() + + puppet = cls(stranger.fake_mid) + # NOTE An update will insert anyways, so just do it now + await puppet.insert() + await puppet.update_info(info, client) + puppet._add_to_cache() + + # Get path from puppet in case it uses the URL as the path. + # But that should never happen in practice for strangers, + # which should only occur in rooms, where avatars have paths. + stranger.avatar_path = puppet.avatar_path + stranger.name = info.name + await stranger.insert() + # TODO Need a way to keep stranger name/avatar up to date, + # lest name/avatar changes get seen as another stranger. + # Also need to detect when a stranger becomes a friend. + return await cls.get_by_mid(stranger.fake_mid) + + @classmethod + async def get_by_sender(cls, info: Participant, client: Optional[Client] = None) -> 'Puppet': + puppet = await cls.get_by_mid(info.id) + return puppet if puppet else await cls.get_by_profile(info, client) + # TODO When supporting multiple bridge users, this should return the user whose puppet this is @classmethod def is_mid_for_own_puppet(cls, mid) -> bool: - return mid.startswith("_OWN_") if mid else False + return mid and mid.startswith("_OWN_") @classmethod async def get_by_custom_mxid(cls, mxid: UserID) -> Optional['u.User']: diff --git a/matrix_puppeteer_line/rpc/types.py b/matrix_puppeteer_line/rpc/types.py index 9b96413..af27117 100644 --- a/matrix_puppeteer_line/rpc/types.py +++ b/matrix_puppeteer_line/rpc/types.py @@ -41,9 +41,9 @@ class ChatListInfo(SerializableAttrs['ChatListInfo']): @dataclass class Participant(SerializableAttrs['Participant']): - id: str - avatar: Optional[PathImage] name: str + avatar: Optional[PathImage] + id: Optional[str] = None @dataclass diff --git a/matrix_puppeteer_line/user.py b/matrix_puppeteer_line/user.py index c7344ca..daa21a6 100644 --- a/matrix_puppeteer_line/user.py +++ b/matrix_puppeteer_line/user.py @@ -162,12 +162,11 @@ class User(DBUser, BaseUser): async def handle_message(self, evt: Message) -> None: self.log.trace("Received message %s", evt) portal = await po.Portal.get_by_chat_id(evt.chat_id, create=True) - puppet = await pu.Puppet.get_by_mid(evt.sender.id) if not portal.is_direct else None if not portal.mxid: await self.client.set_last_message_ids(await DBMessage.get_max_mids()) chat_info = await self.client.get_chat(evt.chat_id) await portal.create_matrix_room(self, chat_info) - await portal.handle_remote_message(self, puppet, evt) + await portal.handle_remote_message(self, evt) async def handle_receipt(self, receipt: Receipt) -> None: self.log.trace(f"Received receipt for chat {receipt.chat_id}") diff --git a/puppet/src/contentscript.js b/puppet/src/contentscript.js index 394af5a..226a7c4 100644 --- a/puppet/src/contentscript.js +++ b/puppet/src/contentscript.js @@ -238,10 +238,18 @@ class MautrixController { sender.id = this.getUserIdFromFriendsList(sender.name) // Group members aren't necessarily friends, // but the participant list includes their ID. + // ROOMS DO NOT!! Ugh. if (!sender.id) { const participantsList = document.querySelector(participantsListSelector) - imgElement = participantsList.querySelector(`img[alt='${sender.name}'`) - sender.id = imgElement.parentElement.parentElement.getAttribute("data-mid") + // Groups use a participant's name as the alt text of their avatar image, + // but rooms do not...ARGH! But they both use a dedicated element for it. + const participantNameElement = + Array.from(participantsList.querySelectorAll(`.mdRGT13Ttl`)) + .find(e => e.innerText == sender.name) + if (participantNameElement) { + imgElement = participantNameElement.previousElementSibling.firstElementChild + sender.id = imgElement?.parentElement.parentElement.getAttribute("data-mid") + } } else { imgElement = element.querySelector(".mdRGT07Img > img") } From 9270761b206518335e4537109e356610c87929a6 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Fri, 4 Jun 2021 19:29:44 +0900 Subject: [PATCH 14/22] Missing invite for bridgebot Most evident at room creation, but more might need to be added --- matrix_puppeteer_line/portal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index f03d86c..aac255c 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -635,6 +635,9 @@ class Portal(DBPortal, BasePortal): "content": self.bridge_info, }] invites = [source.mxid] + if self.needs_bridgebot: + invites.append(self.az.bot_mxid) + if self.config["bridge.encryption.default"] and self.matrix.e2ee: self.encrypted = True initial_state.append({ From 3286d7e6e2102609504730d8fa9af383f144211a Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Fri, 4 Jun 2021 19:48:11 +0900 Subject: [PATCH 15/22] Invite only on direct chat --- matrix_puppeteer_line/portal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index aac255c..6f180bc 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -635,8 +635,6 @@ class Portal(DBPortal, BasePortal): "content": self.bridge_info, }] invites = [source.mxid] - if self.needs_bridgebot: - invites.append(self.az.bot_mxid) if self.config["bridge.encryption.default"] and self.matrix.e2ee: self.encrypted = True @@ -644,6 +642,8 @@ class Portal(DBPortal, BasePortal): "type": str(EventType.ROOM_ENCRYPTION), "content": {"algorithm": "m.megolm.v1.aes-sha2"}, }) + if self.is_direct: + invites.append(self.az.bot_mxid) # NOTE Set the room title even for direct chats, because # the LINE bot itself may appear in the title otherwise. #if self.encrypted or not self.is_direct: From c5eea7b50b830fd7b488d254650d3b9e72662b9a Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 8 Jun 2021 00:24:00 -0400 Subject: [PATCH 16/22] Fix crash when logging in after a forced logout Also tidy up bridge notices during the login flow --- matrix_puppeteer_line/commands/auth.py | 13 ++++++++---- matrix_puppeteer_line/rpc/client.py | 12 ++++++++--- matrix_puppeteer_line/user.py | 2 +- puppet/src/client.js | 14 ++++++++++--- puppet/src/puppet.js | 28 +++++++++++++++++--------- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/matrix_puppeteer_line/commands/auth.py b/matrix_puppeteer_line/commands/auth.py index 6cf44a9..afd5331 100644 --- a/matrix_puppeteer_line/commands/auth.py +++ b/matrix_puppeteer_line/commands/auth.py @@ -55,6 +55,11 @@ async def login_do(evt: CommandEvent, gen: AsyncGenerator[Tuple[str, str], None] failure = False async for item in gen: if item[0] == "qr": + message = "Open LINE on your primary device and scan this QR code:" + content = TextMessageEventContent(body=message, msgtype=MessageType.NOTICE) + content.set_reply(evt.event_id) + await evt.az.intent.send_message(evt.room_id, content) + url = item[1] buffer = io.BytesIO() image = qrcode.make(url) @@ -69,7 +74,6 @@ async def login_do(evt: CommandEvent, gen: AsyncGenerator[Tuple[str, str], None] content.set_edit(qr_event_id) await evt.az.intent.send_message(evt.room_id, content) else: - content.set_reply(evt.event_id) qr_event_id = await evt.az.intent.send_message(evt.room_id, content) elif item[0] == "pin": pin = item[1] @@ -79,9 +83,10 @@ async def login_do(evt: CommandEvent, gen: AsyncGenerator[Tuple[str, str], None] content.set_edit(pin_event_id) await evt.az.intent.send_message(evt.room_id, content) else: - content.set_reply(evt.event_id) pin_event_id = await evt.az.intent.send_message(evt.room_id, content) - elif item[0] in ("failure", "error"): + elif item[0] == "login_success": + await evt.reply("Successfully logged in, waiting for LINE to load...") + elif item[0] in ("login_failure", "error"): # TODO Handle errors differently? failure = True reason = item[1] @@ -91,7 +96,7 @@ async def login_do(evt: CommandEvent, gen: AsyncGenerator[Tuple[str, str], None] # else: pass if not failure and evt.sender.command_status: - await evt.reply("Successfully logged in") + await evt.reply("LINE loading complete") await evt.sender.sync() # else command was cancelled or failed. Don't post message about it, "cancel" command or failure did already evt.sender.command_status = None diff --git a/matrix_puppeteer_line/rpc/client.py b/matrix_puppeteer_line/rpc/client.py index 2762bff..3e68865 100644 --- a/matrix_puppeteer_line/rpc/client.py +++ b/matrix_puppeteer_line/rpc/client.py @@ -119,8 +119,12 @@ class Client(RPCClient): data.append(("pin", req["pin"])) event.set() + async def success_handler(req: LoginCommand) -> None: + data.append(("login_success", None)) + event.set() + async def failure_handler(req: LoginCommand) -> None: - data.append(("failure", req.get("reason"))) + data.append(("login_failure", req.get("reason"))) event.set() async def cancel_watcher() -> None: @@ -145,7 +149,8 @@ class Client(RPCClient): self.add_event_handler("qr", qr_handler) self.add_event_handler("pin", pin_handler) - self.add_event_handler("failure", failure_handler) + self.add_event_handler("login_success", success_handler) + self.add_event_handler("login_failure", failure_handler) try: while True: await event.wait() @@ -158,4 +163,5 @@ class Client(RPCClient): finally: self.remove_event_handler("qr", qr_handler) self.remove_event_handler("pin", pin_handler) - self.remove_event_handler("failure", failure_handler) + self.remove_event_handler("login_success", success_handler) + self.remove_event_handler("login_failure", failure_handler) diff --git a/matrix_puppeteer_line/user.py b/matrix_puppeteer_line/user.py index daa21a6..1372ac1 100644 --- a/matrix_puppeteer_line/user.py +++ b/matrix_puppeteer_line/user.py @@ -177,7 +177,7 @@ class User(DBUser, BaseUser): await portal.handle_remote_receipt(receipt) async def handle_logged_out(self) -> None: - await self.send_bridge_notice("Logged out of LINE. Please run the \"login\" command to log back in.") + await self.send_bridge_notice("Logged out of LINE. Please run either \"login-qr\" or \"login-email\" to log back in.") if self._connection_check_task: self._connection_check_task.cancel() self._connection_check_task = None diff --git a/puppet/src/client.js b/puppet/src/client.js index 9fbd4cc..636d695 100644 --- a/puppet/src/client.js +++ b/puppet/src/client.js @@ -135,11 +135,19 @@ export default class Client { }) } - sendFailure(reason) { - this.log(`Sending failure to client${reason ? `: "${reason}"` : ""}`) + sendLoginSuccess() { + this.log("Sending login success to client") return this._write({ id: --this.notificationID, - command: "failure", + command: "login_success", + }) + } + + sendLoginFailure(reason) { + this.log(`Sending login failure to client${reason ? `: "${reason}"` : ""}`) + return this._write({ + id: --this.notificationID, + command: "login_failure", reason, }) } diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 5ae78be..caf0d32 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -207,7 +207,7 @@ export default class MessagesPuppeteer { } const result = await Promise.race([ - () => this.page.waitForSelector("#wrap_message_sync", {timeout: 2000}) + () => this.page.waitForSelector("#mainApp:not(.MdNonDisp)", {timeout: 2000}) .then(value => { loginSuccess = true return value @@ -227,11 +227,13 @@ export default class MessagesPuppeteer { delete this.login_email delete this.login_password - if (!loginSuccess) { + const messageSyncElement = loginSuccess ? await this.page.waitForSelector("#wrap_message_sync") : null + if (!loginSuccess || !messageSyncElement) { this._sendLoginFailure(result) return } + this._sendLoginSuccess() this.log("Waiting for sync") try { await this.page.waitForFunction( @@ -242,14 +244,12 @@ export default class MessagesPuppeteer { // TODO Sometimes it gets stuck at 99%...?? }, {timeout: 10000}, // Assume 10 seconds is long enough - result) + messageSyncElement) } catch (err) { //this._sendLoginFailure(`Failed to sync: ${err}`) this.log("LINE's sync took too long, assume it's fine and carry on...") } finally { - const syncText = await this.page.evaluate( - messageSyncElement => messageSyncElement.innerText, - result) + const syncText = await messageSyncElement.evaluate(e => e.innerText) this.log(`Final sync text is: "${syncText}"`) } @@ -758,14 +758,24 @@ export default class MessagesPuppeteer { } } + _sendLoginSuccess() { + this.error("Login success") + if (this.client) { + this.client.sendLoginSuccess().catch(err => + this.error("Failed to send login success to client:", err)) + } else { + this.log("No client connected, not sending login success") + } + } + _sendLoginFailure(reason) { this.loginRunning = false this.error(`Login failure: ${reason ? reason : "cancelled"}`) if (this.client) { - this.client.sendFailure(reason).catch(err => - this.error("Failed to send failure reason to client:", err)) + this.client.sendLoginFailure(reason).catch(err => + this.error("Failed to send login failure to client:", err)) } else { - this.log("No client connected, not sending failure reason") + this.log("No client connected, not sending login failure") } } From 1fffbc625ca71e4828145a2c2577417f699a5ac3 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 8 Jun 2021 01:08:18 -0400 Subject: [PATCH 17/22] Update roadmap --- README.md | 2 +- ROADMAP.md | 64 +++++++++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d1ee4be..f58912d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A very hacky Matrix-LINE bridge based on running LINE's Chrome extension in Puppeteer. Fork of [https://mau.dev/tulir/mautrix-amp/](mautrix-amp). -## Features & roadmap +## Features, roadmap, and limitations [ROADMAP.md](ROADMAP.md) ## Setup diff --git a/ROADMAP.md b/ROADMAP.md index 1dfe532..fa1f41a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,11 +6,13 @@ * [x] Images * [ ] Files * [x] Stickers - * [x] Notification for message send failure * [ ] Read receipts (currently eagerly-sent since message sync requires "reading" a chat) * [ ] Room metadata changes * [ ] Name * [ ] Avatar + * [ ] Member events + * [ ] Invite + * [ ] Kick * LINE → Matrix * [ ] Message content * [x] Text @@ -20,45 +22,54 @@ * [ ] Location * [ ] Videos * [x] Stickers - * [x] Sticons - * [x] Single - * [x] Multiple or mixed with text - * [x] EmojiOne + * [x] Emoji * [ ] Message unsend * [ ] Read receipts * [x] For most recently active chat * [ ] For any chat * [x] User metadata * [ ] Name - * [x] On initial sync + * [x] On sync * [ ] On change * [ ] Avatar - * [x] On initial sync + * [x] On sync * [ ] On change * [ ] Chat metadata * [ ] Name - * [x] On initial sync + * [x] On sync * [ ] On change * [ ] Icon - * [x] On initial sync + * [x] On sync * [ ] On change - * [x] Message history + * [ ] Message history * [x] When creating portal * [x] Missed messages * [x] Message timestamps + * [ ] As many messages that are visible in LINE extension * [x] Chat types * [x] Direct chats * [x] Groups (named chats) * [x] Rooms (unnamed chats / "multi-user direct chats") * [ ] Membership actions - * [x] Add member - * [ ] Remove member - * [ ] Block + * [ ] Join + * [x] When message is sent by new participant + * [x] On sync + * [ ] At join time + * [ ] Leave + * [x] On sync + * [ ] At leave time + * [ ] Invite + * [ ] Remove + * [ ] Friend actions + * [ ] Add friend + * [ ] Block user + * [ ] Unblock user * Misc * [x] Automatic portal creation * [x] At startup * [x] When receiving invite or message * [ ] When sending message in new chat from LINE app + * [x] Notification for message send failure * [ ] Provisioning API for logging in * [x] Use bridge bot for messages sent from LINE app (when double-puppeting is disabled and `bridge.invite_own_puppet_to_pm` is enabled) * [x] Use own Matrix account for messages sent from LINE app (when double-puppeting is enabled) @@ -66,8 +77,8 @@ * [ ] Multiple bridge users * [ ] Relay bridging -## Missing features -### Missing from LINE +# Missing features +## Missing from LINE * Typing notifications * Message edits * Formatted messages @@ -75,13 +86,22 @@ * Timestamped read receipts * Read receipts between users other than yourself -### Missing from LINE on Chrome -* Message redaction (delete/unsend) -* Replies -* Audio message sending -* Location sending -* Voice/video calls +## Missing from LINE on Chrome * Unlimited message history + * Messages that are very old may not be available in LINE on Chrome at all, even after a full sync +* Voice/video calls + * No notification is sent when a call begins + * When a call ends, an automated message of "Your OS version doesn't support this feature" is sent as an ordinary text message from the user who began the call +* Message redaction (delete/unsend) + * But messages unsent from other LINE clients do disappear from LINE on Chrome +* Replies + * Appear as ordinary messages +* Mentions + * Appear as ordinary text +* Audio message sending + * But audio messages can be received +* Location sending + * But locations can be received -### Missing from matrix-puppeteer-line +## Missing from matrix-puppeteer-line * TODO From d8b209b04f115fbd6d448901640ae9c1278af845 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 10 Jun 2021 02:19:12 -0400 Subject: [PATCH 18/22] Send delivery receipts for double-puppeted messages --- matrix_puppeteer_line/portal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 6f180bc..8ac00d3 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -132,6 +132,7 @@ class Portal(DBPortal, BasePortal): elif ((message.get(self.bridge.real_user_content_key, False) and await p.Puppet.get_by_custom_mxid(sender.mxid))): self.log.debug(f"Ignoring puppet-sent message by confirmed puppet user {sender.mxid}") + await self._send_delivery_receipt(event_id) return # TODO deduplication of outgoing messages text = message.body From 02cc6ce0e4f9adce58b57f2a9bfd22acfc37d347 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 10 Jun 2021 02:19:24 -0400 Subject: [PATCH 19/22] Add missing await --- matrix_puppeteer_line/portal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index 8ac00d3..d7c3412 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -233,7 +233,7 @@ class Portal(DBPortal, BasePortal): self.log.warning(f"Could not find ID of LINE user who sent event {evt.id}") sender = await p.Puppet.get_by_profile(evt.sender, source.client) intent = sender.intent - intent.ensure_joined(self.mxid) + await intent.ensure_joined(self.mxid) if evt.image and evt.image.url: if not evt.image.is_sticker or self.config["bridge.receive_stickers"]: From 1cf840e3d9850df21397b6a8273ee1595f879e87 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 10 Jun 2021 02:51:23 -0400 Subject: [PATCH 20/22] maunium -> miscworks --- matrix_puppeteer_line/example-config.yaml | 2 +- matrix_puppeteer_line/portal.py | 2 +- matrix_puppeteer_line/web/provisioning_api.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/matrix_puppeteer_line/example-config.yaml b/matrix_puppeteer_line/example-config.yaml index b519223..2f241f9 100644 --- a/matrix_puppeteer_line/example-config.yaml +++ b/matrix_puppeteer_line/example-config.yaml @@ -45,7 +45,7 @@ appservice: # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty # to leave display name/avatar as-is. bot_displayname: LINE bridge bot - bot_avatar: mxc://maunium.net/VuvevQiMRlOxuBVMBNEZZrxi + bot_avatar: mxc://miscworks.net/uyHgFSLDDtATVGjtjWKwxrkK # Community ID for bridged users (changes registration file) and rooms. # Must be created manually. diff --git a/matrix_puppeteer_line/portal.py b/matrix_puppeteer_line/portal.py index d7c3412..5d42ef3 100644 --- a/matrix_puppeteer_line/portal.py +++ b/matrix_puppeteer_line/portal.py @@ -563,7 +563,7 @@ class Portal(DBPortal, BasePortal): @property def bridge_info_state_key(self) -> str: - return f"net.maunium.line://line/{self.chat_id}" + return f"net.miscworks.line://line/{self.chat_id}" @property def bridge_info(self) -> Dict[str, Any]: diff --git a/matrix_puppeteer_line/web/provisioning_api.py b/matrix_puppeteer_line/web/provisioning_api.py index 82a9a81..4bbf5ef 100644 --- a/matrix_puppeteer_line/web/provisioning_api.py +++ b/matrix_puppeteer_line/web/provisioning_api.py @@ -64,8 +64,8 @@ class ProvisioningAPI: return None for part in auth_parts: part = part.strip() - if part.startswith("net.maunium.line.auth-"): - return part[len("net.maunium.line.auth-"):] + if part.startswith("net.miscworks.line.auth-"): + return part[len("net.miscworks.line.auth-"):] return None def check_token(self, request: web.Request) -> Awaitable['u.User']: @@ -107,7 +107,7 @@ class ProvisioningAPI: if status.is_logged_in: raise web.HTTPConflict(text='{"error": "Already logged in"}', headers=self._headers) - ws = web.WebSocketResponse(protocols=["net.maunium.line.login"]) + ws = web.WebSocketResponse(protocols=["net.miscworks.line.login"]) await ws.prepare(request) try: async for url in user.client.login(): From e37a7c41bfcaad5d6ec41c0ce607b4762eb2b0bf Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 10 Jun 2021 02:52:13 -0400 Subject: [PATCH 21/22] Minor formatting changes --- puppet/src/puppet.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index caf0d32..5750dd1 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -500,8 +500,7 @@ export default class MessagesPuppeteer { } async _getChatInfoUnsafe(chatID) { - const chatListItem = await this.page.$(this._listItemSelector(chatID)) - const chatListInfo = await chatListItem.evaluate( + const chatListInfo = await this.page.$eval(this._listItemSelector(chatID), (element, chatID) => window.__mautrixController.parseChatListItem(element, chatID), chatID) @@ -520,12 +519,12 @@ export default class MessagesPuppeteer { let participants if (!isDirect) { - this.log("Found multi-user chat, so clicking chat header to get participants") + this.log("Found multi-user chat, so viewing it to get participants") // TODO This will mark the chat as "read"! await this._switchChat(chatID) const participantList = await this.page.$("#_chat_detail_area > .mdRGT02Info ul.mdRGT13Ul") // TODO Is a group not actually created until a message is sent(?) - // If so, maybe don't create a portal until there is a message. + // If so, maybe don't create a portal until there is a message. participants = await participantList.evaluate( element => window.__mautrixController.parseParticipantList(element)) } else { @@ -547,9 +546,9 @@ export default class MessagesPuppeteer { } // TODO Catch "An error has occurred" dialog - // Selector is just "dialog", then "button" - // Child of "#layer_contents" - // Always present, just made visible via classes + // Selector is just "dialog", then "button" + // Child of "#layer_contents" + // Always present, just made visible via classes async _sendMessageUnsafe(chatID, text) { await this._switchChat(chatID) From 0d849cf2bb93b0e7acf5bad02f0d9641a7c98ecb Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 10 Jun 2021 02:52:25 -0400 Subject: [PATCH 22/22] JSON stringify objects before logging them Because Eclipse/Wild Web Developer messes them up --- puppet/src/puppet.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index 5750dd1..44a3f03 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -409,7 +409,7 @@ export default class MessagesPuppeteer { this.mostRecentMessages.set(chatID, messageID) } this.log("Updated most recent message ID map:") - this.log(this.mostRecentMessages) + this.log(JSON.stringify(this.mostRecentMessages)) } async readImage(imageUrl) { @@ -540,7 +540,7 @@ export default class MessagesPuppeteer { this.log("Found participants:") for (const participant of participants) { - this.log(participant) + this.log(JSON.stringify(participant)) } return {participants, ...chatListInfo} }