From e7b78f91663bb7bd02c08da9cc86df87697b5138 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 3 Nov 2020 17:57:51 +0200 Subject: [PATCH] Update mautrix-python and add hacky connection metric --- mautrix_amp/__main__.py | 5 +- mautrix_amp/commands/__init__.py | 4 +- mautrix_amp/commands/auth.py | 5 +- mautrix_amp/commands/conn.py | 6 +- mautrix_amp/commands/handler.py | 97 -------------------------------- mautrix_amp/commands/typehint.py | 12 ++++ mautrix_amp/config.py | 3 + mautrix_amp/example-config.yaml | 5 ++ mautrix_amp/matrix.py | 6 +- mautrix_amp/puppet.py | 2 +- mautrix_amp/rpc/client.py | 4 ++ mautrix_amp/user.py | 22 +++++++- optional-requirements.txt | 3 + puppet/src/client.js | 1 + puppet/src/puppet.js | 25 +++++++- requirements.txt | 2 +- 16 files changed, 89 insertions(+), 113 deletions(-) delete mode 100644 mautrix_amp/commands/handler.py create mode 100644 mautrix_amp/commands/typehint.py diff --git a/mautrix_amp/__main__.py b/mautrix_amp/__main__.py index a93a147..770c343 100644 --- a/mautrix_amp/__main__.py +++ b/mautrix_amp/__main__.py @@ -26,6 +26,7 @@ from .user import User from .portal import Portal from .puppet import Puppet from .web import ProvisioningAPI +from . import commands as _ class MessagesBridge(Bridge): @@ -86,8 +87,8 @@ class MessagesBridge(Bridge): await portal.update_bridge_info() self.log.info("Finished re-sending bridge info state events") - async def get_user(self, user_id: UserID) -> User: - return await User.get_by_mxid(user_id) + async def get_user(self, user_id: UserID, create: bool = True) -> User: + return await User.get_by_mxid(user_id, create=create) async def get_portal(self, room_id: RoomID) -> Portal: return await Portal.get_by_mxid(room_id) diff --git a/mautrix_amp/commands/__init__.py b/mautrix_amp/commands/__init__.py index 69e3a3c..88a4bf0 100644 --- a/mautrix_amp/commands/__init__.py +++ b/mautrix_amp/commands/__init__.py @@ -1,2 +1,2 @@ -from .handler import (CommandProcessor, command_handler, CommandEvent, CommandHandler, SECTION_AUTH, SECTION_CONNECTION) -from . import auth, conn +from .auth import SECTION_AUTH +from .conn import SECTION_CONNECTION diff --git a/mautrix_amp/commands/auth.py b/mautrix_amp/commands/auth.py index e4d26c7..7a55257 100644 --- a/mautrix_amp/commands/auth.py +++ b/mautrix_amp/commands/auth.py @@ -20,8 +20,11 @@ import qrcode import PIL as _ from mautrix.types import MediaMessageEventContent, MessageType, ImageInfo, EventID +from mautrix.bridge.commands import HelpSection, command_handler -from . import command_handler, CommandEvent, SECTION_CONNECTION, SECTION_AUTH +from .typehint import CommandEvent + +SECTION_AUTH = HelpSection("Authentication", 10, "") @command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH, diff --git a/mautrix_amp/commands/conn.py b/mautrix_amp/commands/conn.py index 8747ff1..bdc79f6 100644 --- a/mautrix_amp/commands/conn.py +++ b/mautrix_amp/commands/conn.py @@ -13,7 +13,11 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from . import command_handler, CommandEvent, SECTION_CONNECTION +from mautrix.bridge.commands import HelpSection, command_handler + +from .typehint import CommandEvent + +SECTION_CONNECTION = HelpSection("Connection management", 15, "") @command_handler(needs_auth=False, management_only=True, help_section=SECTION_CONNECTION, diff --git a/mautrix_amp/commands/handler.py b/mautrix_amp/commands/handler.py deleted file mode 100644 index 42aa85a..0000000 --- a/mautrix_amp/commands/handler.py +++ /dev/null @@ -1,97 +0,0 @@ -# mautrix-amp - A very hacky Matrix-SMS bridge based on using Android Messages for Web in Puppeteer -# Copyright (C) 2020 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from typing import Awaitable, Callable, List, Optional, NamedTuple, TYPE_CHECKING - -from mautrix.types import RoomID, EventID, MessageEventContent -from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent, - CommandHandler as BaseCommandHandler, - CommandProcessor as BaseCommandProcessor, - CommandHandlerFunc, command_handler as base_command_handler) - -from .. import user as u - -if TYPE_CHECKING: - from ..__main__ import MessagesBridge - -HelpCacheKey = NamedTuple('HelpCacheKey', is_management=bool, is_portal=bool, is_admin=bool, - is_logged_in=bool) -SECTION_CONNECTION = HelpSection("Connection management", 15, "") -SECTION_AUTH = HelpSection("Authentication", 10, "") - - -class CommandEvent(BaseCommandEvent): - sender: u.User - - def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID, - sender: u.User, command: str, args: List[str], content: MessageEventContent, - is_management: bool, is_portal: bool) -> None: - super().__init__(processor, room_id, event_id, sender, command, args, content, - is_management, is_portal) - self.bridge = processor.bridge - self.config = processor.config - - @property - def print_error_traceback(self) -> bool: - return self.sender.is_admin - - async def get_help_key(self) -> HelpCacheKey: - return HelpCacheKey(self.is_management, self.is_portal, self.sender.is_admin, - await self.sender.is_logged_in()) - - -class CommandHandler(BaseCommandHandler): - name: str - - management_only: bool - needs_auth: bool - needs_admin: bool - - def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]], - management_only: bool, name: str, help_text: str, help_args: str, - help_section: HelpSection, needs_auth: bool, needs_admin: bool) -> None: - super().__init__(handler, management_only, name, help_text, help_args, help_section, - needs_auth=needs_auth, needs_admin=needs_admin) - - async def get_permission_error(self, evt: CommandEvent) -> Optional[str]: - if self.management_only and not evt.is_management: - return (f"`{evt.command}` is a restricted command: " - "you may only run it in management rooms.") - elif self.needs_admin and not evt.sender.is_admin: - return "This command requires administrator privileges." - elif self.needs_auth and not await evt.sender.is_logged_in(): - return "This command requires you to be logged in." - return None - - def has_permission(self, key: HelpCacheKey) -> bool: - return ((not self.management_only or key.is_management) and - (not self.needs_admin or key.is_admin) and - (not self.needs_auth or key.is_logged_in)) - - -def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True, - needs_admin: bool = False, management_only: bool = False, - name: Optional[str] = None, help_text: str = "", help_args: str = "", - help_section: HelpSection = None) -> Callable[[CommandHandlerFunc], - CommandHandler]: - return base_command_handler(_func, _handler_class=CommandHandler, name=name, - help_text=help_text, help_args=help_args, - help_section=help_section, management_only=management_only, - needs_auth=needs_auth, needs_admin=needs_admin) - - -class CommandProcessor(BaseCommandProcessor): - def __init__(self, bridge: 'MessagesBridge') -> None: - super().__init__(bridge=bridge, event_class=CommandEvent) diff --git a/mautrix_amp/commands/typehint.py b/mautrix_amp/commands/typehint.py new file mode 100644 index 0000000..a01c7c2 --- /dev/null +++ b/mautrix_amp/commands/typehint.py @@ -0,0 +1,12 @@ +from typing import TYPE_CHECKING + +from mautrix.bridge.commands import CommandEvent as BaseCommandEvent + +if TYPE_CHECKING: + from ..__main__ import MessagesBridge + from ..user import User + + +class CommandEvent(BaseCommandEvent): + bridge: 'MessagesBridge' + sender: 'User' diff --git a/mautrix_amp/config.py b/mautrix_amp/config.py index afb1068..1b32ee6 100644 --- a/mautrix_amp/config.py +++ b/mautrix_amp/config.py @@ -51,6 +51,9 @@ class Config(BaseBridgeConfig): copy("appservice.community_id") + copy("metrics.enabled") + copy("metrics.listen_port") + copy("bridge.username_template") copy("bridge.displayname_template") diff --git a/mautrix_amp/example-config.yaml b/mautrix_amp/example-config.yaml index 1e3a4ac..57bbf13 100644 --- a/mautrix_amp/example-config.yaml +++ b/mautrix_amp/example-config.yaml @@ -57,6 +57,11 @@ appservice: as_token: "This value is generated when generating the registration" hs_token: "This value is generated when generating the registration" +# Prometheus telemetry config. Requires prometheus-client to be installed. +metrics: + enabled: false + listen_port: 8000 + # Bridge config bridge: # Localpart template of MXIDs for remote users. diff --git a/mautrix_amp/matrix.py b/mautrix_amp/matrix.py index f44327a..4c5e8ea 100644 --- a/mautrix_amp/matrix.py +++ b/mautrix_amp/matrix.py @@ -19,22 +19,20 @@ from mautrix.bridge import BaseMatrixHandler from mautrix.types import (Event, ReactionEvent, MessageEvent, StateEvent, EncryptedEvent, RoomID, RedactionEvent) -from . import commands as com, puppet as pu, user as u +from . import puppet as pu, user as u if TYPE_CHECKING: from .__main__ import MessagesBridge class MatrixHandler(BaseMatrixHandler): - commands: 'com.CommandProcessor' - def __init__(self, bridge: 'MessagesBridge') -> None: prefix, suffix = bridge.config["bridge.username_template"].format(userid=":").split(":") homeserver = bridge.config["homeserver.domain"] self.user_id_prefix = f"@{prefix}" self.user_id_suffix = f"{suffix}:{homeserver}" - super().__init__(command_processor=com.CommandProcessor(bridge), bridge=bridge) + super().__init__(bridge=bridge) def filter_matrix_event(self, evt: Event) -> bool: if not isinstance(evt, (ReactionEvent, MessageEvent, StateEvent, EncryptedEvent, diff --git a/mautrix_amp/puppet.py b/mautrix_amp/puppet.py index 729b9dc..da47cbe 100644 --- a/mautrix_amp/puppet.py +++ b/mautrix_amp/puppet.py @@ -56,7 +56,7 @@ class Puppet(DBPuppet, BasePuppet): cls.mxid_template = SimpleTemplate(cls.config["bridge.username_template"], "userid", prefix="@", suffix=f":{cls.hs_domain}", type=str) secret = cls.config["bridge.login_shared_secret"] - cls.login_shared_secret = secret.encode("utf-8") if secret else None + cls.login_shared_secret_map[cls.hs_domain] = secret.encode("utf-8") if secret else None cls.login_device_name = "Android Messages Bridge" async def update_info(self, info: Participant) -> None: diff --git a/mautrix_amp/rpc/client.py b/mautrix_amp/rpc/client.py index 29a4335..12b243e 100644 --- a/mautrix_amp/rpc/client.py +++ b/mautrix_amp/rpc/client.py @@ -49,6 +49,10 @@ class Client(RPCClient): resp = await self.request("get_messages", chat_id=chat_id) return [Message.deserialize(data) for data in resp] + async def is_connected(self) -> bool: + resp = await self.request("is_connected") + return resp["is_connected"] + async def send(self, chat_id: int, text: str) -> int: resp = await self.request("send", chat_id=chat_id, text=text) return resp["id"] diff --git a/mautrix_amp/user.py b/mautrix_amp/user.py index cdf74c1..b84e7fc 100644 --- a/mautrix_amp/user.py +++ b/mautrix_amp/user.py @@ -13,12 +13,14 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, List, Optional, AsyncGenerator, TYPE_CHECKING, cast +from typing import Dict, List, Optional, TYPE_CHECKING, cast +from collections import defaultdict import asyncio from mautrix.bridge import BaseUser from mautrix.types import UserID, RoomID from mautrix.appservice import AppService, IntentAPI +from mautrix.util.opt_prometheus import Gauge from .db import User as DBUser, Portal as DBPortal, Message as DBMessage from .config import Config @@ -28,6 +30,8 @@ from . import puppet as pu, portal as po if TYPE_CHECKING: from .__main__ import MessagesBridge +METRIC_CONNECTED = Gauge("bridge_connected", "Users connected to Android Messages") + class User(DBUser, BaseUser): by_mxid: Dict[UserID, 'User'] = {} @@ -40,12 +44,15 @@ class User(DBUser, BaseUser): is_real_user = True _notice_room_lock: asyncio.Lock + _connection_check_task: Optional[asyncio.Task] def __init__(self, mxid: UserID, notice_room: Optional[RoomID] = None) -> None: super().__init__(mxid=mxid, notice_room=notice_room) self._notice_room_lock = asyncio.Lock() self.is_whitelisted = self.is_admin = self.config["bridge.user"] == mxid self.log = self.log.getChild(self.mxid) + self._metric_value = defaultdict(lambda: False) + self._connection_check_task = None self.client = None self.username = None self.intent = None @@ -87,6 +94,8 @@ class User(DBUser, BaseUser): self.log.debug("Starting client") state = await self.client.start() await self.client.on_message(self.handle_message) + if state.is_connected: + self._track_metric(METRIC_CONNECTED, True) if state.is_logged_in: self.loop.create_task(self._try_sync()) @@ -96,7 +105,15 @@ class User(DBUser, BaseUser): except Exception: self.log.exception("Exception while syncing") + async def _check_connection_loop(self) -> None: + while True: + self._track_metric(METRIC_CONNECTED, await self.client.is_connected()) + await asyncio.sleep(5) + async def sync(self) -> None: + if self._connection_check_task: + self._connection_check_task.cancel() + self._connection_check_task = self.loop.create_task(self._check_connection_loop()) await self.client.set_last_message_ids(await DBMessage.get_max_mids()) self.log.info("Syncing chats") chats = await self.client.get_chats() @@ -112,6 +129,9 @@ class User(DBUser, BaseUser): await portal.create_matrix_room(self, chat) async def stop(self) -> None: + if self._connection_check_task: + self._connection_check_task.cancel() + self._connection_check_task = None if self.client: await self.client.stop() diff --git a/optional-requirements.txt b/optional-requirements.txt index 57320ba..47c98d2 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -5,3 +5,6 @@ python-olm>=3,<4 pycryptodome>=3,<4 unpaddedbase64>=1,<2 + +#/metrics +prometheus_client>=0.6,<0.9 diff --git a/puppet/src/client.js b/puppet/src/client.js index 0ddc67f..978f0a6 100644 --- a/puppet/src/client.js +++ b/puppet/src/client.js @@ -211,6 +211,7 @@ export default class Client { get_chats: () => this.puppet.getRecentChats(), get_chat: req => this.puppet.getChatInfo(req.chat_id), get_messages: req => this.puppet.getMessages(req.chat_id), + is_connected: async () => ({ is_connected: !await this.puppet.isDisconnected() }), }[req.command] || this.handleUnknownCommand } const resp = { id: req.id } diff --git a/puppet/src/puppet.js b/puppet/src/puppet.js index e138ade..87765a6 100644 --- a/puppet/src/puppet.js +++ b/puppet/src/puppet.js @@ -85,7 +85,7 @@ export default class MessagesPuppeteer { this.log("Exposing functions") await this.page.exposeFunction("__mautrixReceiveQR", this._receiveQRChange.bind(this)) await this.page.exposeFunction("__mautrixReceiveMessageID", - id => this.sentMessageIDs.add(id)) + id => this.sentMessageIDs.add(id)) await this.page.exposeFunction("__mautrixReceiveChanges", this._receiveChatListChanges.bind(this)) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) @@ -162,9 +162,28 @@ export default class MessagesPuppeteer { return await this.page.$("mw-unable-to-connect-container") !== null } + async isOpenSomewhereElse() { + try { + const text = await this.page.$eval("mws-dialog mat-dialog-content div", + elem => elem.textContent) + return text?.trim() === "Messages for web is open in more than one tab or browser" + } catch (err) { + return false + } + } + + async clickDialogButton() { + await this.page.click("mws-dialog mat-dialog-actions button") + } + async isDisconnected() { - // TODO we should observe this banner appearing and disappearing to notify about disconnects - return await this.page.$("mw-main-container mw-error-banner") !== null + const offlineIndicators = await Promise.all([ + this.page.$("mw-main-nav mw-banner mw-error-banner"), + this.page.$("mw-main-nav mw-banner mw-information-banner[title='Connecting']"), + this.page.$("mw-unable-to-connect-container"), + this.isOpenSomewhereElse(), + ]) + return offlineIndicators.some(indicator => Boolean(indicator)) } /** diff --git a/requirements.txt b/requirements.txt index f88d3d5..7991500 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ commonmark>=0.8,<0.10 aiohttp>=3,<4 yarl>=1,<2 attrs>=19.1 -mautrix>=0.7.13,<0.8 +mautrix==0.8.0rc4 asyncpg>=0.20,<0.22 pillow>=4,<8 qrcode>=6,<7