fair/matrix-puppeteer-line
fair
/
matrix-puppeteer-line
3
0
Fork 3
Code Issues 26 Pull Requests 2 Projects Releases Wiki Activity

Working avatars and icons

This commit is contained in:
Andrew Ferrazzutti 2021-03-26 02:27:21 -04:00
parent b007751610
commit 884d0d32fe
12 changed files with 204 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
matrix_puppeteer_line/db/portal.py
View File

@ -17,7 +17,7 @@ from typing import Optional, ClassVar, List, TYPE_CHECKING
from attr import dataclass
from mautrix.types import RoomID
from mautrix.types import RoomID, ContentURI
from mautrix.util.async_db import Database
fake_db = Database("") if TYPE_CHECKING else None
@ -28,27 +28,31 @@ class Portal:
db: ClassVar[Database] = fake_db
chat_id: str
other_user: str
other_user: str # TODO Remove, as it's redundant: other_user == chat_id for direct chats
mxid: Optional[RoomID]
name: Optional[str]
icon_url: Optional[str]
icon_path: Optional[str]
icon_mxc: Optional[ContentURI]
encrypted: bool
async def insert(self) -> None:
q = ("INSERT INTO portal (chat_id, other_user, mxid, name, icon_url, encrypted) "
"VALUES ($1, $2, $3, $4, $5, $6)")
q = ("INSERT INTO portal (chat_id, other_user, mxid, name, icon_path, icon_mxc, encrypted) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)")
await self.db.execute(q, self.chat_id, self.other_user, self.mxid, self.name,
self.icon_url, self.encrypted)
self.icon_path, self.icon_mxc,
self.encrypted)
async def update(self) -> None:
q = ("UPDATE portal SET other_user=$2, mxid=$3, name=$4, icon_url=$5, encrypted=$6 "
q = ("UPDATE portal SET other_user=$2, mxid=$3, name=$4, "
" icon_path=$5, icon_mxc=$6, encrypted=$7 "
"WHERE chat_id=$1")
await self.db.execute(q, self.chat_id, self.other_user,
self.mxid, self.name, self.icon_url, self.encrypted)
await self.db.execute(q, self.chat_id, self.other_user, self.mxid, self.name,
self.icon_path, self.icon_mxc,
self.encrypted)
@classmethod
async def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
q = ("SELECT chat_id, other_user, mxid, name, icon_url, encrypted "
q = ("SELECT chat_id, other_user, mxid, name, icon_path, icon_mxc, encrypted "
"FROM portal WHERE mxid=$1")
row = await cls.db.fetchrow(q, mxid)
if not row:
@ -57,7 +61,7 @@ class Portal:
@classmethod
async def get_by_chat_id(cls, chat_id: int) -> Optional['Portal']:
q = ("SELECT chat_id, other_user, mxid, name, icon_url, encrypted "
q = ("SELECT chat_id, other_user, mxid, name, icon_path, icon_mxc, encrypted "
"FROM portal WHERE chat_id=$1")
row = await cls.db.fetchrow(q, chat_id)
if not row:
@ -66,12 +70,14 @@ class Portal:
@classmethod
async def find_private_chats(cls) -> List['Portal']:
rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, icon_url, encrypted "
rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, "
" icon_path, icon_mxc, encrypted "
"FROM portal WHERE other_user IS NOT NULL")
return [cls(**row) for row in rows]
@classmethod
async def all_with_room(cls) -> List['Portal']:
rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, icon_url, encrypted "
rows = await cls.db.fetch("SELECT chat_id, other_user, mxid, name, "
" icon_path, icon_mxc, encrypted "
"FROM portal WHERE mxid IS NOT NULL")
return [cls(**row) for row in rows]

29
matrix_puppeteer_line/db/puppet.py
View File

@ -17,6 +17,7 @@ from typing import Optional, ClassVar, TYPE_CHECKING
from attr import dataclass
from mautrix.types import ContentURI
from mautrix.util.async_db import Database
fake_db = Database("") if TYPE_CHECKING else None
@ -28,21 +29,35 @@ class Puppet:
mid: str
name: Optional[str]
avatar_url: Optional[str]
avatar_path: Optional[str]
avatar_mxc: Optional[ContentURI]
name_set: bool
avatar_set: bool
is_registered: bool
async def insert(self) -> None:
q = "INSERT INTO puppet (mid, name, avatar_url, is_registered) VALUES ($1, $2, $3, $4)"
await self.db.execute(q, self.mid, self.name, self.avatar_url, self.is_registered)
q = ("INSERT INTO puppet (mid, name, "
" avatar_path, avatar_mxc, name_set, avatar_set, "
" is_registered) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)")
await self.db.execute(q, self.mid, self.name,
self.avatar_path, self.avatar_mxc, self.name_set, self.avatar_set,
self.is_registered)
async def update(self) -> None:
q = "UPDATE puppet SET name=$2, avatar_url=$3, is_registered=$4 WHERE mid=$1"
await self.db.execute(q, self.mid, self.name, self.avatar_url, self.is_registered)
q = ("UPDATE puppet SET name=$2, "
" avatar_path=$3, avatar_mxc=$4, name_set=$5, avatar_set=$6, "
" is_registered=$7 "
"WHERE mid=$1")
await self.db.execute(q, self.mid, self.name,
self.avatar_path, self.avatar_mxc, self.name_set, self.avatar_set,
self.is_registered)
@classmethod
async def get_by_mid(cls, mid: str) -> Optional['Puppet']:
row = await cls.db.fetchrow("SELECT mid, name, avatar_url, is_registered FROM puppet WHERE mid=$1",
mid)
q = ("SELECT mid, name, avatar_path, avatar_mxc, name_set, avatar_set, is_registered "
"FROM puppet WHERE mid=$1")
row = await cls.db.fetchrow(q, mid)
if not row:
return None
return cls(**row)

16
matrix_puppeteer_line/db/upgrade.py
View File

@ -50,9 +50,13 @@ async def upgrade_v1(conn: Connection) -> None:
@upgrade_table.register(description="Avatars and icons")
async def upgrade_avatars(conn: Connection) -> None:
for (table, column) in [('puppet', 'avatar_url'), ('portal', 'icon_url')]:
column_exists = await conn.fetchval(
"SELECT EXISTS(SELECT FROM information_schema.columns "
f"WHERE table_name='{table}' AND column_name='{column}')")
if not column_exists:
await conn.execute(f'ALTER TABLE "{table}" ADD COLUMN {column} TEXT')
await conn.execute("""ALTER TABLE puppet
ADD COLUMN IF NOT EXISTS avatar_path TEXT,
ADD COLUMN IF NOT EXISTS avatar_mxc TEXT,
ADD COLUMN IF NOT EXISTS name_set BOOLEAN,
ADD COLUMN IF NOT EXISTS avatar_set BOOLEAN
""")
await conn.execute("""ALTER TABLE portal
ADD COLUMN IF NOT EXISTS icon_path TEXT,
ADD COLUMN IF NOT EXISTS icon_mxc TEXT
""")

48
matrix_puppeteer_line/portal.py
View File

@ -30,7 +30,7 @@ from mautrix.util.network_retry import call_with_net_retry
from .db import Portal as DBPortal, Message as DBMessage
from .config import Config
from .rpc import ChatInfo, Participant, Message
from .rpc import ChatInfo, Participant, Message, Client, PathImage
from . import user as u, puppet as p, matrix as m
if TYPE_CHECKING:
@ -62,9 +62,10 @@ class Portal(DBPortal, BasePortal):
_last_participant_update: Set[str]
def __init__(self, chat_id: int, other_user: Optional[str] = None,
mxid: Optional[RoomID] = None, name: Optional[str] = None, icon_url: Optional[str] = None,
mxid: Optional[RoomID] = None, name: Optional[str] = None,
icon_path: Optional[str] = None, icon_mxc: Optional[ContentURI] = None,
encrypted: bool = False) -> None:
super().__init__(chat_id, other_user, mxid, name, icon_url, encrypted)
super().__init__(chat_id, other_user, mxid, name, icon_path, icon_mxc, encrypted)
self._create_room_lock = asyncio.Lock()
self.log = self.log.getChild(str(chat_id))
@ -229,18 +230,25 @@ class Portal(DBPortal, BasePortal):
return ReuploadedMediaInfo(mxc, decryption_info, mime_type, file_name, len(data))
async def update_info(self, conv: ChatInfo) -> None:
async def update_info(self, conv: ChatInfo, client: Optional[Client]) -> None:
if self.is_direct:
self.other_user = conv.participants[0].id
if self._main_intent is self.az.intent:
self._main_intent = (await p.Puppet.get_by_mid(self.other_user)).intent
for participant in conv.participants:
puppet = await p.Puppet.get_by_mid(participant.id)
await puppet.update_info(participant)
await puppet.update_info(participant, client)
# TODO Consider setting no room name for non-group chats.
# But then the LINE bot itself may appear in the title...
changed = await self._update_name(f"{conv.name} (LINE)")
changed = await self._update_icon(conv.iconURL) or changed
if client:
if not self.is_direct:
changed = await self._update_icon(conv.icon, client) or changed
elif puppet and puppet.avatar_mxc != self.icon_mxc:
changed = True
self.icon_mxc = puppet.avatar_mxc
if self.mxid:
await self.main_intent.set_room_avatar(self.mxid, self.icon_mxc)
if changed:
await self.update_bridge_info()
await self.update()
@ -256,12 +264,17 @@ class Portal(DBPortal, BasePortal):
return True
return False
async def _update_icon(self, icon_url: Optional[str]) -> bool:
if self.icon_url != icon_url:
self.icon_url = icon_url
if icon_url:
# TODO set icon from bytes
pass
async def _update_icon(self, icon: Optional[PathImage], client: Client) -> bool:
icon_path = icon.path if icon else None
if icon_path != self.icon_path:
self.icon_path = icon_path
if icon and icon.url:
resp = await client.read_image(icon.url)
self.icon_mxc = await self.main_intent.upload_media(resp.data, mime_type=resp.mime)
else:
self.icon_mxc = ContentURI("")
if self.mxid:
await self.main_intent.set_room_avatar(self.mxid, self.icon_mxc)
return True
return False
@ -319,7 +332,7 @@ class Portal(DBPortal, BasePortal):
for evt in messages:
puppet = await p.Puppet.get_by_mid(evt.sender.id) if not self.is_direct else None
if puppet and evt.sender.id not in members_known:
await puppet.update_info(evt.sender)
await puppet.update_info(evt.sender, source.client)
members_known.add(evt.sender.id)
await self.handle_remote_message(source, puppet, evt)
self.log.info("Backfilled %d messages through %s", len(messages), source.mxid)
@ -377,7 +390,7 @@ class Portal(DBPortal, BasePortal):
if puppet:
await puppet.az.intent.ensure_joined(self.mxid)
await self.update_info(info)
await self.update_info(info, source.client)
await self.backfill(source)
await self._update_participants(info.participants)
@ -385,7 +398,7 @@ class Portal(DBPortal, BasePortal):
if self.mxid:
await self._update_matrix_room(source, info)
return self.mxid
await self.update_info(info)
await self.update_info(info, source.client)
self.log.debug("Creating Matrix room")
name: Optional[str] = None
initial_state = [{
@ -483,7 +496,10 @@ class Portal(DBPortal, BasePortal):
self.by_mxid.pop(self.mxid, None)
self.mxid = None
self.name = None
self.icon_url = None
self.icon_path = None
self.icon_mxc = None
self.name_set = False
self.icon_set = False
self.encrypted = False
await self.update()

49
matrix_puppeteer_line/puppet.py
View File

@ -16,12 +16,12 @@
from typing import Optional, Dict, TYPE_CHECKING, cast
from mautrix.bridge import BasePuppet
from mautrix.types import UserID
from mautrix.types import UserID, ContentURI
from mautrix.util.simple_template import SimpleTemplate
from .db import Puppet as DBPuppet
from .config import Config
from .rpc import Participant
from .rpc import Participant, Client, PathImage
from . import user as u
if TYPE_CHECKING:
@ -38,9 +38,11 @@ class Puppet(DBPuppet, BasePuppet):
default_mxid: UserID
def __init__(self, mid: str, name: Optional[str] = None, avatar_url: Optional[str] = None,
def __init__(self, mid: str, name: Optional[str] = None,
avatar_path: Optional[str] = None, avatar_mxc: Optional[ContentURI] = None,
name_set: bool = False, avatar_set: bool = False,
is_registered: bool = False) -> None:
super().__init__(mid=mid, name=name, avatar_url=avatar_url, is_registered=is_registered)
super().__init__(mid, name, avatar_path, avatar_mxc, name_set, avatar_set, is_registered)
self.log = self.log.getChild(mid)
self.default_mxid = self.get_mxid_from_id(mid)
@ -61,27 +63,46 @@ class Puppet(DBPuppet, BasePuppet):
cls.login_shared_secret_map[cls.hs_domain] = secret.encode("utf-8")
cls.login_device_name = "LINE Bridge"
async def update_info(self, info: Participant) -> None:
async def update_info(self, info: Participant, client: Optional[Client]) -> None:
update = False
update = await self._update_name(info.name) or update
update = await self._update_avatar(info.avatarURL) or update
if client:
update = await self._update_avatar(info.avatar, client) or update
if update:
await self.update()
async def _update_name(self, name: str) -> bool:
name = self.config["bridge.displayname_template"].format(displayname=name)
if name != self.name:
if name != self.name or not self.name_set:
self.name = name
await self.intent.set_displayname(self.name)
try:
await self.intent.set_displayname(self.name)
self.name_set = True
except Exception:
self.log.exception("Failed to set displayname")
self.name_set = False
return True
return False
async def _update_avatar(self, avatar_url: Optional[str]) -> bool:
if avatar_url != self.avatar_url:
self.avatar_url = avatar_url
if avatar_url:
# TODO set the avatar from bytes
pass
async def _update_avatar(self, avatar: Optional[PathImage], client: Client) -> bool:
if avatar and avatar.url and not avatar.path:
# Avatar exists, but in a form that cannot be uniquely identified.
# Skip it for now.
return False
avatar_path = avatar.path if avatar else None
if avatar_path != self.avatar_path or not self.avatar_set:
self.avatar_path = avatar_path
if avatar and avatar.url:
resp = await client.read_image(avatar.url)
self.avatar_mxc = await self.intent.upload_media(resp.data, mime_type=resp.mime)
else:
self.avatar_mxc = ContentURI("")
try:
await self.intent.set_avatar_url(self.avatar_mxc)
self.avatar_set = True
except Exception:
self.log.exception("Failed to set user avatar")
self.avatar_set = False
return True
return False

2
matrix_puppeteer_line/rpc/__init__.py
View File

@ -1,2 +1,2 @@
from .client import Client
from .types import RPCError, ChatListInfo, ChatInfo, Participant, Message, StartStatus
from .types import RPCError, PathImage, ChatListInfo, ChatInfo, Participant, Message, StartStatus

22
matrix_puppeteer_line/rpc/client.py
View File

@ -15,10 +15,11 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import AsyncGenerator, TypedDict, List, Tuple, Dict, Callable, Awaitable, Any
from collections import deque
from base64 import b64decode
import asyncio
from .rpc import RPCClient
from .types import ChatListInfo, ChatInfo, Message, StartStatus
from .types import ChatListInfo, ChatInfo, Message, ImageData, StartStatus
class LoginCommand(TypedDict):
@ -45,6 +46,25 @@ class Client(RPCClient):
resp = await self.request("get_messages", chat_id=chat_id)
return [Message.deserialize(data) for data in resp]
async def read_image(self, image_url: str) -> ImageData:
resp = await self.request("read_image", image_url=image_url)
if not resp.startswith("data:"):
raise TypeError("Image data is not in the form of a Data URL")
typestart = 5
typeend = resp.find(",", typestart)
data = bytes(resp[typeend+1:], "utf-8")
paramstart = resp.rfind(";", typestart, typeend)
if paramstart == -1:
mime = resp[typestart:typeend]
else:
mime = resp[typestart:paramstart]
if resp[paramstart+1:typeend] == "base64":
data = b64decode(data)
return ImageData(mime=mime, data=data)
async def is_connected(self) -> bool:
resp = await self.request("is_connected")
return resp["is_connected"]

2
matrix_puppeteer_line/rpc/rpc.py
View File

@ -161,6 +161,6 @@ class RPCClient:
await self._writer.drain()
return future
async def request(self, command: str, **data: Any) -> Dict[str, Any]:
async def request(self, command: str, **data: Any) -> Any:
future = await self._raw_request(command, **data)
return await future

16
matrix_puppeteer_line/rpc/types.py
View File

@ -24,11 +24,17 @@ class RPCError(Exception):
pass
@dataclass
class PathImage(SerializableAttrs['PathImage']):
path: str
url: str
@dataclass
class ChatListInfo(SerializableAttrs['ChatListInfo']):
id: int
name: str
iconURL: Optional[str]
icon: Optional[PathImage]
lastMsg: str
lastMsgDate: str
@ -36,7 +42,7 @@ class ChatListInfo(SerializableAttrs['ChatListInfo']):
@dataclass
class Participant(SerializableAttrs['Participant']):
id: str
avatarURL: Optional[str]
avatar: Optional[PathImage]
name: str
@ -56,6 +62,12 @@ class Message(SerializableAttrs['Message']):
image: Optional[str] = None
@dataclass
class ImageData:
mime: str
data: bytes
@dataclass
class StartStatus(SerializableAttrs['StartStatus']):
started: bool

1
puppet/src/client.js
View File

@ -232,6 +232,7 @@ export default class Client {
get_chats: () => this.puppet.getRecentChats(),
get_chat: req => this.puppet.getChatInfo(req.chat_id),
get_messages: req => this.puppet.getMessages(req.chat_id),
read_image: req => this.puppet.readImage(req.image_url),
is_connected: async () => ({ is_connected: !await this.puppet.isDisconnected() }),
}[req.command] || this.handleUnknownCommand
}

73
puppet/src/contentscript.js
View File

@ -182,7 +182,7 @@ class MautrixController {
const participantsList = document.querySelector(participantsListSelector)
sender.id = participantsList.querySelector(`img[alt='${senderName}'`).parentElement.parentElement.getAttribute("data-mid")
}
sender.avatarURL = this.getParticipantListItemAvatarURL(element)
sender.avatar = this.getParticipantListItemAvatar(element)
} else {
// TODO Get own ID and store it somewhere appropriate.
// Unable to get own ID from a room chat...
@ -196,7 +196,7 @@ class MautrixController {
await window.__mautrixShowParticipantsList()
const participantsList = document.querySelector(participantsListSelector)
sender.name = this.getParticipantListItemName(participantsList.children[0])
sender.avatarURL = this.getParticipantListItemAvatarURL(participantsList.children[0])
sender.avatar = this.getParticipantListItemAvatar(participantsList.children[0])
sender.id = this.ownID
}
@ -299,25 +299,41 @@ class MautrixController {
return messages
}
/**
* @typedef PathImage
* @type object
* @property {string} path - The virtual path of the image (behaves like an ID)
* @property {string} src - The URL of the image
*/
_getPathImage(img) {
if (img && img.src.startsWith("blob:")) {
// NOTE Having a blob but no path means the image exists,
// but in a form that cannot be uniquely identified.
// If instead there is no blob, the image is blank.
return {
path: img.getAttribute("data-picture-path"),
url: img.src,
}
} else {
return null
}
}
/**
* @typedef Participant
* @type object
* @property {string} id - The member ID for the participant
* @property {string} avatarURL - The URL of the participant's avatar
* @property {string} name - The contact list name of the participant
* @property {string} id - The member ID for the participant
* @property {PathImage} avatar - The path and blob URL of the participant's avatar
* @property {string} name - The contact list name of the participant
*/
getParticipantListItemName(element) {
return element.querySelector(".mdRGT13Ttl").innerText
}
getParticipantListItemAvatarURL(element) {
const img = element.querySelector(".mdRGT13Img img[src]")
if (img && img.getAttribute("data-picture-path") != "" && img.src.startsWith("blob:")) {
return img.src
} else {
return ""
}
getParticipantListItemAvatar(element) {
return this._getPathImage(element.querySelector(".mdRGT13Img img[src]"))
}
getParticipantListItemId(element) {
@ -340,7 +356,7 @@ class MautrixController {
// One idea is to add real ID as suffix if we're in a group, and
// put in the puppet DB table somehow.
id: this.ownID,
avatarURL: this.getParticipantListItemAvatarURL(element.children[0]),
avatar: this.getParticipantListItemAvatar(element.children[0]),
name: this.getParticipantListItemName(element.children[0]),
}
@ -349,7 +365,7 @@ class MautrixController {
const id = this.getParticipantListItemId(child) || this.getUserIdFromFriendsList(name)
return {
id: id, // NOTE Don't want non-own user's ID to ever be null.
avatarURL: this.getParticipantListItemAvatarURL(child),
avatar: this.getParticipantListItemAvatar(child),
name: name,
}
}))
@ -358,9 +374,9 @@ class MautrixController {
/**
* @typedef ChatListInfo
* @type object
* @property {number} id - The ID of the chat.
* @property {string} name - The name of the chat.
* @property {string} iconURL - The URL of the chat icon.
* @property {number} id - The ID of the chat.
* @property {string} name - The name of the chat.
* @property {PathImage} icon - The path and blob URL of the chat icon.
* @property {string} lastMsg - The most recent message in the chat.
* May be prefixed by sender name.
* @property {string} lastMsgDate - An imprecise date for the most recent message
@ -375,13 +391,8 @@ class MautrixController {
return element.querySelector(".mdCMN04Ttl").innerText
}
getChatListItemIconURL(element) {
const img = element.querySelector(".mdCMN04Img > :not(.mdCMN04ImgInner) > img[src]")
if (img && img.getAttribute("data-picture-path") != "" && img.src.startsWith("blob:")) {
return img.src
} else {
return ""
}
getChatListItemIcon(element) {
return this._getPathImage(element.querySelector(".mdCMN04Img > :not(.mdCMN04ImgInner) > img[src]"))
}
getChatListItemLastMsg(element) {
@ -403,7 +414,7 @@ class MautrixController {
return !element.classList.contains("chatList") ? null : {
id: knownId || this.getChatListItemId(element),
name: this.getChatListItemName(element),
iconURL: this.getChatListItemIconURL(element),
icon: this.getChatListItemIcon(element),
lastMsg: this.getChatListItemLastMsg(element),
lastMsgDate: this.getChatListItemLastMsgDate(element),
}
@ -434,17 +445,13 @@ class MautrixController {
}
/**
* TODO
* Download an image and return it as a data URL.
* Used for downloading the blob: URLs in image messages.
* Download an image at a given URL and return it as a data URL.
*
* @param {number} id - The ID of the message whose image to download.
* @param {string} url - The URL of the image to download.
* @return {Promise<string>} - The data URL (containing the mime type and base64 data)
*/
async readImage(id) {
const imageElement = document.querySelector(
`mws-message-wrapper[msg-id="${id}"] mws-image-message-part .image-msg`)
const resp = await fetch(imageElement.getAttribute("src"))
async readImage(url) {
const resp = await fetch(url)
const reader = new FileReader()
const promise = new Promise((resolve, reject) => {
reader.onload = () => resolve(reader.result)

9
puppet/src/puppet.js
View File

@ -396,6 +396,13 @@ export default class MessagesPuppeteer {
this.log("Updated most recent message ID map:", this.mostRecentMessages)
}
async readImage(imageUrl) {
return await this.taskQueue.push(() =>
this.page.evaluate(
url => window.__mautrixController.readImage(url),
imageUrl))
}
async startObserving() {
this.log("Adding chat list observer")
await this.page.evaluate(
@ -496,7 +503,7 @@ export default class MessagesPuppeteer {
//await chatDetailArea.$(".MdTxtDesc02") || // 1:1 chat with custom title - get participant's real name
participants = [{
id: id,
avatarURL: chatListInfo.iconURL,
avatar: chatListInfo.icon,
name: chatListInfo.name,
}]
}