From ace4eefec7efeab3cf19336fd023dfd4dd46dca1 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Sat, 26 Mar 2022 03:37:53 -0400 Subject: [PATCH] Outbound media messages --- matrix_appservice_kakaotalk/db/message.py | 8 +- .../db/upgrade/v01_initial_revision.py | 14 ++-- .../kt/client/client.py | 28 ++++++- .../kt/client/types.py | 22 ++++- matrix_appservice_kakaotalk/portal.py | 84 +++++++++++++++---- node/src/client.js | 25 ++++++ 6 files changed, 154 insertions(+), 27 deletions(-) diff --git a/matrix_appservice_kakaotalk/db/message.py b/matrix_appservice_kakaotalk/db/message.py index ed01c36..6a33d67 100644 --- a/matrix_appservice_kakaotalk/db/message.py +++ b/matrix_appservice_kakaotalk/db/message.py @@ -34,7 +34,7 @@ class Message: mxid: EventID mx_room: RoomID - ktid: Long = field(converter=Long) + ktid: Long | None = field(converter=lambda ktid: Long(ktid) if ktid is not None else None) index: int kt_chat: Long = field(converter=Long) kt_receiver: Long = field(converter=Long) @@ -104,9 +104,9 @@ class Message: @classmethod async def bulk_create( cls, - ktid: str, - kt_chat: int, - kt_receiver: int, + ktid: Long, + kt_chat: Long, + kt_receiver: Long, event_ids: list[EventID], timestamp: int, mx_room: RoomID, diff --git a/matrix_appservice_kakaotalk/db/upgrade/v01_initial_revision.py b/matrix_appservice_kakaotalk/db/upgrade/v01_initial_revision.py index d24afae..e490c62 100644 --- a/matrix_appservice_kakaotalk/db/upgrade/v01_initial_revision.py +++ b/matrix_appservice_kakaotalk/db/upgrade/v01_initial_revision.py @@ -41,7 +41,7 @@ async def create_v1_tables(conn: Connection) -> None: """CREATE TABLE portal ( ktid BIGINT, kt_receiver BIGINT, - kt_type TEXT, + kt_type TEXT NOT NULL, mxid TEXT UNIQUE, name TEXT, photo_id TEXT, @@ -72,13 +72,13 @@ async def create_v1_tables(conn: Connection) -> None: ) await conn.execute( """CREATE TABLE message ( - mxid TEXT, - mx_room TEXT, + mxid TEXT NOT NULL, + mx_room TEXT NOT NULL, ktid BIGINT, - kt_receiver BIGINT, - "index" SMALLINT, - kt_chat BIGINT, - timestamp BIGINT, + kt_receiver BIGINT NOT NULL, + "index" SMALLINT NOT NULL, + kt_chat BIGINT NOT NULL, + timestamp BIGINT NOT NULL, PRIMARY KEY (ktid, kt_receiver, "index"), FOREIGN KEY (kt_chat, kt_receiver) REFERENCES portal(ktid, kt_receiver) ON UPDATE CASCADE ON DELETE CASCADE, diff --git a/matrix_appservice_kakaotalk/kt/client/client.py b/matrix_appservice_kakaotalk/kt/client/client.py index c93800c..6cad474 100644 --- a/matrix_appservice_kakaotalk/kt/client/client.py +++ b/matrix_appservice_kakaotalk/kt/client/client.py @@ -40,7 +40,7 @@ from ..types.api.struct.profile import ProfileReqStruct, ProfileStruct from ..types.api.struct import FriendListStruct from ..types.bson import Long from ..types.client.client_session import LoginResult -from ..types.chat.chat import Chatlog +from ..types.chat import Chatlog, KnownChatType from ..types.oauth import OAuthCredential, OAuthInfo from ..types.request import ( deserialize_result, @@ -254,6 +254,32 @@ class Client: text=text, ) + async def send_media( + self, + channel_props: ChannelProps, + media_type: KnownChatType, + data: bytes, + filename: str, + *, + width: int | None = None, + height: int | None = None, + ext: str | None = None, + ) -> Chatlog: + return await self._api_user_request_result( + Chatlog, + "send_media", + channel_props=channel_props.serialize(), + type=media_type, + data=list(data), + name=filename, + width=width, + height=height, + ext=ext, + # Don't log the bytes + # TODO Disable logging per-argument, not per-command + is_secret=True + ) + # TODO Combine these into one diff --git a/matrix_appservice_kakaotalk/kt/client/types.py b/matrix_appservice_kakaotalk/kt/client/types.py index aa8d0b1..b00ba94 100644 --- a/matrix_appservice_kakaotalk/kt/client/types.py +++ b/matrix_appservice_kakaotalk/kt/client/types.py @@ -19,11 +19,17 @@ from typing import Optional, NewType, Union from attr import dataclass -from mautrix.types import SerializableAttrs, JSON, deserializer +from mautrix.types import ( + SerializableAttrs, + JSON, + deserializer, + MessageType, +) from ..types.bson import Long from ..types.channel.channel_info import NormalChannelInfo from ..types.channel.channel_type import ChannelType +from ..types.chat import KnownChatType from ..types.openlink.open_channel_info import OpenChannelInfo from ..types.user.channel_user_info import NormalChannelUserInfo, OpenChannelUserInfo @@ -64,3 +70,17 @@ class PortalChannelInfo(SerializableAttrs): class ChannelProps(SerializableAttrs): id: Long type: ChannelType + + +# TODO Add non-media types, like polls & maps +TO_MSGTYPE_MAP: dict[MessageType, KnownChatType] = { + MessageType.TEXT: KnownChatType.TEXT, + MessageType.IMAGE: KnownChatType.PHOTO, + MessageType.STICKER: KnownChatType.PHOTO, + MessageType.VIDEO: KnownChatType.VIDEO, + MessageType.AUDIO: KnownChatType.AUDIO, + MessageType.FILE: KnownChatType.FILE, +} + +# https://stackoverflow.com/a/483833 +FROM_MSGTYPE_MAP: dict[KnownChatType, MessageType] = {v: k for k, v in TO_MSGTYPE_MAP.items()} diff --git a/matrix_appservice_kakaotalk/portal.py b/matrix_appservice_kakaotalk/portal.py index e012fc2..eab03ea 100644 --- a/matrix_appservice_kakaotalk/portal.py +++ b/matrix_appservice_kakaotalk/portal.py @@ -25,6 +25,7 @@ from typing import ( cast, ) from io import BytesIO +from mimetypes import guess_extension import asyncio import re import time @@ -50,7 +51,7 @@ from mautrix.types import ( UserID, VideoInfo, ) -from mautrix.util import ffmpeg, magic, variation_selector +from mautrix.util import ffmpeg, magic from mautrix.util.message_send_checkpoint import MessageSendCheckpointStatus from mautrix.util.simple_lock import SimpleLock @@ -70,7 +71,12 @@ from .kt.types.chat.attachment import ( PhotoAttachment, ) -from .kt.client.types import UserInfoUnion, PortalChannelInfo, ChannelProps +from .kt.client.types import ( + UserInfoUnion, + PortalChannelInfo, + ChannelProps, + TO_MSGTYPE_MAP, +) from .kt.client.errors import CommandException if TYPE_CHECKING: @@ -745,24 +751,24 @@ class Portal(DBPortal, BasePortal): if message.msgtype == MessageType.TEXT or message.msgtype == MessageType.NOTICE: await self._handle_matrix_text(event_id, sender, message) elif message.msgtype.is_media: - await self._handle_matrix_media(event_id, sender, message, is_relay) + await self._handle_matrix_media(event_id, sender, message) # elif message.msgtype == MessageType.LOCATION: # await self._handle_matrix_location(sender, message) else: raise NotImplementedError(f"Unsupported message type {message.msgtype}") - async def _make_dbm(self, sender: u.User, event_id: EventID, ktid: Long) -> None: - await DBMessage( + async def _make_dbm(self, event_id: EventID, ktid: Long | None = None) -> DBMessage: + dbm = DBMessage( mxid=event_id, mx_room=self.mxid, ktid=ktid, index=0, kt_chat=self.ktid, kt_receiver=self.kt_receiver, - # TODO? - #kt_sender=sender.ktid, timestamp=int(time.time() * 1000), - ).insert() + ) + await dbm.insert() + return dbm async def _handle_matrix_text( self, event_id: EventID, sender: u.User, message: TextMessageEventContent @@ -776,11 +782,10 @@ class Portal(DBPortal, BasePortal): #mentions=converted.mentions, #reply_to=converted.reply_to, ) - except CommandException: - self.log.debug(f"Error handling Matrix message {event_id}") + except CommandException as e: + self.log.debug(f"Error handling Matrix message {event_id}: {e!s}") raise - - await self._make_dbm(sender, event_id, chatlog.logId) + await self._make_dbm(event_id, chatlog.logId) self.log.debug(f"Handled Matrix message {event_id} -> {chatlog.logId}") sender.send_remote_checkpoint( MessageSendCheckpointStatus.SUCCESS, @@ -791,9 +796,60 @@ class Portal(DBPortal, BasePortal): ) async def _handle_matrix_media( - self, event_id: EventID, sender: u.User, message: MediaMessageEventContent, is_relay: bool + self, event_id: EventID, sender: u.User, message: MediaMessageEventContent ) -> None: - self.log.info("TODO: _handle_matrix_media") + if message.file and decrypt_attachment: + data = await self.main_intent.download_media(message.file.url) + data = decrypt_attachment( + data, message.file.key.key, message.file.hashes.get("sha256"), message.file.iv + ) + elif message.url: + data = await self.main_intent.download_media(message.url) + else: + raise NotImplementedError("No file or URL specified") + mime = message.info.mimetype or magic.mimetype(data) + """ TODO Replies + reply_to = None + if message.relates_to.rel_type == RelationType.REPLY: + reply_to_msg = await DBMessage.get_by_mxid(message.relates_to.event_id, self.mxid) + if reply_to_msg: + reply_to = reply_to_msg.fbid + else: + self.log.warning( + f"Couldn't find reply target {message.relates_to.event_id}" + " to bridge media message reply metadata to KakaoTalk" + ) + """ + filename = message.body + width, height = None, None + # TODO Find out why/if stickers are always blank + if message.info in (MessageType.IMAGE, MessageType.STICKER, MessageType.VIDEO): + width = message.info.width + height = message.info.height + try: + chatlog = await sender.client.send_media( + self.channel_props, + TO_MSGTYPE_MAP[message.msgtype], + data, + filename, + width=width, + height=height, + ext=guess_extension(mime)[1:], + # TODO + #reply_to=reply_to, + ) + except CommandException as e: + self.log.debug(f"Error uploading media for Matrix message {event_id}: {e!s}") + raise + await self._make_dbm(event_id, chatlog.logId) + self.log.debug(f"Handled Matrix message {event_id} -> {chatlog.logId}") + sender.send_remote_checkpoint( + MessageSendCheckpointStatus.SUCCESS, + event_id, + self.mxid, + EventType.ROOM_MESSAGE, + message.msgtype, + ) async def _handle_matrix_location( self, sender: u.User, message: LocationMessageEventContent diff --git a/node/src/client.js b/node/src/client.js index 33d3c47..a6a6982 100644 --- a/node/src/client.js +++ b/node/src/client.js @@ -451,6 +451,30 @@ export default class PeerClient { }) } + /** + * @param {Object} req + * @param {string} req.mxid + * @param {Object} req.channel_props + * @param {int} req.type + * @param {[number]} req.data + * @param {string} req.name + * @param {?int} req.width + * @param {?int} req.height + * @param {?string} req.ext + */ + sendMedia = async (req) => { + const userClient = this.#getUser(req.mxid) + const talkChannel = await userClient.getChannel(req.channel_props) + + return await talkChannel.sendMedia(req.type, { + data: Uint8Array.from(req.data), + name: req.name, + width: req.width, + height: req.height, + ext: req.ext, + }) + } + #makeCommandResult(result) { return { success: true, @@ -529,6 +553,7 @@ export default class PeerClient { get_chats: this.getChats, list_friends: this.listFriends, send_message: this.sendMessage, + send_media: this.sendMedia, }[req.command] || this.handleUnknownCommand } const resp = { id: req.id }