From 48ff7b3ceb660fc07eb8178e7143ca1e8b19a9e7 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Mon, 28 Mar 2022 03:36:16 -0400 Subject: [PATCH] Inbound multiphoto, video, audio --- .../kt/types/chat/attachment/__init__.py | 1 - .../kt/types/chat/attachment/media.py | 25 +- .../kt/types/chat/chat.py | 2 + matrix_appservice_kakaotalk/portal.py | 223 +++++++++++++++--- 4 files changed, 207 insertions(+), 44 deletions(-) diff --git a/matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py b/matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py index f034f85..a6e38d0 100644 --- a/matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py +++ b/matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py @@ -27,7 +27,6 @@ class Attachment(SerializableAttrs): shout: Optional[bool] = None mentions: Optional[list[MentionStruct]] = None urls: Optional[list[str]] = None - url: Optional[str] = None # NOTE Added since this may have replaced urls @dataclass diff --git a/matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py b/matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py index 0bf831d..ff54cad 100644 --- a/matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py +++ b/matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py @@ -21,14 +21,20 @@ from . import Attachment @dataclass -class MediaKeyAttachment(Attachment): - k: str +class MediaAttachment(Attachment): + # NOTE Added to cover Attachments that need a url but might not have a key url: str s: int @dataclass -class PhotoAttachment(Attachment): +class MediaKeyAttachment(MediaAttachment): + k: str + + +@dataclass +class PhotoAttachment(MediaKeyAttachment): + # NOTE Changed superclass from Attachment w: int h: int thumbnailUrl: str @@ -48,11 +54,13 @@ class MultiPhotoAttachment(Attachment): thumbnailUrls: list[str] thumbnailWidths: list[int] thumbnailHeights: list[int] - sl: int + sl: list[int] # NOTE Changed to a list + mtl: list[str] # NOTE Added @dataclass -class VideoAttachment(Attachment): +class VideoAttachment(MediaAttachment): + # NOTE Changed superclass from Attachment tk: str w: int h: int @@ -61,7 +69,8 @@ class VideoAttachment(Attachment): @dataclass -class FileAttachment(Attachment): +class FileAttachment(MediaKeyAttachment): + # NOTE Changed superclass from Attachment name: str size: int expire: int @@ -69,7 +78,8 @@ class FileAttachment(Attachment): @dataclass -class AudioAttachment(Attachment): +class AudioAttachment(MediaKeyAttachment): + # NOTE Changed superclass from Attachment d: int expire: Optional[int] = None @@ -83,6 +93,7 @@ class LongTextAttachment(Attachment): __all__ = [ + "MediaAttachment", "MediaKeyAttachment", "PhotoAttachment", "MultiPhotoAttachment", diff --git a/matrix_appservice_kakaotalk/kt/types/chat/chat.py b/matrix_appservice_kakaotalk/kt/types/chat/chat.py index 62201ed..4561a0f 100644 --- a/matrix_appservice_kakaotalk/kt/types/chat/chat.py +++ b/matrix_appservice_kakaotalk/kt/types/chat/chat.py @@ -35,6 +35,7 @@ from .attachment import ( FileAttachment, PostAttachment, ReplyAttachment, + MultiPhotoAttachment, ) from .chat_type import ChatType, KnownChatType @@ -74,6 +75,7 @@ _attachment_type_map: dict[KnownChatType, Type[Attachment]] = { KnownChatType.FILE: FileAttachment, KnownChatType.POST: PostAttachment, KnownChatType.REPLY: ReplyAttachment, + KnownChatType.MULTIPHOTO: MultiPhotoAttachment, KnownChatType.OPEN_SCHEDULE: OpenScheduleAttachment, KnownChatType.OPEN_VOTE: VoteAttachment, KnownChatType.OPEN_POST: PostAttachment, diff --git a/matrix_appservice_kakaotalk/portal.py b/matrix_appservice_kakaotalk/portal.py index eab03ea..596ac90 100644 --- a/matrix_appservice_kakaotalk/portal.py +++ b/matrix_appservice_kakaotalk/portal.py @@ -42,6 +42,7 @@ from mautrix.types import ( FileInfo, ImageInfo, LocationMessageEventContent, + MediaInfo, MediaMessageEventContent, Membership, MessageEventContent, @@ -68,7 +69,13 @@ from .kt.types.channel.channel_info import ChannelInfo from .kt.types.channel.channel_type import KnownChannelType, ChannelType from .kt.types.chat import Chatlog, KnownChatType from .kt.types.chat.attachment import ( + Attachment, + AudioAttachment, + #FileAttachment, + MediaAttachment, + MultiPhotoAttachment, PhotoAttachment, + VideoAttachment, ) from .kt.client.types import ( @@ -183,14 +190,25 @@ class Portal(DBPortal, BasePortal): NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"] # TODO More - cls._message_handler_type_map: dict[ + cls._message_type_handler_map: dict[ KnownChatType, - Callable[[Portal, u.User, IntentAPI, Chatlog], Awaitable[list[EventID]]] + Callable[ + [ + Portal, + u.User, + IntentAPI, + Attachment | None, + int, + str | None + ], + Awaitable[list[EventID]] + ] ] = { KnownChatType.TEXT: cls._handle_remote_text, KnownChatType.PHOTO: cls._handle_remote_photo, - #KnownChatType.VIDEO: cls._handle_remote_video, - #KnownChatType.AUDIO: cls._handle_remote_audio, + KnownChatType.MULTIPHOTO: cls._handle_remote_multiphoto, + KnownChatType.VIDEO: cls._handle_remote_video, + KnownChatType.AUDIO: cls._handle_remote_audio, #KnownChatType.FILE: cls._handle_remote_file, } @@ -312,6 +330,7 @@ class Portal(DBPortal, BasePortal): intent: IntentAPI, *, filename: str | None = None, + mimetype: str | None, encrypt: bool = False, find_size: bool = False, convert_audio: bool = False, @@ -325,18 +344,19 @@ class Portal(DBPortal, BasePortal): if length > cls.matrix.media_config.upload_size: raise ValueError("File not available: too large") data = await resp.read() - mime = magic.mimetype(data) - if convert_audio and mime != "audio/ogg": + if not mimetype: + mimetype = magic.mimetype(data) + if convert_audio and mimetype != "audio/ogg": data = await ffmpeg.convert_bytes( - data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mime + data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mimetype ) - mime = "audio/ogg" - info = FileInfo(mimetype=mime, size=len(data)) - if Image and mime.startswith("image/") and find_size: + mimetype = "audio/ogg" + info = FileInfo(mimetype=mimetype, size=len(data)) + if Image and mimetype.startswith("image/") and find_size: with Image.open(BytesIO(data)) as img: width, height = img.size - info = ImageInfo(mimetype=mime, size=len(data), width=width, height=height) - upload_mime_type = mime + info = ImageInfo(mimetype=mimetype, size=len(data), width=width, height=height) + upload_mime_type = mimetype decryption_info = None if encrypt and encrypt_attachment: data, decryption_info = encrypt_attachment(data) @@ -807,7 +827,7 @@ class Portal(DBPortal, BasePortal): data = await self.main_intent.download_media(message.url) else: raise NotImplementedError("No file or URL specified") - mime = message.info.mimetype or magic.mimetype(data) + mimetype = message.info.mimetype or magic.mimetype(data) """ TODO Replies reply_to = None if message.relates_to.rel_type == RelationType.REPLY: @@ -822,7 +842,6 @@ class Portal(DBPortal, BasePortal): """ filename = message.body width, height = None, None - # TODO Find out why/if stickers are always blank if message.info in (MessageType.IMAGE, MessageType.STICKER, MessageType.VIDEO): width = message.info.width height = message.info.height @@ -834,7 +853,7 @@ class Portal(DBPortal, BasePortal): filename, width=width, height=height, - ext=guess_extension(mime)[1:], + ext=guess_extension(mimetype)[1:], # TODO #reply_to=reply_to, ) @@ -977,16 +996,24 @@ class Portal(DBPortal, BasePortal): await intent.ensure_joined(self.mxid) self._backfill_leave.add(intent) - event_ids = [] - handler = self._message_handler_type_map.get(message.type) + handler = self._message_type_handler_map.get(message.type) if not handler: self.log.warning(f"No handler for message type {message.type}, falling back to text") handler = Portal._handle_remote_text - event_ids += await handler(self, source, intent, message) + event_ids = [ + event_id for event_id in + await handler( + self, + source, + intent, + message.attachment, + message.sendAt, + message.text) + if event_id + ] if not event_ids: self.log.warning(f"Unhandled KakaoTalk message {message.logId}") return - event_ids = [event_id for event_id in event_ids if event_id] self.log.debug(f"Handled KakaoTalk message {message.logId} -> {event_ids}") await DBMessage.bulk_create( ktid=message.logId, @@ -998,38 +1025,162 @@ class Portal(DBPortal, BasePortal): ) await self._send_delivery_receipt(event_ids[-1]) - async def _handle_remote_text(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]: + async def _handle_remote_text( + self, + source: u.User, + intent: IntentAPI, + attachment: None, + timestamp: int, + message_text: str | None, + ) -> list[EventID]: # TODO Handle mentions properly - content = await kakaotalk_to_matrix(message.text) + content = await kakaotalk_to_matrix(message_text) # TODO Replies - return [await self._send_message(intent, content, timestamp=message.sendAt)] + return [await self._send_message(intent, content, timestamp=timestamp)] - async def _handle_remote_photo(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]: - assert message.attachment - assert message.attachment.url or message.attachment.urls - url = message.attachment.url or message.attachment.urls[0] - assert isinstance(message.attachment, PhotoAttachment) - info = ImageInfo( - width=message.attachment.w, - height=message.attachment.h, + def _handle_remote_photo( + self, + source: u.User, + intent: IntentAPI, + attachment: PhotoAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[list[EventID]]: + return asyncio.gather(self._handle_remote_uniphoto( + source, intent, attachment, timestamp, message_text + )) + + def _handle_remote_multiphoto( + self, + source: u.User, + intent: IntentAPI, + attachment: MultiPhotoAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[list[EventID]]: + # TODO Upload media concurrently, but post messages sequentially + return asyncio.gather( + *[ + self._handle_remote_uniphoto( + source, intent, + PhotoAttachment( + shout=attachment.shout, + mentions=attachment.mentions, + urls=attachment.urls, + url=attachment.imageUrls[i], + s=attachment.sl[i], + k=attachment.kl[i], + w=attachment.wl[i], + h=attachment.hl[i], + thumbnailUrl=attachment.thumbnailUrls[i], + thumbnailWidth=attachment.thumbnailWidths[i], + thumbnailHeight=attachment.thumbnailHeights[i], + cs=attachment.csl[i], + mt=attachment.mtl[i], + ), + timestamp, message_text, + ) + for i in range(len(attachment.imageUrls)) + ] ) - # TODO Animated images? + + def _handle_remote_uniphoto( + self, + source: u.User, + intent: IntentAPI, + attachment: PhotoAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[EventID]: + return self._handle_remote_media( + source, intent, attachment, timestamp, message_text, + ImageInfo( + mimetype=attachment.mt, + size=attachment.s, + width=attachment.w, + height=attachment.h, + ), + MessageType.IMAGE, + ) + + def _handle_remote_video( + self, + source: u.User, + intent: IntentAPI, + attachment: VideoAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[list[EventID]]: + return asyncio.gather(self._handle_remote_media( + source, intent, attachment, timestamp, message_text, + VideoInfo( + duration=attachment.d, + width=attachment.w, + height=attachment.h, + ), + MessageType.VIDEO, + )) + + def _handle_remote_audio( + self, + source: u.User, + intent: IntentAPI, + attachment: AudioAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[list[EventID]]: + return asyncio.gather(self._handle_remote_media( + source, intent, attachment, timestamp, message_text, + AudioInfo( + size=attachment.s, + duration=attachment.d, + ), + MessageType.AUDIO, + )) + + """ TODO Find what auth is required for reading file contents + def _handle_remote_file( + self, + source: u.User, + intent: IntentAPI, + attachment: FileAttachment, + timestamp: int, + message_text: str | None, + ) -> Awaitable[list[EventID]]: + return asyncio.gather(self._handle_remote_media( + source, intent, attachment, timestamp, message_text, + FileInfo( + size=attachment.size, + ), + MessageType.FILE, + )) + """ + + async def _handle_remote_media( + self, + source: u.User, + intent: IntentAPI, + attachment: MediaAttachment, + timestamp: int, + message_text: str | None, + info: MediaInfo, + msgtype: MessageType, + ) -> EventID: mxc, additional_info, decryption_info = await self._reupload_remote_file( - url, + attachment.url, source, intent, + mimetype=info.mimetype, encrypt=self.encrypted, find_size=False, ) info.size = additional_info.size info.mimetype = additional_info.mimetype content = MediaMessageEventContent( - url=mxc, file=decryption_info, msgtype=MessageType.IMAGE, body=message.text, info=info + url=mxc, file=decryption_info, msgtype=msgtype, body=message_text, info=info ) - if not content: - return [] # TODO Replies - return [await self._send_message(intent, content, timestamp=message.sendAt)] + return await self._send_message(intent, content, timestamp=timestamp) # TODO Many more remote handlers