Inbound multiphoto, video, audio

matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py

@@ -27,7 +27,6 @@ class Attachment(SerializableAttrs):
     shout: Optional[bool] = None
     mentions: Optional[list[MentionStruct]] = None
     urls: Optional[list[str]] = None
-    url: Optional[str] = None # NOTE Added since this may have replaced urls
 
 
 @dataclass

matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py

@@ -21,14 +21,20 @@ from . import Attachment
 
 
 @dataclass
-class MediaKeyAttachment(Attachment):
+class MediaAttachment(Attachment):
-    k: str
+    # NOTE Added to cover Attachments that need a url but might not have a key
     url: str
     s: int
 
 
 @dataclass
-class PhotoAttachment(Attachment):
+class MediaKeyAttachment(MediaAttachment):
+    k: str
+
+
+@dataclass
+class PhotoAttachment(MediaKeyAttachment):
+    # NOTE Changed superclass from Attachment
     w: int
     h: int
     thumbnailUrl: str
@@ -48,11 +54,13 @@ class MultiPhotoAttachment(Attachment):
     thumbnailUrls: list[str]
     thumbnailWidths: list[int]
     thumbnailHeights: list[int]
-    sl: int
+    sl: list[int] # NOTE Changed to a list
+    mtl: list[str] # NOTE Added
 
 
 @dataclass
-class VideoAttachment(Attachment):
+class VideoAttachment(MediaAttachment):
+    # NOTE Changed superclass from Attachment
     tk: str
     w: int
     h: int
@@ -61,7 +69,8 @@ class VideoAttachment(Attachment):
 
 
 @dataclass
-class FileAttachment(Attachment):
+class FileAttachment(MediaKeyAttachment):
+    # NOTE Changed superclass from Attachment
     name: str
     size: int
     expire: int
@@ -69,7 +78,8 @@ class FileAttachment(Attachment):
 
 
 @dataclass
-class AudioAttachment(Attachment):
+class AudioAttachment(MediaKeyAttachment):
+    # NOTE Changed superclass from Attachment
     d: int
     expire: Optional[int] = None
 
@@ -83,6 +93,7 @@ class LongTextAttachment(Attachment):
 
 
 __all__ = [
+    "MediaAttachment",
     "MediaKeyAttachment",
     "PhotoAttachment",
     "MultiPhotoAttachment",

matrix_appservice_kakaotalk/kt/types/chat/chat.py

@@ -35,6 +35,7 @@ from .attachment import (
     FileAttachment,
     PostAttachment,
     ReplyAttachment,
+    MultiPhotoAttachment,
 )
 from .chat_type import ChatType, KnownChatType
 
@@ -74,6 +75,7 @@ _attachment_type_map: dict[KnownChatType, Type[Attachment]] = {
     KnownChatType.FILE: FileAttachment,
     KnownChatType.POST: PostAttachment,
     KnownChatType.REPLY: ReplyAttachment,
+    KnownChatType.MULTIPHOTO: MultiPhotoAttachment,
     KnownChatType.OPEN_SCHEDULE: OpenScheduleAttachment,
     KnownChatType.OPEN_VOTE: VoteAttachment,
     KnownChatType.OPEN_POST: PostAttachment,

matrix_appservice_kakaotalk/portal.py

@@ -42,6 +42,7 @@ from mautrix.types import (
     FileInfo,
     ImageInfo,
     LocationMessageEventContent,
+    MediaInfo,
     MediaMessageEventContent,
     Membership,
     MessageEventContent,
@@ -68,7 +69,13 @@ from .kt.types.channel.channel_info import ChannelInfo
 from .kt.types.channel.channel_type import KnownChannelType, ChannelType
 from .kt.types.chat import Chatlog, KnownChatType
 from .kt.types.chat.attachment import (
+    Attachment,
+    AudioAttachment,
+    #FileAttachment,
+    MediaAttachment,
+    MultiPhotoAttachment,
     PhotoAttachment,
+    VideoAttachment,
 )
 
 from .kt.client.types import (
@@ -183,14 +190,25 @@ class Portal(DBPortal, BasePortal):
         NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"]
 
         # TODO More
-        cls._message_handler_type_map: dict[
+        cls._message_type_handler_map: dict[
             KnownChatType,
-            Callable[[Portal, u.User, IntentAPI, Chatlog], Awaitable[list[EventID]]]
+            Callable[
+                [
+                    Portal,
+                    u.User,
+                    IntentAPI,
+                    Attachment | None,
+                    int,
+                    str | None
+                ],
+                Awaitable[list[EventID]]
+            ]
         ] = {
             KnownChatType.TEXT: cls._handle_remote_text,
             KnownChatType.PHOTO: cls._handle_remote_photo,
-            #KnownChatType.VIDEO: cls._handle_remote_video,
+            KnownChatType.MULTIPHOTO: cls._handle_remote_multiphoto,
-            #KnownChatType.AUDIO: cls._handle_remote_audio,
+            KnownChatType.VIDEO: cls._handle_remote_video,
+            KnownChatType.AUDIO: cls._handle_remote_audio,
             #KnownChatType.FILE: cls._handle_remote_file,
         }
 
@@ -312,6 +330,7 @@ class Portal(DBPortal, BasePortal):
         intent: IntentAPI,
         *,
         filename: str | None = None,
+        mimetype: str | None,
         encrypt: bool = False,
         find_size: bool = False,
         convert_audio: bool = False,
@@ -325,18 +344,19 @@ class Portal(DBPortal, BasePortal):
             if length > cls.matrix.media_config.upload_size:
                 raise ValueError("File not available: too large")
             data = await resp.read()
-        mime = magic.mimetype(data)
+        if not mimetype:
-        if convert_audio and mime != "audio/ogg":
+            mimetype = magic.mimetype(data)
+        if convert_audio and mimetype != "audio/ogg":
             data = await ffmpeg.convert_bytes(
-                data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mime
+                data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mimetype
             )
-            mime = "audio/ogg"
+            mimetype = "audio/ogg"
-        info = FileInfo(mimetype=mime, size=len(data))
+        info = FileInfo(mimetype=mimetype, size=len(data))
-        if Image and mime.startswith("image/") and find_size:
+        if Image and mimetype.startswith("image/") and find_size:
             with Image.open(BytesIO(data)) as img:
                 width, height = img.size
-            info = ImageInfo(mimetype=mime, size=len(data), width=width, height=height)
+            info = ImageInfo(mimetype=mimetype, size=len(data), width=width, height=height)
-        upload_mime_type = mime
+        upload_mime_type = mimetype
         decryption_info = None
         if encrypt and encrypt_attachment:
             data, decryption_info = encrypt_attachment(data)
@@ -807,7 +827,7 @@ class Portal(DBPortal, BasePortal):
             data = await self.main_intent.download_media(message.url)
         else:
             raise NotImplementedError("No file or URL specified")
-        mime = message.info.mimetype or magic.mimetype(data)
+        mimetype = message.info.mimetype or magic.mimetype(data)
         """ TODO Replies
         reply_to = None
         if message.relates_to.rel_type == RelationType.REPLY:
@@ -822,7 +842,6 @@ class Portal(DBPortal, BasePortal):
         """
         filename = message.body
         width, height = None, None
-        # TODO Find out why/if stickers are always blank
         if message.info in (MessageType.IMAGE, MessageType.STICKER, MessageType.VIDEO):
             width = message.info.width
             height = message.info.height
@@ -834,7 +853,7 @@ class Portal(DBPortal, BasePortal):
                 filename,
                 width=width,
                 height=height,
-                ext=guess_extension(mime)[1:],
+                ext=guess_extension(mimetype)[1:],
                 # TODO
                 #reply_to=reply_to,
             )
@@ -977,16 +996,24 @@ class Portal(DBPortal, BasePortal):
             await intent.ensure_joined(self.mxid)
             self._backfill_leave.add(intent)
 
-        event_ids = []
+        handler = self._message_type_handler_map.get(message.type)
-        handler = self._message_handler_type_map.get(message.type)
         if not handler:
             self.log.warning(f"No handler for message type {message.type}, falling back to text")
             handler = Portal._handle_remote_text
-        event_ids += await handler(self, source, intent, message)
+        event_ids = [
+            event_id for event_id in 
+            await handler(
+                self,
+                source,
+                intent,
+                message.attachment,
+                message.sendAt,
+                message.text)
+            if event_id
+        ]
         if not event_ids:
             self.log.warning(f"Unhandled KakaoTalk message {message.logId}")
             return
-        event_ids = [event_id for event_id in event_ids if event_id]
         self.log.debug(f"Handled KakaoTalk message {message.logId} -> {event_ids}")
         await DBMessage.bulk_create(
             ktid=message.logId,
@@ -998,38 +1025,162 @@ class Portal(DBPortal, BasePortal):
         )
         await self._send_delivery_receipt(event_ids[-1])
 
-    async def _handle_remote_text(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]:
+    async def _handle_remote_text(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: None,
+        timestamp: int,
+        message_text: str | None,
+    ) -> list[EventID]:
         # TODO Handle mentions properly
-        content = await kakaotalk_to_matrix(message.text)
+        content = await kakaotalk_to_matrix(message_text)
         # TODO Replies
-        return [await self._send_message(intent, content, timestamp=message.sendAt)]
+        return [await self._send_message(intent, content, timestamp=timestamp)]
 
-    async def _handle_remote_photo(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]:
+    def _handle_remote_photo(
-        assert message.attachment
+        self,
-        assert message.attachment.url or message.attachment.urls
+        source: u.User,
-        url = message.attachment.url or message.attachment.urls[0]
+        intent: IntentAPI,
-        assert isinstance(message.attachment, PhotoAttachment)
+        attachment: PhotoAttachment,
-        info = ImageInfo(
+        timestamp: int,
-            width=message.attachment.w,
+        message_text: str | None,
-            height=message.attachment.h,
+    ) -> Awaitable[list[EventID]]:
+        return asyncio.gather(self._handle_remote_uniphoto(
+            source, intent, attachment, timestamp, message_text
+        ))
+
+    def _handle_remote_multiphoto(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: MultiPhotoAttachment,
+        timestamp: int,
+        message_text: str | None,
+    ) -> Awaitable[list[EventID]]:
+        # TODO Upload media concurrently, but post messages sequentially
+        return asyncio.gather(
+            *[
+                self._handle_remote_uniphoto(
+                    source, intent,
+                    PhotoAttachment(
+                        shout=attachment.shout,
+                        mentions=attachment.mentions,
+                        urls=attachment.urls,
+                        url=attachment.imageUrls[i],
+                        s=attachment.sl[i],
+                        k=attachment.kl[i],
+                        w=attachment.wl[i],
+                        h=attachment.hl[i],
+                        thumbnailUrl=attachment.thumbnailUrls[i],
+                        thumbnailWidth=attachment.thumbnailWidths[i],
+                        thumbnailHeight=attachment.thumbnailHeights[i],
+                        cs=attachment.csl[i],
+                        mt=attachment.mtl[i],
+                    ),
+                    timestamp, message_text,
                 )
-        # TODO Animated images?
+                for i in range(len(attachment.imageUrls))
+            ]
+        )
+
+    def _handle_remote_uniphoto(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: PhotoAttachment,
+        timestamp: int,
+        message_text: str | None,
+    ) -> Awaitable[EventID]:
+        return self._handle_remote_media(
+            source, intent, attachment, timestamp, message_text,
+            ImageInfo(
+                mimetype=attachment.mt,
+                size=attachment.s,
+                width=attachment.w,
+                height=attachment.h,
+            ),
+            MessageType.IMAGE,
+        )
+
+    def _handle_remote_video(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: VideoAttachment,
+        timestamp: int,
+        message_text: str | None,
+    ) -> Awaitable[list[EventID]]:
+        return asyncio.gather(self._handle_remote_media(
+            source, intent, attachment, timestamp, message_text,
+            VideoInfo(
+                duration=attachment.d,
+                width=attachment.w,
+                height=attachment.h,
+            ),
+            MessageType.VIDEO,
+        ))
+
+    def _handle_remote_audio(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: AudioAttachment,
+        timestamp: int,
+        message_text: str | None,
+    ) -> Awaitable[list[EventID]]:
+        return asyncio.gather(self._handle_remote_media(
+            source, intent, attachment, timestamp, message_text,
+            AudioInfo(
+                size=attachment.s,
+                duration=attachment.d,
+            ),
+            MessageType.AUDIO,
+        ))
+
+    """ TODO Find what auth is required for reading file contents
+    def _handle_remote_file(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: FileAttachment,
+        timestamp: int,
+        message_text: str | None,
+    ) -> Awaitable[list[EventID]]:
+        return asyncio.gather(self._handle_remote_media(
+            source, intent, attachment, timestamp, message_text,
+            FileInfo(
+                size=attachment.size,
+            ),
+            MessageType.FILE,
+        ))
+    """
+
+    async def _handle_remote_media(
+        self,
+        source: u.User,
+        intent: IntentAPI,
+        attachment: MediaAttachment,
+        timestamp: int,
+        message_text: str | None,
+        info: MediaInfo,
+        msgtype: MessageType,
+    ) -> EventID:
         mxc, additional_info, decryption_info = await self._reupload_remote_file(
-            url,
+            attachment.url,
             source,
             intent,
+            mimetype=info.mimetype,
             encrypt=self.encrypted,
             find_size=False,
         )
         info.size = additional_info.size
         info.mimetype = additional_info.mimetype
         content = MediaMessageEventContent(
-            url=mxc, file=decryption_info, msgtype=MessageType.IMAGE, body=message.text, info=info
+            url=mxc, file=decryption_info, msgtype=msgtype, body=message_text, info=info
         )
-        if not content:
-            return []
         # TODO Replies
-        return [await self._send_message(intent, content, timestamp=message.sendAt)]
+        return await self._send_message(intent, content, timestamp=timestamp)
 
     # TODO Many more remote handlers