fair/matrix-appservice-kakaotalk
fair
/
matrix-appservice-kakaotalk
1
0
Fork 2
Code Issues 2 Pull Requests 1 Projects Releases Wiki Activity

Inbound multiphoto, video, audio

This commit is contained in:
Andrew Ferrazzutti 2022-03-28 03:36:16 -04:00
parent ace4eefec7
commit 48ff7b3ceb
4 changed files with 207 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
matrix_appservice_kakaotalk/kt/types/chat/attachment/__init__.py
View File

@ -27,7 +27,6 @@ class Attachment(SerializableAttrs):
shout: Optional[bool] = None
mentions: Optional[list[MentionStruct]] = None
urls: Optional[list[str]] = None
url: Optional[str] = None # NOTE Added since this may have replaced urls
@dataclass

25
matrix_appservice_kakaotalk/kt/types/chat/attachment/media.py
View File

@ -21,14 +21,20 @@ from . import Attachment
@dataclass
class MediaKeyAttachment(Attachment):
k: str
class MediaAttachment(Attachment):
# NOTE Added to cover Attachments that need a url but might not have a key
url: str
s: int
@dataclass
class PhotoAttachment(Attachment):
class MediaKeyAttachment(MediaAttachment):
k: str
@dataclass
class PhotoAttachment(MediaKeyAttachment):
# NOTE Changed superclass from Attachment
w: int
h: int
thumbnailUrl: str
@ -48,11 +54,13 @@ class MultiPhotoAttachment(Attachment):
thumbnailUrls: list[str]
thumbnailWidths: list[int]
thumbnailHeights: list[int]
sl: int
sl: list[int] # NOTE Changed to a list
mtl: list[str] # NOTE Added
@dataclass
class VideoAttachment(Attachment):
class VideoAttachment(MediaAttachment):
# NOTE Changed superclass from Attachment
tk: str
w: int
h: int
@ -61,7 +69,8 @@ class VideoAttachment(Attachment):
@dataclass
class FileAttachment(Attachment):
class FileAttachment(MediaKeyAttachment):
# NOTE Changed superclass from Attachment
name: str
size: int
expire: int
@ -69,7 +78,8 @@ class FileAttachment(Attachment):
@dataclass
class AudioAttachment(Attachment):
class AudioAttachment(MediaKeyAttachment):
# NOTE Changed superclass from Attachment
d: int
expire: Optional[int] = None
@ -83,6 +93,7 @@ class LongTextAttachment(Attachment):
__all__ = [
"MediaAttachment",
"MediaKeyAttachment",
"PhotoAttachment",
"MultiPhotoAttachment",

2
matrix_appservice_kakaotalk/kt/types/chat/chat.py
View File

@ -35,6 +35,7 @@ from .attachment import (
FileAttachment,
PostAttachment,
ReplyAttachment,
MultiPhotoAttachment,
)
from .chat_type import ChatType, KnownChatType
@ -74,6 +75,7 @@ _attachment_type_map: dict[KnownChatType, Type[Attachment]] = {
KnownChatType.FILE: FileAttachment,
KnownChatType.POST: PostAttachment,
KnownChatType.REPLY: ReplyAttachment,
KnownChatType.MULTIPHOTO: MultiPhotoAttachment,
KnownChatType.OPEN_SCHEDULE: OpenScheduleAttachment,
KnownChatType.OPEN_VOTE: VoteAttachment,
KnownChatType.OPEN_POST: PostAttachment,

223
matrix_appservice_kakaotalk/portal.py
View File

@ -42,6 +42,7 @@ from mautrix.types import (
FileInfo,
ImageInfo,
LocationMessageEventContent,
MediaInfo,
MediaMessageEventContent,
Membership,
MessageEventContent,
@ -68,7 +69,13 @@ from .kt.types.channel.channel_info import ChannelInfo
from .kt.types.channel.channel_type import KnownChannelType, ChannelType
from .kt.types.chat import Chatlog, KnownChatType
from .kt.types.chat.attachment import (
Attachment,
AudioAttachment,
#FileAttachment,
MediaAttachment,
MultiPhotoAttachment,
PhotoAttachment,
VideoAttachment,
)
from .kt.client.types import (
@ -183,14 +190,25 @@ class Portal(DBPortal, BasePortal):
NotificationDisabler.config_enabled = cls.config["bridge.backfill.disable_notifications"]
# TODO More
cls._message_handler_type_map: dict[
cls._message_type_handler_map: dict[
KnownChatType,
Callable[[Portal, u.User, IntentAPI, Chatlog], Awaitable[list[EventID]]]
Callable[
[
Portal,
u.User,
IntentAPI,
Attachment | None,
int,
str | None
],
Awaitable[list[EventID]]
]
] = {
KnownChatType.TEXT: cls._handle_remote_text,
KnownChatType.PHOTO: cls._handle_remote_photo,
#KnownChatType.VIDEO: cls._handle_remote_video,
#KnownChatType.AUDIO: cls._handle_remote_audio,
KnownChatType.MULTIPHOTO: cls._handle_remote_multiphoto,
KnownChatType.VIDEO: cls._handle_remote_video,
KnownChatType.AUDIO: cls._handle_remote_audio,
#KnownChatType.FILE: cls._handle_remote_file,
}
@ -312,6 +330,7 @@ class Portal(DBPortal, BasePortal):
intent: IntentAPI,
*,
filename: str | None = None,
mimetype: str | None,
encrypt: bool = False,
find_size: bool = False,
convert_audio: bool = False,
@ -325,18 +344,19 @@ class Portal(DBPortal, BasePortal):
if length > cls.matrix.media_config.upload_size:
raise ValueError("File not available: too large")
data = await resp.read()
mime = magic.mimetype(data)
if convert_audio and mime != "audio/ogg":
if not mimetype:
mimetype = magic.mimetype(data)
if convert_audio and mimetype != "audio/ogg":
data = await ffmpeg.convert_bytes(
data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mime
data, ".ogg", output_args=("-c:a", "libopus"), input_mime=mimetype
)
mime = "audio/ogg"
info = FileInfo(mimetype=mime, size=len(data))
if Image and mime.startswith("image/") and find_size:
mimetype = "audio/ogg"
info = FileInfo(mimetype=mimetype, size=len(data))
if Image and mimetype.startswith("image/") and find_size:
with Image.open(BytesIO(data)) as img:
width, height = img.size
info = ImageInfo(mimetype=mime, size=len(data), width=width, height=height)
upload_mime_type = mime
info = ImageInfo(mimetype=mimetype, size=len(data), width=width, height=height)
upload_mime_type = mimetype
decryption_info = None
if encrypt and encrypt_attachment:
data, decryption_info = encrypt_attachment(data)
@ -807,7 +827,7 @@ class Portal(DBPortal, BasePortal):
data = await self.main_intent.download_media(message.url)
else:
raise NotImplementedError("No file or URL specified")
mime = message.info.mimetype or magic.mimetype(data)
mimetype = message.info.mimetype or magic.mimetype(data)
""" TODO Replies
reply_to = None
if message.relates_to.rel_type == RelationType.REPLY:
@ -822,7 +842,6 @@ class Portal(DBPortal, BasePortal):
"""
filename = message.body
width, height = None, None
# TODO Find out why/if stickers are always blank
if message.info in (MessageType.IMAGE, MessageType.STICKER, MessageType.VIDEO):
width = message.info.width
height = message.info.height
@ -834,7 +853,7 @@ class Portal(DBPortal, BasePortal):
filename,
width=width,
height=height,
ext=guess_extension(mime)[1:],
ext=guess_extension(mimetype)[1:],
# TODO
#reply_to=reply_to,
)
@ -977,16 +996,24 @@ class Portal(DBPortal, BasePortal):
await intent.ensure_joined(self.mxid)
self._backfill_leave.add(intent)
event_ids = []
handler = self._message_handler_type_map.get(message.type)
handler = self._message_type_handler_map.get(message.type)
if not handler:
self.log.warning(f"No handler for message type {message.type}, falling back to text")
handler = Portal._handle_remote_text
event_ids += await handler(self, source, intent, message)
event_ids = [
event_id for event_id in
await handler(
self,
source,
intent,
message.attachment,
message.sendAt,
message.text)
if event_id
]
if not event_ids:
self.log.warning(f"Unhandled KakaoTalk message {message.logId}")
return
event_ids = [event_id for event_id in event_ids if event_id]
self.log.debug(f"Handled KakaoTalk message {message.logId} -> {event_ids}")
await DBMessage.bulk_create(
ktid=message.logId,
@ -998,38 +1025,162 @@ class Portal(DBPortal, BasePortal):
)
await self._send_delivery_receipt(event_ids[-1])
async def _handle_remote_text(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]:
async def _handle_remote_text(
self,
source: u.User,
intent: IntentAPI,
attachment: None,
timestamp: int,
message_text: str | None,
) -> list[EventID]:
# TODO Handle mentions properly
content = await kakaotalk_to_matrix(message.text)
content = await kakaotalk_to_matrix(message_text)
# TODO Replies
return [await self._send_message(intent, content, timestamp=message.sendAt)]
return [await self._send_message(intent, content, timestamp=timestamp)]
async def _handle_remote_photo(self, source: u.User, intent: IntentAPI, message: Chatlog) -> list[EventID]:
assert message.attachment
assert message.attachment.url or message.attachment.urls
url = message.attachment.url or message.attachment.urls[0]
assert isinstance(message.attachment, PhotoAttachment)
info = ImageInfo(
width=message.attachment.w,
height=message.attachment.h,
def _handle_remote_photo(
self,
source: u.User,
intent: IntentAPI,
attachment: PhotoAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[list[EventID]]:
return asyncio.gather(self._handle_remote_uniphoto(
source, intent, attachment, timestamp, message_text
))
def _handle_remote_multiphoto(
self,
source: u.User,
intent: IntentAPI,
attachment: MultiPhotoAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[list[EventID]]:
# TODO Upload media concurrently, but post messages sequentially
return asyncio.gather(
*[
self._handle_remote_uniphoto(
source, intent,
PhotoAttachment(
shout=attachment.shout,
mentions=attachment.mentions,
urls=attachment.urls,
url=attachment.imageUrls[i],
s=attachment.sl[i],
k=attachment.kl[i],
w=attachment.wl[i],
h=attachment.hl[i],
thumbnailUrl=attachment.thumbnailUrls[i],
thumbnailWidth=attachment.thumbnailWidths[i],
thumbnailHeight=attachment.thumbnailHeights[i],
cs=attachment.csl[i],
mt=attachment.mtl[i],
),
timestamp, message_text,
)
# TODO Animated images?
for i in range(len(attachment.imageUrls))
]
)
def _handle_remote_uniphoto(
self,
source: u.User,
intent: IntentAPI,
attachment: PhotoAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[EventID]:
return self._handle_remote_media(
source, intent, attachment, timestamp, message_text,
ImageInfo(
mimetype=attachment.mt,
size=attachment.s,
width=attachment.w,
height=attachment.h,
),
MessageType.IMAGE,
)
def _handle_remote_video(
self,
source: u.User,
intent: IntentAPI,
attachment: VideoAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[list[EventID]]:
return asyncio.gather(self._handle_remote_media(
source, intent, attachment, timestamp, message_text,
VideoInfo(
duration=attachment.d,
width=attachment.w,
height=attachment.h,
),
MessageType.VIDEO,
))
def _handle_remote_audio(
self,
source: u.User,
intent: IntentAPI,
attachment: AudioAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[list[EventID]]:
return asyncio.gather(self._handle_remote_media(
source, intent, attachment, timestamp, message_text,
AudioInfo(
size=attachment.s,
duration=attachment.d,
),
MessageType.AUDIO,
))
""" TODO Find what auth is required for reading file contents
def _handle_remote_file(
self,
source: u.User,
intent: IntentAPI,
attachment: FileAttachment,
timestamp: int,
message_text: str | None,
) -> Awaitable[list[EventID]]:
return asyncio.gather(self._handle_remote_media(
source, intent, attachment, timestamp, message_text,
FileInfo(
size=attachment.size,
),
MessageType.FILE,
))
"""
async def _handle_remote_media(
self,
source: u.User,
intent: IntentAPI,
attachment: MediaAttachment,
timestamp: int,
message_text: str | None,
info: MediaInfo,
msgtype: MessageType,
) -> EventID:
mxc, additional_info, decryption_info = await self._reupload_remote_file(
url,
attachment.url,
source,
intent,
mimetype=info.mimetype,
encrypt=self.encrypted,
find_size=False,
)
info.size = additional_info.size
info.mimetype = additional_info.mimetype
content = MediaMessageEventContent(
url=mxc, file=decryption_info, msgtype=MessageType.IMAGE, body=message.text, info=info
url=mxc, file=decryption_info, msgtype=msgtype, body=message_text, info=info
)
if not content:
return []
# TODO Replies
return [await self._send_message(intent, content, timestamp=message.sendAt)]
return await self._send_message(intent, content, timestamp=timestamp)
# TODO Many more remote handlers