WIP: testing #40

Draft
yuessir wants to merge 6 commits from yuessir/matrix-puppeteer-line:testing into testing
5 changed files with 167 additions and 81 deletions

View File

@ -10,5 +10,6 @@
"cycle_delay": 5000, "cycle_delay": 5000,
"use_xdotool": false, "use_xdotool": false,
"jiggle_delay": 20000, "jiggle_delay": 20000,
"devtools": false "devtools": false,
"timeZone": "Asia/Taipei"
} }

View File

@ -19,9 +19,10 @@
}, },
"dependencies": { "dependencies": {
"arg": "^4.1.3", "arg": "^4.1.3",
"chrono-node": "^2.1.7", "chrono-node": "^2.3.8",
"systemd-daemon": "^1.1.2", "dayjs": "^1.11.3",
"puppeteer": "9.1.1" "puppeteer": "9.1.1",
"systemd-daemon": "^1.1.2"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^10.1.0", "babel-eslint": "^10.1.0",

View File

@ -22,6 +22,7 @@
* @return {Promise<Date>} * @return {Promise<Date>}
*/ */
window.__chronoParseDate = function (text, ref, option) {} window.__chronoParseDate = function (text, ref, option) {}
window.__tryParseDateByTimeZone = function (text, ref, option) { }
/** /**
* @param {...string} text - The objects to log. * @param {...string} text - The objects to log.
* @return {Promise<void>} * @return {Promise<void>}
@ -143,7 +144,7 @@ class MautrixController {
* @private * @private
*/ */
async _tryParseDate(text, ref, option) { async _tryParseDate(text, ref, option) {
const parsed = await window.__chronoParseDate(text, ref, option) const parsed = await window.__tryParseDateByTimeZone(text, ref, option)
return parsed ? new Date(parsed) : null return parsed ? new Date(parsed) : null
} }

View File

@ -42,7 +42,7 @@ MessagesPuppeteer.extensionDir = config.extension_dir || MessagesPuppeteer.exten
MessagesPuppeteer.cycleDelay = config.cycle_delay || MessagesPuppeteer.cycleDelay MessagesPuppeteer.cycleDelay = config.cycle_delay || MessagesPuppeteer.cycleDelay
MessagesPuppeteer.useXdotool = config.use_xdotool || MessagesPuppeteer.useXdotool MessagesPuppeteer.useXdotool = config.use_xdotool || MessagesPuppeteer.useXdotool
MessagesPuppeteer.jiggleDelay = config.jiggle_delay || MessagesPuppeteer.jiggleDelay MessagesPuppeteer.jiggleDelay = config.jiggle_delay || MessagesPuppeteer.jiggleDelay
MessagesPuppeteer.timeZone = config.timeZone || MessagesPuppeteer.timeZone
const api = new PuppetAPI(config.listen) const api = new PuppetAPI(config.listen)
function stop() { function stop() {

View File

@ -22,6 +22,9 @@ import chrono from "chrono-node"
import TaskQueue from "./taskqueue.js" import TaskQueue from "./taskqueue.js"
import { sleep } from "./util.js" import { sleep } from "./util.js"
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js';
import timezone from 'dayjs/plugin/timezone.js';
export default class MessagesPuppeteer { export default class MessagesPuppeteer {
static profileDir = "./profiles" static profileDir = "./profiles"
@ -34,7 +37,7 @@ export default class MessagesPuppeteer {
static viewport = { width: 960, height: 840 } static viewport = { width: 960, height: 840 }
static url = undefined static url = undefined
static extensionDir = "extension_files" static extensionDir = "extension_files"
static timeZone = "Asia/Taipei"
/** /**
* *
* @param {string} id * @param {string} id
@ -79,7 +82,7 @@ export default class MessagesPuppeteer {
const args = [ const args = [
`--disable-extensions-except=${MessagesPuppeteer.extensionDir}`, `--disable-extensions-except=${MessagesPuppeteer.extensionDir}`,
`--load-extension=${MessagesPuppeteer.extensionDir}`, `--load-extension=${MessagesPuppeteer.extensionDir}`,
`--window-size=${MessagesPuppeteer.viewport.width},${MessagesPuppeteer.viewport.height+120}`, `--window-size=${MessagesPuppeteer.viewport.width},${MessagesPuppeteer.viewport.height + 120}`,
] ]
if (MessagesPuppeteer.noSandbox) { if (MessagesPuppeteer.noSandbox) {
args.push(`--no-sandbox`) args.push(`--no-sandbox`)
@ -142,7 +145,7 @@ export default class MessagesPuppeteer {
await this.page.exposeFunction("__mautrixLoggedOut", await this.page.exposeFunction("__mautrixLoggedOut",
this._onLoggedOut.bind(this)) this._onLoggedOut.bind(this))
await this.page.exposeFunction("__chronoParseDate", chrono.parseDate) await this.page.exposeFunction("__chronoParseDate", chrono.parseDate)
await this.page.exposeFunction("__tryParseDateByTimeZone", this._tryParseDateByTimeZone.bind(this))
// NOTE Must *always* re-login on a browser session, so no need to check if already logged in // NOTE Must *always* re-login on a browser session, so no need to check if already logged in
this.loginRunning = false this.loginRunning = false
this.loginCancelled = false this.loginCancelled = false
@ -150,6 +153,28 @@ export default class MessagesPuppeteer {
this.log("Startup complete") this.log("Startup complete")
} }
async _tryParseDateByTimeZone(text, ref, option) {
const localTz = MessagesPuppeteer.timeZone // This is the ISO string
dayjs.extend(utc);
dayjs.extend(timezone);
const localTime = dayjs.tz(new Date(), localTz)
const localOffset = localTime.utcOffset() // returns in minutes
const custom = chrono.casual.clone()
custom.refiners.push({
refine: (results) => {
Array.from(results).forEach((result) => {
// Returns the time with the offset in included (must use minutes)
result.start.imply('timezoneOffset', localOffset)
result.end && result.end.imply('timezoneOffset', localOffset)
})
return results
}})
const parsed = custom.parseDate(text, ref, option)
this.log(`parsed ${parsed}`)
return parsed
}
async _preparePage(navigateTo) { async _preparePage(navigateTo) {
await this.page.bringToFront() await this.page.bringToFront()
if (navigateTo) { if (navigateTo) {
@ -196,7 +221,7 @@ export default class MessagesPuppeteer {
* @param {string} text - The text to input. * @param {string} text - The text to input.
*/ */
async _enterText(inputElement, text) { async _enterText(inputElement, text) {
await inputElement.click({clickCount: 3}) await inputElement.click({ clickCount: 3 })
await inputElement.type(text) await inputElement.type(text)
} }
@ -213,45 +238,45 @@ export default class MessagesPuppeteer {
const loginContentArea = await this.page.waitForSelector("#login_content") const loginContentArea = await this.page.waitForSelector("#login_content")
switch (login_type) { switch (login_type) {
case "qr": { case "qr": {
this.log("Running QR login") this.log("Running QR login")
const qrButton = await this.page.waitForSelector("#login_qr_btn") const qrButton = await this.page.waitForSelector("#login_qr_btn")
await qrButton.click() await qrButton.click()
const qrElement = await this.page.waitForSelector("#login_qrcode_area div[title]", {visible: true}) const qrElement = await this.page.waitForSelector("#login_qrcode_area div[title]", { visible: true })
const currentQR = await this.page.evaluate(element => element.title, qrElement) const currentQR = await this.page.evaluate(element => element.title, qrElement)
this._receiveQRChange(currentQR) this._receiveQRChange(currentQR)
await this.page.evaluate( await this.page.evaluate(
element => window.__mautrixController.addQRChangeObserver(element), qrElement) element => window.__mautrixController.addQRChangeObserver(element), qrElement)
await this.page.evaluate( await this.page.evaluate(
element => window.__mautrixController.addQRAppearObserver(element), loginContentArea) element => window.__mautrixController.addQRAppearObserver(element), loginContentArea)
break break
}
case "email": {
this.log("Running email login")
if (!login_data) {
this._sendLoginFailure("No login credentials provided for email login")
return
} }
case "email": {
this.log("Running email login")
if (!login_data) {
this._sendLoginFailure("No login credentials provided for email login")
return
}
const emailButton = await this.page.waitForSelector("#login_email_btn") const emailButton = await this.page.waitForSelector("#login_email_btn")
await emailButton.click() await emailButton.click()
await this.page.waitForSelector("#login_email_area", {visible: true}) await this.page.waitForSelector("#login_email_area", { visible: true })
this.login_email = login_data["email"] this.login_email = login_data["email"]
this.login_password = login_data["password"] this.login_password = login_data["password"]
await this._sendEmailCredentials() await this._sendEmailCredentials()
await this.page.evaluate( await this.page.evaluate(
element => window.__mautrixController.addEmailAppearObserver(element), loginContentArea) element => window.__mautrixController.addEmailAppearObserver(element), loginContentArea)
break break
} }
default: default:
this._sendLoginFailure(`Invalid login type: ${login_type}`) this._sendLoginFailure(`Invalid login type: ${login_type}`)
return return
} }
await this.page.evaluate( await this.page.evaluate(
@ -280,12 +305,12 @@ export default class MessagesPuppeteer {
} }
const result = await Promise.race([ const result = await Promise.race([
() => this.page.waitForSelector("#mainApp:not(.MdNonDisp)", {timeout: 2000}) () => this.page.waitForSelector("#mainApp:not(.MdNonDisp)", { timeout: 2000 })
.then(value => { .then(value => {
loginSuccess = true loginSuccess = true
return value return value
}), }),
() => this.page.waitForSelector("#login_incorrect", {visible: true, timeout: 2000}) () => this.page.waitForSelector("#login_incorrect", { visible: true, timeout: 2000 })
.then(value => this.page.evaluate(element => element?.innerText, value)), .then(value => this.page.evaluate(element => element?.innerText, value)),
() => this._waitForLoginCancel(), () => this._waitForLoginCancel(),
].map(promiseFn => cancelableResolve(promiseFn))) ].map(promiseFn => cancelableResolve(promiseFn)))
@ -318,9 +343,9 @@ export default class MessagesPuppeteer {
const text = messageSyncElement.innerText const text = messageSyncElement.innerText
return text.startsWith("Syncing messages...") return text.startsWith("Syncing messages...")
&& (text.endsWith("100%") || text.endsWith("NaN%")) && (text.endsWith("100%") || text.endsWith("NaN%"))
// TODO Sometimes it gets stuck at 99%...?? // TODO Sometimes it gets stuck at 99%...??
}, },
{timeout: 10000}, // Assume 10 seconds is long enough { timeout: 10000 }, // Assume 10 seconds is long enough
messageSyncElement) messageSyncElement)
} catch (err) { } catch (err) {
//this._sendLoginFailure(`Failed to sync: ${err}`) //this._sendLoginFailure(`Failed to sync: ${err}`)
@ -582,7 +607,7 @@ export default class MessagesPuppeteer {
// - the most recently-sent own message is not fully read // - the most recently-sent own message is not fully read
let chatIDToSync let chatIDToSync
for (let i = 0, n = chatList.length; i < n; i++) { for (let i = 0, n = chatList.length; i < n; i++) {
const chatListItem = chatList[(i+offset) % n] const chatListItem = chatList[(i + offset) % n]
if (chatListItem.notificationCount > 0) { if (chatListItem.notificationCount > 0) {
// Chat has unread notifications, so don't view it // Chat has unread notifications, so don't view it
@ -634,15 +659,15 @@ export default class MessagesPuppeteer {
const initialID = this.jiggleTimerID const initialID = this.jiggleTimerID
exec(`xdotool mousemove --sync --window ${this.windowID} 0 0`, {}, exec(`xdotool mousemove --sync --window ${this.windowID} 0 0`, {},
(error, stdout, stderr) => { (error, stdout, stderr) => {
if (error) { if (error) {
this.log(`Error while jiggling mouse: ${error}`) this.log(`Error while jiggling mouse: ${error}`)
} }
if (this.jiggleTimerID == initialID) { if (this.jiggleTimerID == initialID) {
this._jiggleTimerStart() this._jiggleTimerStart()
} }
}) })
} }
async startObserving() { async startObserving() {
@ -693,8 +718,8 @@ export default class MessagesPuppeteer {
await this._interactWithPage(async () => { await this._interactWithPage(async () => {
this.log("Opening settings view") this.log("Opening settings view")
await this.page.click("button.mdGHD01SettingBtn") await this.page.click("button.mdGHD01SettingBtn")
await this.page.waitForSelector("#context_menu li#settings", {visible: true}).then(e => e.click()) await this.page.waitForSelector("#context_menu li#settings", { visible: true }).then(e => e.click())
await this.page.waitForSelector("#settings_contents", {visible: true}) await this.page.waitForSelector("#settings_contents", { visible: true })
this.log("Getting own profile info") this.log("Getting own profile info")
ownProfile = { ownProfile = {
@ -712,7 +737,7 @@ export default class MessagesPuppeteer {
const backSelector = "#label_setting button" const backSelector = "#label_setting button"
await this.page.click(backSelector) await this.page.click(backSelector)
await this.page.waitForSelector(backSelector, {visible: false}) await this.page.waitForSelector(backSelector, { visible: false })
}) })
return ownProfile return ownProfile
} }
@ -729,6 +754,41 @@ export default class MessagesPuppeteer {
return `#joined_group_list_body > li[data-chatid="${id}"]` return `#joined_group_list_body > li[data-chatid="${id}"]`
} }
_contactCountSelector() {
return `#contact_wrap_friends .MdLFT04Head._contactlist_header span[id=contact_friend_count]`
}
async _getWholeContactResults(page, distance) {
await page.bringToFront()
await page.waitForSelector("#contact_wrap_friends > ul.MdCMN03List")
const contactTotalCount = 0
this.log(` distance is ${distance}`)
let incred = 0
const element = await page.$(this._contactCountSelector())
const foundContactCount = await element.evaluate(
element => {
return Number.parseInt(element?.innerText) || 0
})
this.log(` contact_friend_count is ${foundContactCount}`)
//infiniting contact list scrolling
while (incred <= distance) {
if (foundContactCount <= contactTotalCount) {
return
}
incred += 100
this.log(`scrollPosition at ${incred}/${distance}`)
await page.evaluate(d => {
const scrollableSection = document.querySelector("#contact_mode_contact_list");
scrollableSection.scrollTop = 300 + scrollableSection.offsetHeight + d;
}, incred);
}
const $lis = await page.$$("#contact_wrap_friends > ul.MdCMN03List >li")
//lookup lazy loading count
const lis = $lis.slice(contactTotalCount, Math.Infinity)
contactTotalCount = $lis.length
}
async _switchChat(chatID, forceView = false) { async _switchChat(chatID, forceView = false) {
// TODO Allow passing in an element directly // TODO Allow passing in an element directly
this.log(`Switching to chat ${chatID}`) this.log(`Switching to chat ${chatID}`)
@ -777,7 +837,30 @@ export default class MessagesPuppeteer {
} }
chatItem = await this.page.$(this._groupItemSelector(chatID)) chatItem = await this.page.$(this._groupItemSelector(chatID))
} else { } else {
needRealClick = true
const unselectedTabButton = await this.page.$(`#leftSide li[data-type=friends_list] > button:not(.ExSelected)`)
if (unselectedTabButton) {
switchedTabs = true
await unselectedTabButton.evaluate(e => e.click())
await this.page.waitForSelector("#wrap_contact_list > div.MdScroll")
let ulstyleheight = await this.page.evaluate(() => {
const ulList = document.querySelector('#contact_wrap_friends > ul.MdCMN03List')
return getComputedStyle(ulList).getPropertyValue("height")
})
const leg = parseInt(ulstyleheight, 10)
this.log(`found contact_wrap_friends height is ${leg}px`)
this.log(`starting to scroll to buttom`)
await this._getWholeContactResults(this.page, leg)
this.log(`finished to scroll to buttom`)
}
await this.page.waitForTimeout(300)
chatItem = await this.page.$(this._friendItemSelector(chatID)) chatItem = await this.page.$(this._friendItemSelector(chatID))
this.log(`Chat ${chatID} not in recents list, so bot is creating a new chat`)
await this._interactWithPage(async () => {
await chatItem.click()
})
} }
if (!chatItem) { if (!chatItem) {
@ -788,8 +871,8 @@ export default class MessagesPuppeteer {
// HTML of friend/group item titles ever diverge // HTML of friend/group item titles ever diverge
chatName = await chatItem.evaluate( chatName = await chatItem.evaluate(
chatID.charAt(0) == "u" chatID.charAt(0) == "u"
? element => window.__mautrixController.getFriendsListItemName(element) ? element => window.__mautrixController.getFriendsListItemName(element)
: element => window.__mautrixController.getGroupListItemName(element)) : element => window.__mautrixController.getGroupListItemName(element))
} }
await this._retryUntilSuccess(3, "Clicking chat item didn't work...try again", await this._retryUntilSuccess(3, "Clicking chat item didn't work...try again",
@ -805,7 +888,7 @@ export default class MessagesPuppeteer {
this.log(`Waiting for chat header title to be "${chatName}"`) this.log(`Waiting for chat header title to be "${chatName}"`)
await this.page.waitForFunction( await this.page.waitForFunction(
isCorrectChatVisible, isCorrectChatVisible,
{polling: "mutation", timeout: 1000}, { polling: "mutation", timeout: 1000 },
chatName) chatName)
}) })
if (switchedTabs) { if (switchedTabs) {
@ -825,13 +908,13 @@ export default class MessagesPuppeteer {
this.log("Clicking chat header to show detail area") this.log("Clicking chat header to show detail area")
await this.page.click("#_chat_header_area > .mdRGT04Link") await this.page.click("#_chat_header_area > .mdRGT04Link")
this.log("Waiting for detail area") this.log("Waiting for detail area")
await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info", {timeout: 1000}) await this.page.waitForSelector("#_chat_detail_area > .mdRGT02Info", { timeout: 1000 })
}) })
}) })
this.log("Waiting for any item to appear in chat") this.log("Waiting for any item to appear in chat")
try { try {
await this.page.waitForSelector("#_chat_room_msg_list div", {timeout: 2000}) await this.page.waitForSelector("#_chat_room_msg_list div", { timeout: 2000 })
this.log("Waiting for chat to stabilize") this.log("Waiting for chat to stabilize")
await this.page.evaluate(() => window.__mautrixController.waitForMessageListStability()) await this.page.evaluate(() => window.__mautrixController.waitForMessageListStability())
@ -852,17 +935,17 @@ export default class MessagesPuppeteer {
async _getChatInfoUnsafe(chatID, forceView) { async _getChatInfoUnsafe(chatID, forceView) {
// TODO Commonize this // TODO Commonize this
let [isDirect, isGroup, isRoom] = [false,false,false] let [isDirect, isGroup, isRoom] = [false, false, false]
switch (chatID.charAt(0)) { switch (chatID.charAt(0)) {
case "u": case "u":
isDirect = true isDirect = true
break break
case "c": case "c":
isGroup = true isGroup = true
break break
case "r": case "r":
isRoom = true isRoom = true
break break
} }
const chatListItem = await this.page.$(this._chatItemSelector(chatID)) const chatListItem = await this.page.$(this._chatItemSelector(chatID))
@ -924,7 +1007,7 @@ export default class MessagesPuppeteer {
for (const participant of participants) { for (const participant of participants) {
this.log(JSON.stringify(participant)) this.log(JSON.stringify(participant))
} }
return {participants, ...chatListInfo} return { participants, ...chatListInfo }
} }
// TODO Catch "An error has occurred" dialog // TODO Catch "An error has occurred" dialog
@ -951,7 +1034,7 @@ export default class MessagesPuppeteer {
await input.press("Enter") await input.press("Enter")
await this.page.waitForFunction( await this.page.waitForFunction(
e => e.innerText == "", e => e.innerText == "",
{timeout: 500}, { timeout: 500 },
input) input)
}) })
}) })
@ -1043,14 +1126,14 @@ export default class MessagesPuppeteer {
// Sync receipts seen from newly-synced messages // Sync receipts seen from newly-synced messages
// TODO When user leaves, clear the read-by count for the old number of other participants // TODO When user leaves, clear the read-by count for the old number of other participants
let minCountToFind = 1 let minCountToFind = 1
for (let i = messages.length-1; i >= 0; i--) { for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i] const message = messages[i]
if (!message.is_outgoing) { if (!message.is_outgoing) {
continue continue
} }
const count = message.receipt_count const count = message.receipt_count
if (count >= minCountToFind && message.id > (receiptMap.get(count) || 0)) { if (count >= minCountToFind && message.id > (receiptMap.get(count) || 0)) {
minCountToFind = count+1 minCountToFind = count + 1
receiptMap.set(count, message.id) receiptMap.set(count, message.id)
} }
// TODO Early exit when count == num other participants // TODO Early exit when count == num other participants
@ -1150,12 +1233,12 @@ export default class MessagesPuppeteer {
// If >1, a notification was missed. Only way to get them is to view the chat. // If >1, a notification was missed. Only way to get them is to view the chat.
// If == 0, might be own message...or just a shuffled chat, or something else. // If == 0, might be own message...or just a shuffled chat, or something else.
// To play it safe, just sync them. Should be no harm, as they're viewed already. // To play it safe, just sync them. Should be no harm, as they're viewed already.
diffNumNotifications != 1 diffNumNotifications != 1
// Without placeholders, some messages require visiting their chat to be synced. // Without placeholders, some messages require visiting their chat to be synced.
|| !this.sendPlaceholders || !this.sendPlaceholders
&& ( && (
// Can only use previews for DMs, because sender can't be found otherwise! // Can only use previews for DMs, because sender can't be found otherwise!
chatListInfo.id.charAt(0) != 'u' chatListInfo.id.charAt(0) != 'u'
// Sync when lastMsg is a canned message for a non-previewable message type. // Sync when lastMsg is a canned message for a non-previewable message type.
|| chatListInfo.lastMsg.endsWith(" sent a photo.") || chatListInfo.lastMsg.endsWith(" sent a photo.")
|| chatListInfo.lastMsg.endsWith(" sent a sticker.") || chatListInfo.lastMsg.endsWith(" sent a sticker.")
@ -1182,7 +1265,7 @@ export default class MessagesPuppeteer {
} }
async _syncChat(chatID) { async _syncChat(chatID) {
const {messages, receipts} = await this._getMessagesUnsafe(chatID) const { messages, receipts } = await this._getMessagesUnsafe(chatID)
if (messages.length == 0) { if (messages.length == 0) {
this.log("No new messages found in", chatID) this.log("No new messages found in", chatID)
@ -1219,7 +1302,7 @@ export default class MessagesPuppeteer {
this.log(`Received read receipt ${receipt_id} (since ${prevReceiptID}) for chat ${chat_id}`) this.log(`Received read receipt ${receipt_id} (since ${prevReceiptID}) for chat ${chat_id}`)
if (this.client) { if (this.client) {
this.client.sendReceipt({chat_id: chat_id, id: receipt_id}) this.client.sendReceipt({ chat_id: chat_id, id: receipt_id })
.catch(err => this.error("Error handling read receipt:", err)) .catch(err => this.error("Error handling read receipt:", err))
} else { } else {
this.log("No client connected, not sending receipts") this.log("No client connected, not sending receipts")
@ -1252,7 +1335,7 @@ export default class MessagesPuppeteer {
receipt.chat_id = chat_id receipt.chat_id = chat_id
try { try {
await this.client.sendReceipt(receipt) await this.client.sendReceipt(receipt)
} catch(err) { } catch (err) {
this.error("Error handling read receipt:", err) this.error("Error handling read receipt:", err)
} }
} }