Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,9 @@ import kotlin.math.abs
* one as its new text said, a live label as one live node, a link followed by the engine, by its action or a tap, with
* its href as Qt reads it, a `prompt:` one giving the input line the focus, but a web link opened in the browser, a
* click-through label's links neither tapped nor actions, also once a label already drawn is made click-through, and
* both again once it no longer is, a label's own colour drawn only where its sheet draws no box of its own, and the
* images the engine is asked for.
* both again once it no longer is, a label's own colour drawn only where its sheet draws no box of its own, the
* images the engine is asked for, an image and a heading in a label's text, and a label of only a linked image read
* by its alt, and one of only images, white space between them or not, by its tool tip.
*/
@RunWith(AndroidJUnit4::class)
class ScriptLabelsTest {
Expand Down Expand Up @@ -586,6 +587,71 @@ class ScriptLabelsTest {
compose.onRoot().tryPerformAccessibilityChecks()
}

@Test
fun aLabelsTextDrawsItsImagesAndHeadings() {
// Below the main console's line, on black, so only the text and the image are drawn in a colour
val sheet = "QLabel { background-color: black; color: white; font-size: 10pt }"
val labels = listOf(label("Heading", 0, 300, 300, 100), label("Body", 400, 300, 300, 100), label("Pictured", 0, 450, 300, 100), label("Small", 400, 450, 300, 100))
session.windows(labels, full = true)
session.label("Heading", text = "<h1>HH</h1>", accessibleName = "Heading", styleSheet = sheet)
session.label("Body", text = "HH", accessibleName = "Body", styleSheet = sheet)
session.label("Pictured", text = "a<img src=\"/data/dot.png\" width=\"40\" height=\"20\">b", styleSheet = sheet)
// The same image in the smallest font size, as Qt reads a `font size` with a unit: drawn at the same size
session.label("Small", text = "c<font size=\"0px\"><img src=\"/data/dot.png\" width=\"40\" height=\"20\"></font>d", styleSheet = sheet)
// Each read as its text without the image; the test's clock recomposes only while the test waits on Compose, so
// the labels are drawn, and ask for their image, first
compose.waitUntil(WAIT_MS) { listOf("ab", "cd").all { compose.onAllNodes(hasText(it)).fetchSemanticsNodes().isNotEmpty() } }
// The image is asked of the engine at its own size
val request = session.receive("getImage")
assertEquals("/data/dot.png", request.getString("path"))
assertFalse(request.toString(), request.has("w") || request.has("frame"))
session.image("/data/dot.png", png(4, 2, android.graphics.Color.RED), 4, 2, request.get("id"))
compose.waitUntil(WAIT_MS) { EngineClient.front.windows.images[ScriptWindowsClient.ImageRequest("/data/dot.png")] != null }
compose.waitForIdle()

// Drawn in the line at the size its attributes give, 40 by 20 of the label's 300 by 100 pixels, in both fonts
for (shown in listOf("ab", "cd")) {
val pictured = compose.onNode(hasText(shown)).captureToImage().toPixelMap()
val red = (0 until pictured.height).flatMap { y -> (0 until pictured.width).map { x -> x to y } }.filter { (x, y) ->
val c = pictured[x, y]
c.red > 0.8f && c.green < 0.2f && c.blue < 0.2f
}
assertTrue("$shown: no image drawn", red.isNotEmpty())
// Scaled as the label's text is, by the less of its window's two scales
val scale = minOf(pictured.width / 300f, pictured.height / 100f)
assertNear("$shown: image width", 40 * scale, (red.maxOf { it.first } - red.minOf { it.first } + 1).toFloat())
assertNear("$shown: image height", 20 * scale, (red.maxOf { it.second } - red.minOf { it.second } + 1).toFloat())
}

// A heading's text is taller than the same text in the body, as Qt draws an h1 at twice the label's size
fun inkRows(name: String): Int {
val pixels = compose.onNode(hasContentDescription(name)).captureToImage().toPixelMap()
return (0 until pixels.height).count { y -> (0 until pixels.width).any { x -> pixels[x, y].red > 0.5f } }
}
val heading = inkRows("Heading")
val body = inkRows("Body")
assertTrue("heading $heading rows, body $body", body > 0 && heading > body * 3 / 2)
compose.onRoot().tryPerformAccessibilityChecks()
}

@Test
fun aLabelOfOnlyALinkedImageIsReadByItsAltWithItsLinkAnAction() {
session.windows(listOf(label("eye", w = 100, h = 40), label("lid", y = 50, w = 100, h = 40)), full = true)
session.label("eye", text = """<a href="send:look"><img src="/data/eye.png" alt="Look"></a>""")
// Not left out as decoration: read by the image's alt, its link an action that has the engine follow it
compose.waitUntil(WAIT_MS) { compose.onAllNodes(hasText("Look")).fetchSemanticsNodes().isNotEmpty() }
val follow = string(R.string.label_follow_link, "send:look")
val action = compose.onNode(hasText("Look")).fetchSemanticsNode().config[SemanticsActions.CustomActions].single { it.label == follow }
compose.runOnIdle { action.action() }
assertEquals("send:look", session.receive("followLabelLink").getString("href"))
// One of only an image that is no link, read by its tool tip, as Mudlet reads it, not left out as decoration
session.label("lid", text = """<img src="/data/eye.png">""", toolTip = "Peek")
compose.waitUntil(WAIT_MS) { compose.onAllNodes(hasText("Peek")).fetchSemanticsNodes().isNotEmpty() }
// So too with white space between its images, which Mudlet simplifies away before it reads the text
session.label("lid", text = """<img src="/data/eye.png"> <img src="/data/eye.png">""", toolTip = "Both")
compose.waitUntil(WAIT_MS) { compose.onAllNodes(hasText("Both")).fetchSemanticsNodes().isNotEmpty() }
}

// The pointer events of one tap, up to its release
private fun tap(): List<JSONObject> {
val events = ArrayList<JSONObject>()
Expand All @@ -606,9 +672,9 @@ class ScriptLabelsTest {

private fun label(name: String, x: Int = 0, y: Int = 0, w: Int, h: Int) = FakeEngine.window(name, kind = "label", x = x, y = y, w = w, h = h)

private fun png(w: Int, h: Int): ByteArray {
private fun png(w: Int, h: Int, colour: Int = 0): ByteArray {
val out = ByteArrayOutputStream()
Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888).compress(Bitmap.CompressFormat.PNG, 100, out)
Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888).apply { eraseColor(colour) }.compress(Bitmap.CompressFormat.PNG, 100, out)
return out.toByteArray()
}

Expand Down
73 changes: 63 additions & 10 deletions app/src/main/kotlin/org/mudlet/android/ui/ScriptWindowsClient.kt
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,16 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.withContext
import org.mudlet.android.protocol.AddonCommand
import org.mudlet.android.protocol.ButtonBar
import org.mudlet.android.protocol.EngineMessage
import org.mudlet.android.protocol.KeyBinding
import org.mudlet.android.protocol.KeyPressResult
import org.mudlet.android.protocol.LabelLink
import org.mudlet.android.protocol.LabelStyle
import org.mudlet.android.protocol.LayoutMode
import org.mudlet.android.protocol.MapInfo
import org.mudlet.android.protocol.MapPoint
Expand All @@ -28,6 +32,7 @@ import org.mudlet.android.protocol.POINTER_LEFT
import org.mudlet.android.protocol.Placement
import org.mudlet.android.protocol.PointerType
import org.mudlet.android.protocol.Requests
import org.mudlet.android.protocol.RichText
import org.mudlet.android.protocol.ScriptWindows
import org.mudlet.android.protocol.TextBoxContent

Expand Down Expand Up @@ -93,8 +98,12 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi
// Where this engine instance was last told each user window is
private val sentPlacements = HashMap<String, Placement>()

/** An image file of the engine's, as a label shows it: `getImage`'s fields. */
data class ImageRequest(val path: String, val w: Int? = null, val h: Int? = null, val frame: Int? = null) {
/**
* An image file of the engine's, as a label shows it: `getImage`'s fields; or, [decoded], the image of the `data:`
* URL [path] in a label's text, which the app decodes itself ([decodeImage]) and the engine is never asked for, so
* a style sheet's `url()` of the same text, which the engine refuses as Qt's style sheet draws none, is apart.
*/
data class ImageRequest(val path: String, val w: Int? = null, val h: Int? = null, val frame: Int? = null, val decoded: Boolean = false) {
init {
require((w == null) == (h == null)) { "an image is asked for at a size with both w and h, or neither" }
}
Expand All @@ -107,10 +116,14 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi
*/
class MapImage(val bitmap: ImageBitmap?, val w: Int, val h: Int, val viewOnly: Boolean, val info: MapInfo? = null)

/** The images the engine sent, by what was asked; one not there yet is asked for with [requestImage]. */
/**
* The images the engine sent, and those the app decoded of `data:` URLs ([decodeImage]), by what was asked; one not
* there yet is asked for with [requestImage].
*/
val images = mutableStateMapOf<ImageRequest, ImageBitmap>()

// What was asked and not yet answered, by request id, and what of this engine instance's is kept or awaited
// What was asked and not yet answered, by request id, and what of this engine instance's is kept or awaited, or
// being decoded
private val pendingImages = HashMap<Long, ImageRequest>()
private val askedImages = HashSet<ImageRequest>()

Expand Down Expand Up @@ -192,9 +205,27 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi
val request = pendingImages.remove((message.replyTo as? Number)?.toLong()) ?: return
if (bitmap == null) {
// Not asked again: the same file would not decode better
Log.w(TAG, "the engine's image ${request.path} did not decode")
Log.w(TAG, "an image of the engine's did not decode")
return
}
keep(request, bitmap)
}

/**
* Decodes [request]'s `data:` URL, [ImageRequest.decoded], off the main thread into [images], where the engine's
* files are kept and let go: once, as [requestImage] asks once, and to its end, so each label that shows it gets it.
* One that does not decode is forgotten, so script-made URLs that fail do not pile up; it is decoded again only when
* a label shows it afresh.
*/
suspend fun decodeImage(request: ImageRequest) {
if (request in images || !askedImages.add(request)) return
withContext(NonCancellable) {
val bitmap = withContext(Dispatchers.Default) { decodeData(request.path) }
if (bitmap != null) keep(request, bitmap) else askedImages.remove(request)
}
}

private fun keep(request: ImageRequest, bitmap: ImageBitmap) {
images[request] = bitmap
imageOrder.add(request)
trimImages()
Expand All @@ -211,15 +242,16 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi
trimImages()
}

// Lets go of the oldest images no label shows while there are more than MAX_IMAGES; one let go is asked again
// when a label shows it
// Lets go of the oldest images no label shows while there are more than MAX_IMAGES, or those no label shows hold
// more than MAX_IMAGE_BYTES; one let go is asked again when a label shows it
private fun trimImages() {
var bytes = images.entries.sumOf { (request, image) -> if (request in shownImages) 0L else image.width * image.height * 4L }
val unshown = imageOrder.iterator()
while (images.size > MAX_IMAGES && unshown.hasNext()) {
while ((images.size > MAX_IMAGES || bytes > MAX_IMAGE_BYTES) && unshown.hasNext()) {
val request = unshown.next()
if (request in shownImages) continue
unshown.remove()
images.remove(request)
images.remove(request)?.let { bytes -= it.width * it.height * 4L }
askedImages.remove(request)
}
}
Expand Down Expand Up @@ -451,7 +483,24 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi
/** [message]'s PNG, decoded off the main thread; null for one that does not decode. */
internal fun decode(message: EngineMessage.Map): ImageBitmap? = decodePng(message.png)

private fun decodePng(png: ByteArray): ImageBitmap? = BitmapFactory.decodeByteArray(png, 0, png.size)?.asImageBitmap()
// An image's bytes decoded, null for bytes that do not decode or an image too large for the memory left
private fun decodePng(png: ByteArray): ImageBitmap? = try {
BitmapFactory.decodeByteArray(png, 0, png.size)?.asImageBitmap()
} catch (_: OutOfMemoryError) {
null
}

// A `data:` URL's image, null as for the engine's, and for one wider or higher than the engine sends one
// (`PngImage::maxSide`), found from its size alone first; its base64 read too within the memory left
private fun decodeData(path: String): ImageBitmap? = try {
RichText.Image(path, null, null).data()?.let { bytes ->
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds)
if (maxOf(bounds.outWidth, bounds.outHeight) > LabelStyle.MAX_LENGTH) null else decodePng(bytes)
}
} catch (_: OutOfMemoryError) {
null
}

private const val TAG = "ScriptWindows"

Expand All @@ -460,5 +509,9 @@ class ScriptWindowsClient internal constructor(private val session: ProfileSessi

// The images kept beyond those shown: few enough that sized ones asked again as labels resize do not pile up
private const val MAX_IMAGES = 64

// And the bytes those hold at most, as images in a label's text are kept at their own size: one of the
// largest the engine sends, 4096 pixels a side
private const val MAX_IMAGE_BYTES = 64L shl 20
}
}
Loading
Loading