[WIP] add support for images

this crashes because decode_image calls
`client.media().get_file(content, /* use_cache */ true).await` which
then causes `poll_updates` to be called again while this `poll_updates`
is still running, causing inconsistent updates when doing
`.apply(&mut self.items)`.

Cargo.toml

@@ -8,11 +8,19 @@ repository = "https://git.tf/val/ratatrix"
 authors = ["Val Lorentz"]
 license = "AGPL-3.0-only AND MIT"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = [
+    "images",
+]
+images = [
+    "dep:image",
+    "dep:ratatui-image",
+]
 
 [dependencies]
 
 # Async
+async-std = { version = "1.12.0", features = ["tokio1", "attributes"] }
 futures = "0.3.28"
 tokio = { version = "1.32.0", features = ["full"] }
 tokio-util = "0.7.9"
@@ -64,8 +72,8 @@ sorted-vec = "0.8.3"
 eyeball = "0.8.7" # data structures observer returned by matrix-sdk-ui
 eyeball-im = "0.4.2" # immutable data structures observer returned by matrix-sdk-ui
 imbl = "2.0" # ditto
-matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2", features = ["eyre", "markdown"] }
-matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2" }
+matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4", features = ["eyre", "markdown"] }
+matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4" }
 #matrix-sdk = { path = "../matrix-rust-sdk/crates/matrix-sdk", features = ["eyre", "markdown"] }
 #matrix-sdk-ui = { path = "../matrix-rust-sdk/crates/matrix-sdk-ui" }
 
@@ -78,8 +86,13 @@ strip-ansi-escapes = "0.2.0"
 tui-textarea = "0.3.0"
 unicode-width = "0.1"
 
+# UI (images)
+ratatui-image = { version = "0.8.0", optional = true }
+image = { version = "0.24.8", optional = true }
+
 [patch.crates-io]
 #ratatui = { path = "../ratatui", features = ["serde", "macros"] }
+eyeball-im = { path = "../eyeball/eyeball-im" }
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.73.0"
+channel = "1.76.0"

src/buffers/mod.rs

@@ -32,7 +32,7 @@ use crate::widgets::Prerender;
 
 mod log;
 pub use log::LogBuffer;
-mod room;
+pub(crate) mod room;
 pub use room::RoomBuffer;
 
 /// Maximum time before reordering the buffer list based on parent/child relationships.
@@ -105,6 +105,10 @@ pub enum BufferItemContent<'buf> {
   SimpleText(Text<'buf>),
   /// Pairs of `(padding, content)`
   Text(Vec<(String, Text<'buf>)>),
+  #[cfg(feature = "images")]
+  Image {
+    image: &'buf image::DynamicImage,
+  },
   Divider(Text<'buf>),
   Empty,
 }

src/buffers/room.rs

@@ -20,18 +20,19 @@ use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::{Arc, OnceLock};
 
 use chrono::{offset::Local, DateTime};
-use color_eyre::eyre::{eyre, Result};
+use color_eyre::eyre::{eyre, Result, WrapErr};
 use eyeball_im::VectorDiff;
 use futures::future::OptionFuture;
 use futures::stream::FuturesUnordered;
 use futures::{FutureExt, Stream, StreamExt};
+use image::{DynamicImage, ImageResult};
 use itertools::Itertools;
 use matrix_sdk::async_trait;
 use matrix_sdk::deserialized_responses::SyncOrStrippedState;
 use matrix_sdk::room::ParentSpace;
 use matrix_sdk::ruma::events::fully_read::FullyReadEventContent;
 use matrix_sdk::ruma::events::room::message::{
-  FormattedBody, MessageFormat, MessageType, TextMessageEventContent,
+  FormattedBody, ImageMessageEventContent, MessageFormat, MessageType, TextMessageEventContent,
 };
 use matrix_sdk::ruma::events::space::child::SpaceChildEventContent;
 use matrix_sdk::ruma::events::RoomAccountDataEvent;
@@ -69,6 +70,12 @@ pub enum OwnedBufferItemContent {
     /// `(padding, content)` pairs
     text: Vec<(String, Text<'static>)>,
   },
+  #[cfg(feature = "images")]
+  Image {
+    event_id: Option<OwnedEventId>,
+    is_message: bool,
+    image: DynamicImage,
+  },
   Divider(String),
   Empty,
 }
@@ -86,6 +93,7 @@ impl DynamicUsage for OwnedBufferItemContent {
             .map(|item| item.1.width() * item.1.height() * 4)
             .sum() // FIXME: rough approx
         },
+        OwnedBufferItemContent::Image { .. } => 10 * 1024 * 1024, // FIXME: that's just the maximum size
         OwnedBufferItemContent::Divider(s) => s.dynamic_usage(),
         OwnedBufferItemContent::Empty => 0,
       }
@@ -103,6 +111,7 @@ impl DynamicUsage for OwnedBufferItemContent {
           .sum();
         (area, Some(area * 12)) // FIXME: rough approx
       },
+      OwnedBufferItemContent::Image { .. } => (0, Some(10 * 1024 * 1024)), // FIXME: that's just the bounds
       OwnedBufferItemContent::Divider(s) => s.dynamic_usage_bounds(),
       OwnedBufferItemContent::Empty => (0, Some(0)),
     };
@@ -177,8 +186,8 @@ impl SingleClientRoomBuffer {
       changes = self.timeline_stream.next() => {
         let Some(changes) = changes else { return None; };
         for change in changes {
-          change
-            .map(|item| (Some(item.unique_id()), self.format_timeline_item(item), Prerender::new()))
+          change.async_map(|item| self.make_timeline_item(item))
+            .await
             .apply(&mut self.items);
         }
         self.latest_event_id = self.timeline.latest_event().await.and_then(|e| e.event_id().map(ToOwned::to_owned));
@@ -187,7 +196,21 @@ impl SingleClientRoomBuffer {
     }
   }
 
-  fn format_timeline_item(&self, tl_item: impl AsRef<TimelineItem>) -> OwnedBufferItemContent {
+  async fn make_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> (Option<u64>, OwnedBufferItemContent, Prerender) {
+    (
+      Some(tl_item.as_ref().unique_id()),
+      self.format_timeline_item(tl_item).await,
+      Prerender::new(),
+    )
+  }
+
+  async fn format_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> OwnedBufferItemContent {
     match tl_item.as_ref().kind() {
       TimelineItemKind::Event(event) => {
         use matrix_sdk_ui::timeline::TimelineItemContent::*;
@@ -241,6 +264,17 @@ impl SingleClientRoomBuffer {
             MessageType::Text(TextMessageEventContent { body, .. }) => {
               msg!("   ", "&lt;{}> {}", sender, escape_html(body))
             },
+            #[cfg(feature = "images")]
+            MessageType::Image(content) => {
+              crate::images::decode_image(
+                &self.client,
+                &self.config,
+                event.event_id().map(ToOwned::to_owned),
+                &sender,
+                content,
+              )
+              .await
+            },
             _ =>
             // Fallback to text body
             {
@@ -532,7 +566,12 @@ impl RoomBuffer {
       );
       eyre!("Unknown room {} for client {:?}", self.room_id, client)
     })?;
-    let timeline = room.timeline_builder().build().await;
+    let timeline = room.timeline_builder().build().await.with_context(|| {
+      format!(
+        "Could not get timeline for {:?} from {:?}",
+        self.room_id, client
+      )
+    })?;
     let (items, timeline_stream) = timeline.subscribe_batched().await;
     tracing::info!(
       "Added client for {}, initial items: {:?}",
@@ -700,6 +739,7 @@ impl Buffer for RoomBuffer {
               BufferItemContent::SimpleText(text.clone())
             },
             OwnedBufferItemContent::Text { text, .. } => BufferItemContent::Text(text.clone()),
+            OwnedBufferItemContent::Image { image, .. } => BufferItemContent::Image { image },
             OwnedBufferItemContent::Divider(text) => BufferItemContent::Divider(Text::raw(text)),
             OwnedBufferItemContent::Empty => BufferItemContent::Empty,
           },

src/components/backlog.rs

@@ -140,6 +140,8 @@ impl Backlog {
       )
       .scroll(scroll)
       .into(),
+      #[cfg(feature = "images")]
+      BufferItemContent::Image { image } => todo!("render image"),
       BufferItemContent::Divider(text) => {
         if scroll == 0 {
           Divider::new(Paragraph::new(text).alignment(Alignment::Center)).into()

src/images.rs

@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2023  Valentin Lorentz
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use std::io::Cursor;
+use std::sync::Arc;
+
+use matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
+use matrix_sdk::ruma::{OwnedEventId, OwnedRoomId, RoomId};
+use matrix_sdk::Client;
+
+use crate::buffers::room::OwnedBufferItemContent;
+use crate::config::Config;
+use crate::html::{escape_html, format_html};
+
+pub(crate) async fn decode_image(
+  client: &Client,
+  config: &Config,
+  event_id: Option<OwnedEventId>,
+  sender: &str,
+  content: &ImageMessageEventContent,
+) -> OwnedBufferItemContent {
+  // Like `format!()` but returns OwnedBufferItemContent::Text, with is_message=true
+  macro_rules! msg {
+    ($prefix: expr, $($tokens:tt)*) => {
+      OwnedBufferItemContent::Text {
+        event_id,
+        is_message: true,
+        text: format_html(&config, $prefix, &format!($($tokens)*))
+      }
+    }
+  }
+
+  let ImageMessageEventContent { body, info, .. } = content;
+  match client.media().get_file(content, /* use_cache */ true).await {
+    Ok(Some(file)) => {
+
+      let mut reader = match info
+        .as_ref()
+        .and_then(|info| info.mimetype.as_ref())
+        .and_then(|mimetype| image::ImageFormat::from_mime_type(mimetype))
+      {
+        Some(format) => {
+          // Known format provided by the sender
+          image::io::Reader::with_format(Cursor::new(file), format)
+        },
+        None => {
+          // Unknown format or not provided by the sender, try guessing it
+          let reader = image::io::Reader::new(Cursor::new(file));
+          match reader.with_guessed_format() {
+            Ok(reader) => reader,
+            Err(e) => {
+              log::warn!("Could not guess image format: {:?}", e);
+              return msg!("   ", "&lt;{}> {}", sender, escape_html(body));
+            },
+          }
+        },
+      };
+
+      // Arbitrary values to avoid DoS. TODO: make them configurable
+      let mut limits = image::io::Limits::default();
+      limits.max_image_width = Some(1024);
+      limits.max_image_height = Some(1024);
+      limits.max_alloc = Some(10 * 1024 * 1024);
+      reader.limits(limits);
+
+      let image = match reader.decode() {
+        Ok(image) => image,
+        Err(e) => {
+          log::warn!("Could not decode image: {:?}", e);
+          return msg!("   ", "&lt;{}> {}", sender, escape_html(body));
+        },
+      };
+/*
+      OwnedBufferItemContent::Image {
+        event_id,
+        is_message: true,
+        image,
+      }*/
+      msg!("   ", "&lt;{}> IMAGE {}", sender, escape_html(body))
+    },
+    Ok(None) => msg!("   ", "&lt;{}> {}", sender, escape_html(body)),
+    Err(e) => {
+      log::error!("Could not get image file: {:?}", e);
+      msg!("   ", "&lt;{}> {}", sender, escape_html(body))
+    },
+  }
+}

src/lib.rs

@@ -10,6 +10,8 @@ pub mod commands;
 pub mod components;
 pub mod config;
 pub mod html;
+#[cfg(feature = "images")]
+pub mod images;
 pub mod log;
 pub mod mode;
 pub mod plugins;

src/main.rs

@@ -15,7 +15,7 @@ async fn tokio_main() -> Result<()> {
   Ok(())
 }
 
-#[tokio::main]
+#[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<()> {
   if let Err(e) = tokio_main().await {
     eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME"));