Actually render images

Cargo.toml

@@ -8,11 +8,19 @@ repository = "https://git.tf/val/ratatrix"
 authors = ["Val Lorentz"]
 license = "AGPL-3.0-only AND MIT"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = [
+    "images",
+]
+images = [
+    "dep:image",
+    "dep:ratatui-image",
+]
 
 [dependencies]
 
 # Async
+async-std = { version = "1.12.0", features = ["tokio1", "attributes"] }
 futures = "0.3.28"
 tokio = { version = "1.32.0", features = ["full"] }
 tokio-util = "0.7.9"
@@ -64,8 +72,8 @@ sorted-vec = "0.8.3"
 eyeball = "0.8.7" # data structures observer returned by matrix-sdk-ui
 eyeball-im = "0.4.2" # immutable data structures observer returned by matrix-sdk-ui
 imbl = "2.0" # ditto
-matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2", features = ["eyre", "markdown"] }
-matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2" }
+matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4", features = ["eyre", "markdown"] }
+matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4" }
 #matrix-sdk = { path = "../matrix-rust-sdk/crates/matrix-sdk", features = ["eyre", "markdown"] }
 #matrix-sdk-ui = { path = "../matrix-rust-sdk/crates/matrix-sdk-ui" }
 
@@ -78,8 +86,13 @@ strip-ansi-escapes = "0.2.0"
 tui-textarea = "0.3.0"
 unicode-width = "0.1"
 
+# UI (images)
+ratatui-image = { version = "0.8.0", optional = true, features = ["crossterm"] }
+image = { version = "0.24.8", optional = true }
+
 [patch.crates-io]
 #ratatui = { path = "../ratatui", features = ["serde", "macros"] }
+eyeball-im = { path = "../eyeball/eyeball-im" }
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"

src/app.rs

@@ -53,6 +53,13 @@ pub struct App {
 
 impl App {
   pub async fn new(frame_rate: f64, log_receiver: mpsc::UnboundedReceiver<String>) -> Result<Self> {
+    // Forces PROTOCOL_PICKER to be initialized early, as it may get stuck if called
+    // after we started rendering.
+    #[cfg(feature = "images")]
+    log::info!(
+      "Using image protocol: {:?}",
+      crate::widgets::PROTOCOL_PICKER.protocol_type
+    );
     let config = Config::new()?;
     let datadir = config.config._data_dir.join("default");
     let future_clients = config.accounts.clone().map(|conf| {

src/buffers/mod.rs

@@ -32,7 +32,7 @@ use crate::widgets::Prerender;
 
 mod log;
 pub use log::LogBuffer;
-mod room;
+pub(crate) mod room;
 pub use room::RoomBuffer;
 
 /// Maximum time before reordering the buffer list based on parent/child relationships.
@@ -105,6 +105,10 @@ pub enum BufferItemContent<'buf> {
   SimpleText(Text<'buf>),
   /// Pairs of `(padding, content)`
   Text(Vec<(String, Text<'buf>)>),
+  #[cfg(feature = "images")]
+  Image {
+    image: &'buf image::DynamicImage,
+  },
   Divider(Text<'buf>),
   Empty,
 }

src/buffers/room.rs

@@ -20,19 +20,21 @@ use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::{Arc, OnceLock};
 
 use chrono::{offset::Local, DateTime};
-use color_eyre::eyre::{eyre, Result};
+use color_eyre::eyre::{eyre, Result, WrapErr};
 use eyeball::SharedObservable;
 use eyeball_im::VectorDiff;
+use futures::future::join_all;
 use futures::future::OptionFuture;
 use futures::stream::FuturesUnordered;
 use futures::{FutureExt, Stream, StreamExt};
+use image::{DynamicImage, ImageResult};
 use itertools::Itertools;
 use matrix_sdk::async_trait;
 use matrix_sdk::deserialized_responses::SyncOrStrippedState;
 use matrix_sdk::room::ParentSpace;
 use matrix_sdk::ruma::events::fully_read::FullyReadEventContent;
 use matrix_sdk::ruma::events::room::message::{
-  FormattedBody, MessageFormat, MessageType, TextMessageEventContent,
+  FormattedBody, ImageMessageEventContent, MessageFormat, MessageType, TextMessageEventContent,
 };
 use matrix_sdk::ruma::events::space::child::SpaceChildEventContent;
 use matrix_sdk::ruma::events::RoomAccountDataEvent;
@@ -72,6 +74,12 @@ pub enum OwnedBufferItemContent {
     /// `(padding, content)` pairs
     text: Vec<(String, Text<'static>)>,
   },
+  #[cfg(feature = "images")]
+  Image {
+    event_id: Option<OwnedEventId>,
+    is_message: bool,
+    image: DynamicImage,
+  },
   Divider(String),
   Empty,
 }
@@ -89,6 +97,7 @@ impl DynamicUsage for OwnedBufferItemContent {
             .map(|item| item.1.width() * item.1.height() * 4)
             .sum() // FIXME: rough approx
         },
+        OwnedBufferItemContent::Image { .. } => 10 * 1024 * 1024, // FIXME: that's just the maximum size
         OwnedBufferItemContent::Divider(s) => s.dynamic_usage(),
         OwnedBufferItemContent::Empty => 0,
       }
@@ -106,6 +115,7 @@ impl DynamicUsage for OwnedBufferItemContent {
           .sum();
         (area, Some(area * 12)) // FIXME: rough approx
       },
+      OwnedBufferItemContent::Image { .. } => (0, Some(10 * 1024 * 1024)), // FIXME: that's just the bounds
       OwnedBufferItemContent::Divider(s) => s.dynamic_usage_bounds(),
       OwnedBufferItemContent::Empty => (0, Some(0)),
     };
@@ -210,12 +220,11 @@ impl SingleClientRoomBuffer {
         },
         changes = timeline_stream.next() => {
           if let Some(changes) = changes {
-            write_items.send_modify(|items| {
-              for change in changes {
-                change
-                  .map(|item| (Some(item.unique_id()), self.format_timeline_item(item), Prerender::new()))
-                  .apply(items);
-              }
+            let changes = join_all(changes.into_iter().map(|change|
+              change.async_map(|tl_item| self.make_timeline_item(tl_item))
+            )).await;
+            write_items.send_modify(|items| for change in changes {
+              change.apply(items)
             });
             let mut inner = self.inner.write().await;
             inner.latest_event_id = inner.timeline.latest_event().await.and_then(|e| e.event_id().map(ToOwned::to_owned));
@@ -225,7 +234,21 @@ impl SingleClientRoomBuffer {
     }
   }
 
-  fn format_timeline_item(&self, tl_item: impl AsRef<TimelineItem>) -> OwnedBufferItemContent {
+  async fn make_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> (Option<u64>, OwnedBufferItemContent, Prerender) {
+    (
+      Some(tl_item.as_ref().unique_id()),
+      self.format_timeline_item(tl_item).await,
+      Prerender::new(),
+    )
+  }
+
+  async fn format_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> OwnedBufferItemContent {
     match tl_item.as_ref().kind() {
       TimelineItemKind::Event(event) => {
         use matrix_sdk_ui::timeline::TimelineItemContent::*;
@@ -279,6 +302,17 @@ impl SingleClientRoomBuffer {
             MessageType::Text(TextMessageEventContent { body, .. }) => {
               msg!("   ", "&lt;{}> {}", sender, escape_html(body))
             },
+            #[cfg(feature = "images")]
+            MessageType::Image(content) => {
+              crate::images::decode_image(
+                &self.client,
+                &self.config,
+                event.event_id().map(ToOwned::to_owned),
+                &sender,
+                content,
+              )
+              .await
+            },
             _ =>
             // Fallback to text body
             {
@@ -575,7 +609,12 @@ impl RoomBuffer {
     let initial_roominfo_hash = hash_roominfo(room.clone_info());
     let computed_roominfo = compute_room_info(room, initial_roominfo_hash).await;
 
-    tokio::task::spawn(update_roominfo_worker(room_id.clone(), initial_roominfo_hash, roominfo_rx, computed_roominfo_tx));
+    tokio::task::spawn(update_roominfo_worker(
+      room_id.clone(),
+      initial_roominfo_hash,
+      roominfo_rx,
+      computed_roominfo_tx,
+    ));
 
     let mut self_ = RoomBuffer {
       config,
@@ -600,7 +639,12 @@ impl RoomBuffer {
       );
       eyre!("Unknown room {} for client {:?}", self.room_id, client)
     })?;
-    let timeline = room.timeline_builder().build().await;
+    let timeline = room.timeline_builder().build().await.with_context(|| {
+      format!(
+        "Could not get timeline for {:?} from {:?}",
+        self.room_id, client
+      )
+    })?;
     let (items, timeline_stream) = timeline.subscribe_batched().await;
     tracing::info!(
       "Added client for {}, initial items: {:?}",
@@ -652,7 +696,8 @@ impl Buffer for RoomBuffer {
   fn short_name(&self) -> String {
     self
       .computed_roominfo
-      .display_name.as_ref()
+      .display_name
+      .as_ref()
       .map(|dn| dn.to_string())
       .unwrap_or_else(|| {
         self
@@ -675,13 +720,15 @@ impl Buffer for RoomBuffer {
   fn parent(&self) -> Option<BufferId> {
     self
       .computed_roominfo
-      .parent.as_ref()
+      .parent
+      .as_ref()
       .map(|parent| BufferId::Room(parent.clone()))
   }
   fn children(&self) -> Option<SortedVec<(BufferSortKey, BufferId)>> {
     self
       .computed_roominfo
-      .children.as_ref()
+      .children
+      .as_ref()
       .map(|children: &SortedVec<_>| {
         let children = children
           .iter()
@@ -728,6 +775,7 @@ impl Buffer for RoomBuffer {
               BufferItemContent::SimpleText(text.clone())
             },
             OwnedBufferItemContent::Text { text, .. } => BufferItemContent::Text(text.clone()),
+            OwnedBufferItemContent::Image { image, .. } => BufferItemContent::Image { image },
             OwnedBufferItemContent::Divider(text) => BufferItemContent::Divider(Text::raw(text)),
             OwnedBufferItemContent::Empty => BufferItemContent::Empty,
           },
@@ -773,10 +821,7 @@ impl Buffer for RoomBuffer {
   }
 
   fn fully_read(&self) -> FullyReadStatus {
-    match self
-      .computed_roominfo
-      .fully_read_at.as_ref()
-    {
+    match self.computed_roominfo.fully_read_at.as_ref() {
       None => FullyReadStatus::All, // Unknown, assume it's read for now, we'll probably find out later
       Some(fully_read_at) => {
         // Iterate through all buffers, and if any buffer's last event is not the one where
@@ -1015,10 +1060,9 @@ async fn update_roominfo_worker(
     }
     last_roominfo_hash = roominfo_hash;
     tracing::trace!("visible change to {}", room_id);
-    let room = buf
-      .client
-      .get_room(&room_id)
-      .expect("client missing room");
-    computed_roominfo_tx.send(compute_room_info(room, roominfo_hash).await).expect("failed to send to computed_roominfo_tx");
+    let room = buf.client.get_room(&room_id).expect("client missing room");
+    computed_roominfo_tx
+      .send(compute_room_info(room, roominfo_hash).await)
+      .expect("failed to send to computed_roominfo_tx");
   }
 }

src/components/backlog.rs

@@ -140,6 +140,8 @@ impl Backlog {
       )
       .scroll(scroll)
       .into(),
+      #[cfg(feature = "images")]
+      BufferItemContent::Image { image } => crate::widgets::BottomAlignedImage::new(image).into(),
       BufferItemContent::Divider(text) => {
         if scroll == 0 {
           Divider::new(Paragraph::new(text).alignment(Alignment::Center)).into()

src/images.rs

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2023  Valentin Lorentz
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use std::io::Cursor;
+use std::sync::Arc;
+
+use matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
+use matrix_sdk::ruma::{OwnedEventId, OwnedRoomId, RoomId};
+use matrix_sdk::Client;
+
+use crate::buffers::room::OwnedBufferItemContent;
+use crate::config::Config;
+use crate::html::{escape_html, format_html};
+
+pub(crate) async fn decode_image(
+  client: &Client,
+  config: &Config,
+  event_id: Option<OwnedEventId>,
+  sender: &str,
+  content: &ImageMessageEventContent,
+) -> OwnedBufferItemContent {
+  // Like `format!()` but returns OwnedBufferItemContent::Text, with is_message=true
+  macro_rules! msg {
+    ($prefix: expr, $($tokens:tt)*) => {
+      OwnedBufferItemContent::Text {
+        event_id,
+        is_message: true,
+        text: format_html(&config, $prefix, &format!($($tokens)*))
+      }
+    }
+  }
+
+  let ImageMessageEventContent { body, info, .. } = content;
+  match client.media().get_file(content, /* use_cache */ true).await {
+    Ok(Some(file)) => {
+      let mut reader = match info
+        .as_ref()
+        .and_then(|info| info.mimetype.as_ref())
+        .and_then(|mimetype| image::ImageFormat::from_mime_type(mimetype))
+      {
+        Some(format) => {
+          // Known format provided by the sender
+          image::io::Reader::with_format(Cursor::new(file), format)
+        },
+        None => {
+          // Unknown format or not provided by the sender, try guessing it
+          let reader = image::io::Reader::new(Cursor::new(file));
+          match reader.with_guessed_format() {
+            Ok(reader) => reader,
+            Err(e) => {
+              log::warn!("Could not guess image format: {:?}", e);
+              return msg!("   ", "&lt;{}> {}", sender, escape_html(body));
+            },
+          }
+        },
+      };
+
+      // Arbitrary values to avoid DoS. TODO: make them configurable
+      let mut limits = image::io::Limits::default();
+      limits.max_image_width = Some(10 * 1024);
+      limits.max_image_height = Some(10 * 1024);
+      limits.max_alloc = Some(100 * 1024 * 1024);
+      reader.limits(limits);
+
+      let image = match reader.decode() {
+        Ok(image) => image,
+        Err(e) => {
+          log::warn!("Could not decode image: {:?}", e);
+          return msg!("   ", "&lt;{}> {}", sender, escape_html(body));
+        },
+      };
+      OwnedBufferItemContent::Image {
+        event_id,
+        is_message: true,
+        image,
+      }
+    },
+    Ok(None) => msg!("   ", "&lt;{}> {}", sender, escape_html(body)),
+    Err(e) => {
+      log::error!("Could not get image file: {:?}", e);
+      msg!("   ", "&lt;{}> {}", sender, escape_html(body))
+    },
+  }
+}

src/lib.rs

@@ -10,6 +10,8 @@ pub mod commands;
 pub mod components;
 pub mod config;
 pub mod html;
+#[cfg(feature = "images")]
+pub mod images;
 pub mod log;
 pub mod mode;
 pub mod plugins;

src/main.rs

@@ -15,7 +15,7 @@ async fn tokio_main() -> Result<()> {
   Ok(())
 }
 
-#[tokio::main]
+#[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<()> {
   if let Err(e) = tokio_main().await {
     eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME"));

src/widgets/mod.rs

@@ -51,6 +51,8 @@ pub enum BacklogItemWidget<'a> {
   Paragraph(BottomAlignedParagraph<'a>),
   Container(BottomAlignedContainer<'a>),
   Divider(Divider<'a>),
+  #[cfg(feature = "images")]
+  Image(BottomAlignedImage<'a>),
   Empty(EmptyWidget),
 }
 
@@ -66,6 +68,88 @@ impl OverlappableWidget for EmptyWidget {
   }
 }
 
+#[cfg(feature = "images")]
+mod images {
+  use std::sync::Mutex;
+
+  use ratatui_image::picker::Picker;
+
+  use super::*;
+
+  lazy_static::lazy_static! {
+    static ref PROTOCOL_PICKER_LOCK: Mutex<()> = Mutex::new(());
+    pub static ref PROTOCOL_PICKER: Picker = make_protocol_picker();
+  }
+
+  fn make_protocol_picker() -> Picker {
+    let guard = PROTOCOL_PICKER_LOCK.lock().unwrap();
+    let mut picker = ratatui_image::picker::Picker::from_termios()
+      .expect("ratatui_image::picker::Picker::from_termios() failed");
+    picker.guess_protocol();
+    drop(guard);
+    picker
+  }
+
+  pub struct BottomAlignedImage<'a>(&'a image::DynamicImage);
+
+  impl<'a> BottomAlignedImage<'a> {
+    pub fn new(image: &'a image::DynamicImage) -> Self {
+      BottomAlignedImage(image)
+    }
+  }
+
+  impl<'a> OverlappableWidget for BottomAlignedImage<'a> {
+    fn height(&self, width: u16) -> u64 {
+      let width_px: u32 = u32::min(
+        self.0.width(),
+        (width as u32) * (PROTOCOL_PICKER.font_size.0 as u32),
+      );
+      let height_px = u32::min(self.0.height(), width_px * self.0.height() / self.0.width());
+      u32::min(
+        u16::MAX.into(),
+        height_px / (PROTOCOL_PICKER.font_size.1 as u32),
+      )
+      .into()
+    }
+
+    fn render_overlap(self, mut area: Rect, buf: &mut Buffer) -> (u16, u16) {
+      let width = u16::min(
+        u32::min(
+          u16::MAX.into(),
+          self.0.width() / (PROTOCOL_PICKER.font_size.0 as u32),
+        ) as u16,
+        area.width,
+      );
+      let height = u64::min(self.height(width), u16::MAX.into()) as u16;
+
+      area.y = u16::max(
+        area.y,
+        area.y.saturating_add(area.height).saturating_sub(height),
+      );
+
+      let protocol = PROTOCOL_PICKER
+        .clone()
+        .new_protocol(
+          self.0.clone(),
+          Rect {
+            x: area.x,
+            y: area.y,
+            width,
+            height,
+          },
+          ratatui_image::Resize::Crop,
+        )
+        .expect("picker.new_protocol failed");
+      let widget = ratatui_image::Image::new(protocol.as_ref());
+      widget.render(area, buf); // TODO: scroll
+      (area.width, 20)
+    }
+  }
+}
+
+#[cfg(feature = "images")]
+pub use images::*;
+
 /*
 impl<W: OverlappableWidget> Widget for W {
   fn render(self, area: Rect, buf: &mut Buffer) {