[WIP] images

Cargo.toml

@@ -8,11 +8,19 @@ repository = "https://git.tf/val/ratatrix"
 authors = ["Val Lorentz"]
 license = "AGPL-3.0-only AND MIT"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = [
+    "images",
+]
+images = [
+    "dep:image",
+    "dep:ratatui-image",
+]
 
 [dependencies]
 
 # Async
+async-std = { version = "1.12.0", features = ["tokio1", "attributes"] }
 futures = "0.3.28"
 tokio = { version = "1.32.0", features = ["full"] }
 tokio-util = "0.7.9"
@@ -64,8 +72,8 @@ sorted-vec = "0.8.3"
 eyeball = "0.8.7" # data structures observer returned by matrix-sdk-ui
 eyeball-im = "0.4.2" # immutable data structures observer returned by matrix-sdk-ui
 imbl = "2.0" # ditto
-matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2", features = ["eyre", "markdown"] }
+matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4", features = ["eyre", "markdown"] }
-matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "5c37acb81ce624d83be54b5140cd60399b556fb2" }
+matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "7bbd07cc7703051e5276b87f26bf88b6239d64a4" }
 #matrix-sdk = { path = "../matrix-rust-sdk/crates/matrix-sdk", features = ["eyre", "markdown"] }
 #matrix-sdk-ui = { path = "../matrix-rust-sdk/crates/matrix-sdk-ui" }
 
@@ -78,8 +86,13 @@ strip-ansi-escapes = "0.2.0"
 tui-textarea = "0.3.0"
 unicode-width = "0.1"
 
+# UI (images)
+ratatui-image = { version = "0.8.0", optional = true }
+image = { version = "0.24.8", optional = true }
+
 [patch.crates-io]
 #ratatui = { path = "../ratatui", features = ["serde", "macros"] }
+eyeball-im = { path = "../eyeball/eyeball-im" }
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"

src/buffers/mod.rs

@@ -32,7 +32,7 @@ use crate::widgets::Prerender;
 
 mod log;
 pub use log::LogBuffer;
-mod room;
+pub(crate) mod room;
 pub use room::RoomBuffer;
 
 /// Maximum time before reordering the buffer list based on parent/child relationships.
@@ -105,6 +105,10 @@ pub enum BufferItemContent<'buf> {
   SimpleText(Text<'buf>),
   /// Pairs of `(padding, content)`
   Text(Vec<(String, Text<'buf>)>),
+  #[cfg(feature = "images")]
+  Image {
+    image: &'buf image::DynamicImage,
+  },
   Divider(Text<'buf>),
   Empty,
 }

src/buffers/room.rs

@@ -20,19 +20,21 @@ use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::{Arc, OnceLock};
 
 use chrono::{offset::Local, DateTime};
-use color_eyre::eyre::{eyre, Result};
+use color_eyre::eyre::{eyre, Result, WrapErr};
 use eyeball::SharedObservable;
 use eyeball_im::VectorDiff;
+use futures::future::join_all;
 use futures::future::OptionFuture;
 use futures::stream::FuturesUnordered;
 use futures::{FutureExt, Stream, StreamExt};
+use image::{DynamicImage, ImageResult};
 use itertools::Itertools;
 use matrix_sdk::async_trait;
 use matrix_sdk::deserialized_responses::SyncOrStrippedState;
 use matrix_sdk::room::ParentSpace;
 use matrix_sdk::ruma::events::fully_read::FullyReadEventContent;
 use matrix_sdk::ruma::events::room::message::{
-  FormattedBody, MessageFormat, MessageType, TextMessageEventContent,
+  FormattedBody, ImageMessageEventContent, MessageFormat, MessageType, TextMessageEventContent,
 };
 use matrix_sdk::ruma::events::space::child::SpaceChildEventContent;
 use matrix_sdk::ruma::events::RoomAccountDataEvent;
@@ -72,6 +74,12 @@ pub enum OwnedBufferItemContent {
     /// `(padding, content)` pairs
     text: Vec<(String, Text<'static>)>,
   },
+  #[cfg(feature = "images")]
+  Image {
+    event_id: Option<OwnedEventId>,
+    is_message: bool,
+    image: DynamicImage,
+  },
   Divider(String),
   Empty,
 }
@@ -89,6 +97,7 @@ impl DynamicUsage for OwnedBufferItemContent {
             .map(|item| item.1.width() * item.1.height() * 4)
             .sum() // FIXME: rough approx
         },
+        OwnedBufferItemContent::Image { .. } => 10 * 1024 * 1024, // FIXME: that's just the maximum size
         OwnedBufferItemContent::Divider(s) => s.dynamic_usage(),
         OwnedBufferItemContent::Empty => 0,
       }
@@ -106,6 +115,7 @@ impl DynamicUsage for OwnedBufferItemContent {
           .sum();
         (area, Some(area * 12)) // FIXME: rough approx
       },
+      OwnedBufferItemContent::Image { .. } => (0, Some(10 * 1024 * 1024)), // FIXME: that's just the bounds
       OwnedBufferItemContent::Divider(s) => s.dynamic_usage_bounds(),
       OwnedBufferItemContent::Empty => (0, Some(0)),
     };
@@ -210,12 +220,11 @@ impl SingleClientRoomBuffer {
         },
         changes = timeline_stream.next() => {
           if let Some(changes) = changes {
-            write_items.send_modify(|items| {
+            let changes = join_all(changes.into_iter().map(|change|
-              for change in changes {
+              change.async_map(|tl_item| self.make_timeline_item(tl_item))
-                change
+            )).await;
-                  .map(|item| (Some(item.unique_id()), self.format_timeline_item(item), Prerender::new()))
+            write_items.send_modify(|items| for change in changes {
-                  .apply(items);
+              change.apply(items)
-              }
             });
             let mut inner = self.inner.write().await;
             inner.latest_event_id = inner.timeline.latest_event().await.and_then(|e| e.event_id().map(ToOwned::to_owned));
@@ -225,7 +234,21 @@ impl SingleClientRoomBuffer {
     }
   }
 
-  fn format_timeline_item(&self, tl_item: impl AsRef<TimelineItem>) -> OwnedBufferItemContent {
+  async fn make_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> (Option<u64>, OwnedBufferItemContent, Prerender) {
+    (
+      Some(tl_item.as_ref().unique_id()),
+      self.format_timeline_item(tl_item).await,
+      Prerender::new(),
+    )
+  }
+
+  async fn format_timeline_item(
+    &self,
+    tl_item: impl AsRef<TimelineItem>,
+  ) -> OwnedBufferItemContent {
     match tl_item.as_ref().kind() {
       TimelineItemKind::Event(event) => {
         use matrix_sdk_ui::timeline::TimelineItemContent::*;
@@ -279,6 +302,17 @@ impl SingleClientRoomBuffer {
             MessageType::Text(TextMessageEventContent { body, .. }) => {
               msg!("   ", "&lt;{}> {}", sender, escape_html(body))
             },
+            #[cfg(feature = "images")]
+            MessageType::Image(content) => {
+              crate::images::decode_image(
+                &self.client,
+                &self.config,
+                event.event_id().map(ToOwned::to_owned),
+                &sender,
+                content,
+              )
+              .await
+            },
             _ =>
             // Fallback to text body
             {
@@ -594,7 +628,12 @@ impl RoomBuffer {
       );
       eyre!("Unknown room {} for client {:?}", self.room_id, client)
     })?;
-    let timeline = room.timeline_builder().build().await;
+    let timeline = room.timeline_builder().build().await.with_context(|| {
+      format!(
+        "Could not get timeline for {:?} from {:?}",
+        self.room_id, client
+      )
+    })?;
     let (items, timeline_stream) = timeline.subscribe_batched().await;
     tracing::info!(
       "Added client for {}, initial items: {:?}",
@@ -778,6 +817,7 @@ impl Buffer for RoomBuffer {
               BufferItemContent::SimpleText(text.clone())
             },
             OwnedBufferItemContent::Text { text, .. } => BufferItemContent::Text(text.clone()),
+            OwnedBufferItemContent::Image { image, .. } => BufferItemContent::Image { image },
             OwnedBufferItemContent::Divider(text) => BufferItemContent::Divider(Text::raw(text)),
             OwnedBufferItemContent::Empty => BufferItemContent::Empty,
           },

src/components/backlog.rs

@@ -140,6 +140,8 @@ impl Backlog {
       )
       .scroll(scroll)
       .into(),
+      #[cfg(feature = "images")]
+      BufferItemContent::Image { image } => crate::widgets::BottomAlignedImage::new(image).into(),
       BufferItemContent::Divider(text) => {
         if scroll == 0 {
           Divider::new(Paragraph::new(text).alignment(Alignment::Center)).into()

src/lib.rs

@@ -10,6 +10,8 @@ pub mod commands;
 pub mod components;
 pub mod config;
 pub mod html;
+#[cfg(feature = "images")]
+pub mod images;
 pub mod log;
 pub mod mode;
 pub mod plugins;

src/main.rs

@@ -15,7 +15,7 @@ async fn tokio_main() -> Result<()> {
   Ok(())
 }
 
-#[tokio::main]
+#[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<()> {
   if let Err(e) = tokio_main().await {
     eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME"));

src/widgets/mod.rs

@@ -51,6 +51,8 @@ pub enum BacklogItemWidget<'a> {
   Paragraph(BottomAlignedParagraph<'a>),
   Container(BottomAlignedContainer<'a>),
   Divider(Divider<'a>),
+  #[cfg(feature = "images")]
+  Image(BottomAlignedImage<'a>),
   Empty(EmptyWidget),
 }
 
@@ -66,6 +68,61 @@ impl OverlappableWidget for EmptyWidget {
   }
 }
 
+#[cfg(feature = "images")]
+mod images {
+  use super::*;
+
+  lazy_static::lazy_static! {
+    static ref PROTOCOL_PICKER: ratatui_image::picker::Picker = make_protocol_picker();
+  }
+
+  fn make_protocol_picker() -> ratatui_image::picker::Picker {
+    let mut picker = ratatui_image::picker::Picker::from_termios()
+      .expect("ratatui_image::picker::Picker::from_termios() failed");
+    picker.guess_protocol();
+    picker
+  }
+
+  pub struct BottomAlignedImage<'a>(&'a image::DynamicImage);
+
+  impl<'a> BottomAlignedImage<'a> {
+    pub fn new(image: &'a image::DynamicImage) -> Self {
+      BottomAlignedImage(image)
+    }
+  }
+
+  impl<'a> OverlappableWidget for BottomAlignedImage<'a> {
+    fn height(&self, _width: u16) -> u64 {
+      20 // TODO
+    }
+    fn render_overlap(self, area: Rect, buf: &mut Buffer) -> (u16, u16) {
+      let width = u16::min(100, area.width);
+      let width_px: u32 = (width as u32) * (PROTOCOL_PICKER.font_size.0 as u32);
+      let height_px = width_px * self.0.height() / self.0.width();
+      let height = u32::min(u16::MAX.into(), height_px / (PROTOCOL_PICKER.font_size.1 as u32)) as u16;
+
+      let protocol = PROTOCOL_PICKER
+        .clone()
+        .new_protocol(
+          self.0.clone(),
+          Rect {
+            x: area.x,
+            y: area.y,
+            width,
+            height,
+          },
+          ratatui_image::Resize::Crop,
+        )
+        .expect("picker.new_protocol failed");
+      let widget= ratatui_image::Image::new(protocol.as_ref());
+      widget.render(area, buf); // TODO: scroll
+      (area.width, area.height)
+    }
+  }
+}
+
+pub use images::*;
+
 /*
 impl<W: OverlappableWidget> Widget for W {
   fn render(self, area: Rect, buf: &mut Buffer) {