diff --git a/src/buffers/room.rs b/src/buffers/room.rs
index f4017c8..e4a9578 100644
--- a/src/buffers/room.rs
+++ b/src/buffers/room.rs
@@ -205,24 +205,27 @@ impl SingleClientRoomBuffer {
     let room_id = self.room_id.clone();
     let timeline = self.timeline.clone();
     let mut back_pagination_status = timeline.back_pagination_status();
-    if back_pagination_status.get() == BackPaginationStatus::Paginating {
-      // We are already waiting for a backfill from the server
-      return;
-    }
-    tokio::spawn(async move {
-      tracing::info!("Starting pagination for {}", room_id);
-      timeline
-        .paginate_backwards(matrix_sdk_ui::timeline::PaginationOptions::until_num_items(
-          num, num,
-        ))
-        .await
-        .unwrap_or_else(|e| tracing::error!("Failed to paginate {} backward: {}", room_id, e));
-      tracing::info!("Ended pagination for {}", room_id);
-    });
+    match back_pagination_status.get() {
+      BackPaginationStatus::Paginating | BackPaginationStatus::TimelineStartReached => {
+        // We are already waiting for a backfill from the server
+      },
+      BackPaginationStatus::Idle => {
+        tokio::spawn(async move {
+          tracing::info!("Starting pagination for {}", room_id);
+          timeline
+            .paginate_backwards(matrix_sdk_ui::timeline::PaginationOptions::until_num_items(
+              num, num,
+            ))
+            .await
+            .unwrap_or_else(|e| tracing::error!("Failed to paginate {} backward: {}", room_id, e));
+          tracing::info!("Ended pagination for {}", room_id);
+        });
 
-    // Wait for the task we just spawned to change the status, so we don't risk starting
-    // a new one in the meantime
-    back_pagination_status.next().await;
+        // Wait for the task we just spawned to change the status, so we don't risk starting
+        // a new one in the meantime
+        back_pagination_status.next().await;
+      },
+    }
   }
 }