diff --git a/src/widgets/image.rs b/src/widgets/image.rs
index 1d482d5..46820ec 100644
--- a/src/widgets/image.rs
+++ b/src/widgets/image.rs
@@ -414,10 +414,16 @@ mod imp {
             );
 
             // Drag for moving image around
-            let drag_gesture = gtk::GestureDrag::new();
+            let drag_gesture = gtk::GestureDrag::builder().button(0).build();
             obj.add_controller(drag_gesture.clone());
 
             drag_gesture.connect_drag_begin(glib::clone!(@weak obj => move |gesture, _, _| {
+                // Allow only left and middle button
+                if ![1, 2].contains(&gesture.current_button()) {
+                    gesture.set_state(gtk::EventSequenceState::Denied);
+                    return;
+                }
+
                 if obj.is_hscrollable() || obj.is_vscrollable() {
                     obj.set_cursor(gdk::Cursor::from_name("grabbing", None).as_ref());
                     obj.imp().last_drag_value.set(Some((0., 0.)));