diff --git a/video.c b/video.c
index 4ea67cb..691e51f 100644
--- a/video.c
+++ b/video.c
@@ -2336,17 +2336,23 @@ static void render_scanline_text_transparent_alpha(u32 layer,
 s32 affine_reference_x[2];
 s32 affine_reference_y[2];
 
+static inline s32 signext28(u32 value)
+{
+  s32 ret = (s32)(value << 4);
+  return ret >> 4;
+}
+
 void video_reload_counters()
 {
   /* This happens every Vblank */
-  affine_reference_x[0] = (s32)(read_ioreg(REG_BG2X_L) |
-     ((read_ioreg(REG_BG2X_H) & 0xFFF) << 16));
-  affine_reference_y[0] = (s32)(read_ioreg(REG_BG2Y_L) |
-     ((read_ioreg(REG_BG2Y_H) & 0xFFF) << 16));
-  affine_reference_x[1] = (s32)(read_ioreg(REG_BG3X_L) |
-     ((read_ioreg(REG_BG3X_H) & 0xFFF) << 16));
-  affine_reference_y[1] = (s32)(read_ioreg(REG_BG3Y_L) |
-     ((read_ioreg(REG_BG3Y_H) & 0xFFF) << 16));
+  affine_reference_x[0] = signext28(read_ioreg(REG_BG2X_L) |
+                                   (read_ioreg(REG_BG2X_H) << 16));
+  affine_reference_y[0] = signext28(read_ioreg(REG_BG2Y_L) |
+                                   (read_ioreg(REG_BG2Y_H) << 16));
+  affine_reference_x[1] = signext28(read_ioreg(REG_BG3X_L) |
+                                   (read_ioreg(REG_BG3X_H) << 16));
+  affine_reference_y[1] = signext28(read_ioreg(REG_BG3Y_L) |
+                                   (read_ioreg(REG_BG3Y_H) << 16));
 }
 
 #define affine_render_bg_pixel_normal()                                       \