Fix accessing pixels under the mouse cursor:

Reading of single pixels didn't look under the cursor.

Copying of 1x1 bitmaps didn't look under the cursor for either reading
or writing.

Copying of larger bitmaps looked under the cursor for at most the
destination.

Copying of larger bitmaps looked under a garbage cursor (for the Display
bitmap) when the destination is a MEMBUF.  The results are not used, so
this only wasted time and flickered the cursor.

Writing of single pixels looked under a garbage cursor for MEMBUF
destinations, as above except this clobbered the current cursor and
didn't update the MEMBUF.  Writing of single pixels is not implemented
yet in depths > 8.  Otherwise, writing of single pixels worked.  It was
the only working case for accessing pixels under the cursor.

Clearing of MEMBUFs wasted time freezing the cursor in the Display bitmap.

The fixes abuse the top bits in the color arg to the cursor freezing
function to control the function.  Also clear the top 8 bits so that
applications can't clobber the control bits or create 256 aliases for
every 24-bit pixel value in depth 32.

Races fixed:

Showing and hiding the cursor only tried to avoid races with the mouse
event signal handler for internal operations.  There are still many
shorter races from not using volatile or sig_atomic_t for the variable
to control this.  This variable also controls freezes, and has more
complicated states than before.

The internal operation of unfreezing the cursor opened a race window
by unsetting the signal/freeze variable before showing the cursor.

lib/libvgl/bitmap.c

@@ -361,9 +361,13 @@ VGLBitmapCopy(VGLBitmap *src, int srcx, int srcy,
 {
   int error;
 
-  VGLMouseFreeze(dstx, dsty, width, hight, 0);
+  if (src->Type != MEMBUF)
+    VGLMouseFreeze(srcx, srcy, width, hight, 0);
+  if (dst->Type != MEMBUF)
+    VGLMouseFreeze(dstx, dsty, width, hight, 0);
   error = __VGLBitmapCopy(src, srcx, srcy, dst, dstx, dsty, width, hight);
-  VGLMouseUnFreeze();
+  if (src->Type != MEMBUF || dst->Type != MEMBUF)
+    VGLMouseUnFreeze();
   return error;
 }
 

lib/libvgl/mouse.c

@@ -102,6 +102,7 @@ VGLMousePointerShow()
   int i, pos, pos1;
 
   if (!VGLMouseVisible) {
+    VGLMouseFrozen++;
     VGLMouseVisible = 1;
     crtcidx = inb(0x3c4);
     crtcval = inb(0x3c5);
@@ -124,6 +125,7 @@ VGLMousePointerShow()
     outb(0x3c5, crtcval);
     outb(0x3ce, gdcidx);
     outb(0x3cf, gdcval);
+    VGLMouseFrozen--;
   }
 }
 
@@ -133,6 +135,7 @@ VGLMousePointerHide()
   byte crtcidx, crtcval, gdcidx, gdcval;
 
   if (VGLMouseVisible) {
+    VGLMouseFrozen++;
     VGLMouseVisible = 0;
     crtcidx = inb(0x3c4);
     crtcval = inb(0x3c5);
@@ -144,6 +147,7 @@ VGLMousePointerHide()
     outb(0x3c5, crtcval);
     outb(0x3ce, gdcidx);
     outb(0x3cf, gdcval);
+    VGLMouseFrozen--;
   }
 }
 
@@ -170,7 +174,7 @@ VGLMouseAction(int dummy)
   struct mouse_info mouseinfo;
 
   if (VGLMouseFrozen) {
-    VGLMouseFrozen++;
+    VGLMouseFrozen += 8;
     return;
   }
   mouseinfo.operation = MOUSE_GETINFO;
@@ -257,9 +261,8 @@ VGLMouseFreeze(int x, int y, int width, int hight, u_long color)
 {
   int i, xstride, ystride;
 
-  if (!VGLMouseFrozen) {
-    VGLMouseFrozen = 1;
-    if (width > 1 || hight > 1) {		/* bitmap */
+    VGLMouseFrozen++;
+    if (width > 1 || hight > 1 || (color & 0xc0000000) == 0) { /* bitmap */
       if (VGLMouseShown == 1) {
         int overlap;
 
@@ -283,29 +286,38 @@ VGLMouseFreeze(int x, int y, int width, int hight, u_long color)
           y >= VGLMouseYpos && y < VGLMouseYpos + MOUSE_IMG_SIZE) {
         xstride = VGLDisplay->PixelBytes;
         ystride = MOUSE_IMG_SIZE * xstride;
-        for (i = 0; i < xstride; i++, color >>= 8)
-          VGLMouseSave.Bitmap[(y-VGLMouseYpos)*ystride+
-                              (x-VGLMouseXpos)*xstride+i] = color;
-        if (VGLMouseAndMask->Bitmap 
-          [(y-VGLMouseYpos)*MOUSE_IMG_SIZE+(x-VGLMouseXpos)]) {
-          return 1;
+        if (color & 0x40000000) {	/* Get */
+          color = 0;
+          for (i = xstride - 1; i >= 0; i--)
+            color = (color << 8) |
+                    VGLMouseSave.Bitmap[(y-VGLMouseYpos)*ystride+
+                                        (x-VGLMouseXpos)*xstride+i];
+          return 0x40000000 | (color & 0xffffff);
+        } else {			/* Set */
+          color &= 0xffffff;		/* discard flag and other garbage */
+          for (i = 0; i < xstride; i++, color >>= 8)
+            VGLMouseSave.Bitmap[(y-VGLMouseYpos)*ystride+
+                                (x-VGLMouseXpos)*xstride+i] = color;
+          if (VGLMouseAndMask->Bitmap 
+            [(y-VGLMouseYpos)*MOUSE_IMG_SIZE+(x-VGLMouseXpos)]) {
+            return 1;
+          }   
         }   
       }       
     }
-  }
   return 0;
 }
 
 void
 VGLMouseUnFreeze()
 {
-  if (VGLMouseFrozen > 1) {
+  if (VGLMouseFrozen > 8) {
     VGLMouseFrozen = 0;
     VGLMouseAction(0);
   }
   else {
-    VGLMouseFrozen = 0;
     if (VGLMouseShown == VGL_MOUSESHOW && !VGLMouseVisible)
       VGLMousePointerShow();
+    VGLMouseFrozen = 0;
   }
 }

lib/libvgl/simple.c

@@ -98,7 +98,8 @@ VGLSetXY(VGLBitmap *object, int x, int y, u_long color)
 
   VGLCheckSwitch();
   if (x>=0 && x<object->VXsize && y>=0 && y<object->VYsize) {
-    if (!VGLMouseFreeze(x, y, 1, 1, color)) {
+    if (object->Type == MEMBUF ||
+        !VGLMouseFreeze(x, y, 1, 1, 0x80000000 | color)) {
       switch (object->Type) {
       case MEMBUF:
       case VIDBUF8:
@@ -139,12 +140,13 @@ VGLSetXY(VGLBitmap *object, int x, int y, u_long color)
 	object->Bitmap[offset] |= (byte)color;
       }
     }
-    VGLMouseUnFreeze();
+    if (object->Type != MEMBUF)
+      VGLMouseUnFreeze();
   }
 }
 
-u_long
-VGLGetXY(VGLBitmap *object, int x, int y)
+static u_long
+__VGLGetXY(VGLBitmap *object, int x, int y)
 {
   int offset;
   byte b[4];
@@ -152,9 +154,6 @@ VGLGetXY(VGLBitmap *object, int x, int y)
   u_long color;
   byte mask;
 
-  VGLCheckSwitch();
-  if (x<0 || x>=object->VXsize || y<0 || y>=object->VYsize)
-    return 0;
   switch (object->Type) {
     case MEMBUF:
     case VIDBUF8:
@@ -195,6 +194,27 @@ VGLGetXY(VGLBitmap *object, int x, int y)
   return 0;		/* XXX black? */
 }
 
+u_long
+VGLGetXY(VGLBitmap *object, int x, int y)
+{
+  u_long color;
+
+  VGLCheckSwitch();
+  if (x<0 || x>=object->VXsize || y<0 || y>=object->VYsize)
+    return 0;
+  if (object->Type != MEMBUF) {
+    color = VGLMouseFreeze(x, y, 1, 1, 0x40000000);
+    if (color & 0x40000000) {
+      VGLMouseUnFreeze();
+      return color & 0xffffff;
+    }
+  }
+  color = __VGLGetXY(object, x, y);
+  if (object->Type != MEMBUF)
+    VGLMouseUnFreeze();
+  return color;
+}
+
  /*
   * Symmetric Double Step Line Algorithm by Brian Wyvill from
   * "Graphics Gems", Academic Press, 1990.
@@ -501,7 +521,8 @@ VGLClear(VGLBitmap *object, u_long color)
   byte b[4];
 
   VGLCheckSwitch();
-  VGLMouseFreeze(0, 0, object->Xsize, object->Ysize, color); /* XXX */
+  if (object->Type != MEMBUF)
+    VGLMouseFreeze(0, 0, object->Xsize, object->Ysize, color);
   switch (object->Type) {
   case MEMBUF:
   case VIDBUF8:
@@ -565,7 +586,8 @@ VGLClear(VGLBitmap *object, u_long color)
     outb(0x3ce, 0x05); outb(0x3cf, 0x00);
     break;
   }
-  VGLMouseUnFreeze();
+  if (object->Type != MEMBUF)
+    VGLMouseUnFreeze();
 }
 
 void