Avoid undefined behavior in color table hashing.

* image.c (CT_HASH_RGB) [COLOR_TABLE_SUPPORT]: Remove, replacing with ...
(ct_hash_rgb) [COLOR_TABLE_SUPPORT]: New function.  All uses changed.
This function avoids undefined behavior with signed shift overflow.

diff --git a/src/ChangeLog b/src/ChangeLog
index d44de652dde0b3aaddf682ed80bfbd20f70dbf4d..2be24fa851b45966062a984d81426916af2d9ed8 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,10 @@
+2014-11-13  Paul Eggert  <eggert@cs.ucla.edu>
+
+       Avoid undefined behavior in color table hashing.
+       * image.c (CT_HASH_RGB) [COLOR_TABLE_SUPPORT]: Remove, replacing with ...
+       (ct_hash_rgb) [COLOR_TABLE_SUPPORT]: New function.  All uses changed.
+       This function avoids undefined behavior with signed shift overflow.
+
 2014-11-10  Eli Zaretskii  <eliz@gnu.org>
 
        * fileio.c (Finsert_file_contents): Invalidate buffer caches also

diff --git a/src/image.c b/src/image.c
index 4b73a5fe80cded38edda9f2b0d6cb1d3b1625e0e..1a2c0e29dde0ab3c6ae09e8bb5023a8f3997f45f 100644 (file)
--- a/src/image.c
+++ b/src/image.c
@@ -4294,7 +4294,11 @@ struct ct_color
 
 /* Value is a hash of the RGB color given by R, G, and B.  */
 
-#define CT_HASH_RGB(R, G, B) (((R) << 16) ^ ((G) << 8) ^ (B))
+static unsigned
+ct_hash_rgb (unsigned r, unsigned g, unsigned b)
+{
+  return (r << 16) ^ (g << 8) ^ b;
+}
 
 /* The color hash table.  */
 
@@ -4349,7 +4353,7 @@ free_color_table (void)
 static unsigned long
 lookup_rgb_color (struct frame *f, int r, int g, int b)
 {
-  unsigned hash = CT_HASH_RGB (r, g, b);
+  unsigned hash = ct_hash_rgb (r, g, b);
   int i = hash % CT_SIZE;
   struct ct_color *p;
   Display_Info *dpyinfo;