Avoid undefined behavior with signed left shift.

Caught by 'gcc -fsanitize=undefined'.
* dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
* dispnew.c (line_hash_code, scrolling):
* scroll.c (calculate_scrolling, calculate_direct_scrolling):
Use 'unsigned', not 'int', for line hashes.
(scrolling_max_lines_saved): Avoid mystery constants for hash sizes.

diff --git a/src/ChangeLog b/src/ChangeLog
index 28367ed34f5acb6c6bb743ff7b3df41f9e1e0bce..dce13035766a9a1ac092df5d68f6bcd792eedf7d 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,13 @@
+2014-08-03  Paul Eggert  <eggert@cs.ucla.edu>
+
+       Avoid undefined behavior with signed left shift.
+       Caught by 'gcc -fsanitize=undefined'.
+       * dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
+       * dispnew.c (line_hash_code, scrolling):
+       * scroll.c (calculate_scrolling, calculate_direct_scrolling):
+       Use 'unsigned', not 'int', for line hashes.
+       (scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
+
 2014-08-02  Paul Eggert  <eggert@cs.ucla.edu>
 
        Make compare-strings more compatible with old behavior (Bug#17903).

diff --git a/src/dispextern.h b/src/dispextern.h
index bb6f1eb2b5e73ee9e2b739b4e2f967e2c8f231df..ebd4260d40841366a8eaed95635f765502f06516 100644 (file)
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -3507,13 +3507,13 @@ extern void tty_append_glyph (struct it *);
 
 /* Defined in scroll.c */
 
-extern int scrolling_max_lines_saved (int, int, int *, int *, int *);
+extern int scrolling_max_lines_saved (int, int, unsigned *, unsigned *, int *);
 extern void do_line_insertion_deletion_costs (struct frame *, const char *,
                                               const char *, const char *,
                                              const char *, const char *,
                                              const char *, int);
-void scrolling_1 (struct frame *, int, int, int, int *, int *, int *,
-                  int *, int);
+void scrolling_1 (struct frame *, int, int, int, int *, int *, unsigned *,
+                  unsigned *, int);
 
 /* Defined in frame.c */
 

diff --git a/src/dispnew.c b/src/dispnew.c
index 70862985e66ae8cfb471ebb7473cdc0a64feaf04..78df22ddbe285544abbb8b0ba3981c13f08d51d3 100644 (file)
--- a/src/dispnew.c
+++ b/src/dispnew.c
@@ -1108,10 +1108,10 @@ prepare_desired_row (struct window *w, struct glyph_row *row, bool mode_line_p)
 /* Return a hash code for glyph row ROW, which may
    be from current or desired matrix of frame F.  */
 
-static int
+static unsigned
 line_hash_code (struct frame *f, struct glyph_row *row)
 {
-  int hash = 0;
+  unsigned hash = 0;
 
   if (row->enabled_p)
     {
@@ -4647,8 +4647,8 @@ scrolling (struct frame *frame)
   int unchanged_at_top, unchanged_at_bottom;
   int window_size;
   int changed_lines;
-  int *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
-  int *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *old_draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   register int i;

diff --git a/src/scroll.c b/src/scroll.c
index 3da236ca8a6fac7ae3645c483d573122445ee3e3..6c559663f80ec30036df2bf1d045ae3e0ad9e6db 100644 (file)
--- a/src/scroll.c
+++ b/src/scroll.c
@@ -90,7 +90,7 @@ calculate_scrolling (struct frame *frame,
                     /* matrix is of size window_size + 1 on each side.  */
                     struct matrix_elt *matrix,
                     int window_size, int lines_below,
-                    int *draw_cost, int *old_hash, int *new_hash,
+                    int *draw_cost, unsigned *old_hash, unsigned *new_hash,
                     int free_at_end)
 {
   register int i, j;
@@ -427,7 +427,7 @@ calculate_direct_scrolling (struct frame *frame,
                            struct matrix_elt *matrix,
                            int window_size, int lines_below,
                            int *draw_cost, int *old_draw_cost,
-                           int *old_hash, int *new_hash,
+                           unsigned *old_hash, unsigned *new_hash,
                            int free_at_end)
 {
   register int i, j;
@@ -794,7 +794,7 @@ do_direct_scrolling (struct frame *frame, struct glyph_matrix *current_matrix,
 void
 scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
             int unchanged_at_bottom, int *draw_cost, int *old_draw_cost,
-            int *old_hash, int *new_hash, int free_at_end)
+            unsigned *old_hash, unsigned *new_hash, int free_at_end)
 {
   struct matrix_elt *matrix
     = alloca ((window_size + 1) * (window_size + 1) * sizeof *matrix);
@@ -829,12 +829,14 @@ scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
 
 int
 scrolling_max_lines_saved (int start, int end,
-                           int *oldhash, int *newhash,
+                           unsigned *oldhash, unsigned *newhash,
                            int *cost)
 {
-  struct { int hash; int count; } lines[01000];
-  register int i, h;
-  register int matchcount = 0;
+  enum { LOG2_NLINES = 9 };
+  enum { NLINES = 1 << LOG2_NLINES };
+  struct { unsigned hash; int count; } lines[NLINES];
+  int i, h;
+  int matchcount = 0;
   int avg_length = 0;
   int threshold;
 
@@ -855,7 +857,7 @@ scrolling_max_lines_saved (int start, int end,
     {
       if (cost[i] > threshold)
        {
-         h = newhash[i] & 0777;
+         h = newhash[i] & (NLINES - 1);
          lines[h].hash = newhash[i];
          lines[h].count++;
        }
@@ -865,7 +867,7 @@ scrolling_max_lines_saved (int start, int end,
      matches between old lines and new.  */
   for (i = start; i < end; i++)
     {
-      h = oldhash[i] & 0777;
+      h = oldhash[i] & (NLINES - 1);
       if (oldhash[i] == lines[h].hash)
        {
          matchcount++;