From: Paul Eggert <eggert@cs.ucla.edu>
Date: Sun, 3 Aug 2014 12:34:44 +0000 (-0700)
Subject: Avoid undefined behavior with signed left shift.
X-Git-Tag: emacs-25.0.90~2635^2~679^2~535
X-Git-Url: http://git.eshelyaron.com/gitweb/?a=commitdiff_plain;h=8f88f7d3c5da38cd2d781770b533dc6c93c52d59;p=emacs.git

Avoid undefined behavior with signed left shift.

Caught by 'gcc -fsanitize=undefined'.
* dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
* dispnew.c (line_hash_code, scrolling):
* scroll.c (calculate_scrolling, calculate_direct_scrolling):
Use 'unsigned', not 'int', for line hashes.
(scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
---

diff --git a/src/ChangeLog b/src/ChangeLog
index 28367ed34f5..dce13035766 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,13 @@
+2014-08-03  Paul Eggert  <eggert@cs.ucla.edu>
+
+	Avoid undefined behavior with signed left shift.
+	Caught by 'gcc -fsanitize=undefined'.
+	* dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
+	* dispnew.c (line_hash_code, scrolling):
+	* scroll.c (calculate_scrolling, calculate_direct_scrolling):
+	Use 'unsigned', not 'int', for line hashes.
+	(scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
+
 2014-08-02  Paul Eggert  <eggert@cs.ucla.edu>
 
 	Make compare-strings more compatible with old behavior (Bug#17903).
diff --git a/src/dispextern.h b/src/dispextern.h
index bb6f1eb2b5e..ebd4260d408 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -3507,13 +3507,13 @@ extern void tty_append_glyph (struct it *);
 
 /* Defined in scroll.c */
 
-extern int scrolling_max_lines_saved (int, int, int *, int *, int *);
+extern int scrolling_max_lines_saved (int, int, unsigned *, unsigned *, int *);
 extern void do_line_insertion_deletion_costs (struct frame *, const char *,
                                               const char *, const char *,
 					      const char *, const char *,
 					      const char *, int);
-void scrolling_1 (struct frame *, int, int, int, int *, int *, int *,
-                  int *, int);
+void scrolling_1 (struct frame *, int, int, int, int *, int *, unsigned *,
+                  unsigned *, int);
 
 /* Defined in frame.c */
 
diff --git a/src/dispnew.c b/src/dispnew.c
index 70862985e66..78df22ddbe2 100644
--- a/src/dispnew.c
+++ b/src/dispnew.c
@@ -1108,10 +1108,10 @@ prepare_desired_row (struct window *w, struct glyph_row *row, bool mode_line_p)
 /* Return a hash code for glyph row ROW, which may
    be from current or desired matrix of frame F.  */
 
-static int
+static unsigned
 line_hash_code (struct frame *f, struct glyph_row *row)
 {
-  int hash = 0;
+  unsigned hash = 0;
 
   if (row->enabled_p)
     {
@@ -4647,8 +4647,8 @@ scrolling (struct frame *frame)
   int unchanged_at_top, unchanged_at_bottom;
   int window_size;
   int changed_lines;
-  int *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
-  int *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *old_draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   register int i;
diff --git a/src/scroll.c b/src/scroll.c
index 3da236ca8a6..6c559663f80 100644
--- a/src/scroll.c
+++ b/src/scroll.c
@@ -90,7 +90,7 @@ calculate_scrolling (struct frame *frame,
 		     /* matrix is of size window_size + 1 on each side.  */
 		     struct matrix_elt *matrix,
 		     int window_size, int lines_below,
-		     int *draw_cost, int *old_hash, int *new_hash,
+		     int *draw_cost, unsigned *old_hash, unsigned *new_hash,
 		     int free_at_end)
 {
   register int i, j;
@@ -427,7 +427,7 @@ calculate_direct_scrolling (struct frame *frame,
 			    struct matrix_elt *matrix,
 			    int window_size, int lines_below,
 			    int *draw_cost, int *old_draw_cost,
-			    int *old_hash, int *new_hash,
+			    unsigned *old_hash, unsigned *new_hash,
 			    int free_at_end)
 {
   register int i, j;
@@ -794,7 +794,7 @@ do_direct_scrolling (struct frame *frame, struct glyph_matrix *current_matrix,
 void
 scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
 	     int unchanged_at_bottom, int *draw_cost, int *old_draw_cost,
-	     int *old_hash, int *new_hash, int free_at_end)
+	     unsigned *old_hash, unsigned *new_hash, int free_at_end)
 {
   struct matrix_elt *matrix
     = alloca ((window_size + 1) * (window_size + 1) * sizeof *matrix);
@@ -829,12 +829,14 @@ scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
 
 int
 scrolling_max_lines_saved (int start, int end,
-                           int *oldhash, int *newhash,
+                           unsigned *oldhash, unsigned *newhash,
                            int *cost)
 {
-  struct { int hash; int count; } lines[01000];
-  register int i, h;
-  register int matchcount = 0;
+  enum { LOG2_NLINES = 9 };
+  enum { NLINES = 1 << LOG2_NLINES };
+  struct { unsigned hash; int count; } lines[NLINES];
+  int i, h;
+  int matchcount = 0;
   int avg_length = 0;
   int threshold;
 
@@ -855,7 +857,7 @@ scrolling_max_lines_saved (int start, int end,
     {
       if (cost[i] > threshold)
 	{
-	  h = newhash[i] & 0777;
+	  h = newhash[i] & (NLINES - 1);
 	  lines[h].hash = newhash[i];
 	  lines[h].count++;
 	}
@@ -865,7 +867,7 @@ scrolling_max_lines_saved (int start, int end,
      matches between old lines and new.  */
   for (i = start; i < end; i++)
     {
-      h = oldhash[i] & 0777;
+      h = oldhash[i] & (NLINES - 1);
       if (oldhash[i] == lines[h].hash)
 	{
 	  matchcount++;