* term.c: Integer and memory overflow issues.

(max_frame_lines): Remove; unused.
(encode_terminal_src_size, encode_terminal_dst_size): Now ptrdiff_t,
not int.
(encode_terminal_code, calculate_costs): Check for size
calculation overflow.
(encode_terminal_code): Use ptrdiff_t, not int, to record glyph
table lengths and related sizes.  Don't update size until alloc
done.  Redo calculations to avoid overflow.
(calculate_costs): Don't bother calling xmalloc when xrealloc will do.

diff --git a/src/ChangeLog b/src/ChangeLog
index d1db5e48dafea714d26ed0aa0675ff76547c3eca..7485afb00f48c3a93dfdc06d0813053ee8f9a31e 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,16 @@
 2011-07-29  Paul Eggert  <eggert@cs.ucla.edu>
 
+       * term.c: Integer and memory overflow issues.
+       (max_frame_lines): Remove; unused.
+       (encode_terminal_src_size, encode_terminal_dst_size): Now ptrdiff_t,
+       not int.
+       (encode_terminal_code, calculate_costs): Check for size
+       calculation overflow.
+       (encode_terminal_code): Use ptrdiff_t, not int, to record glyph
+       table lengths and related sizes.  Don't update size until alloc
+       done.  Redo calculations to avoid overflow.
+       (calculate_costs): Don't bother calling xmalloc when xrealloc will do.
+
        * sysdep.c: Integer and memory overflow issues.
        (system_process_attributes): Use ptrdiff_t, not int, for command
        line length.  Do not attempt to address one before the beginning

diff --git a/src/term.c b/src/term.c
index 22056451cb90ff0098d0dfc9719b581bfe4c0a09..bc6fa8f80f98a79415e6cfcaabaf42a58006f159 100644 (file)
--- a/src/term.c
+++ b/src/term.c
@@ -136,10 +136,6 @@ enum no_color_bit
 
 static int max_frame_cols;
 
-/* The largest frame height in any call to calculate_costs.  */
-
-static int max_frame_lines;
-
 /* Non-zero if we have dropped our controlling tty and therefore
    should not open a frame on stdout. */
 static int no_controlling_tty;
@@ -497,8 +493,8 @@ tty_clear_end_of_line (struct frame *f, int first_unused_hpos)
 static unsigned char *encode_terminal_src;
 static unsigned char *encode_terminal_dst;
 /* Allocated sizes of the above buffers.  */
-static int encode_terminal_src_size;
-static int encode_terminal_dst_size;
+static ptrdiff_t encode_terminal_src_size;
+static ptrdiff_t encode_terminal_dst_size;
 
 /* Encode SRC_LEN glyphs starting at SRC to terminal output codes.
    Set CODING->produced to the byte-length of the resulting byte
@@ -509,8 +505,8 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
 {
   struct glyph *src_end = src + src_len;
   unsigned char *buf;
-  int nchars, nbytes, required;
-  register int tlen = GLYPH_TABLE_LENGTH;
+  ptrdiff_t nchars, nbytes, required;
+  ptrdiff_t tlen = GLYPH_TABLE_LENGTH;
   register Lisp_Object *tbase = GLYPH_TABLE_BASE;
   Lisp_Object charset_list;
 
@@ -518,13 +514,13 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
      multibyte-form.  But, it may be enlarged on demand if
      Vglyph_table contains a string or a composite glyph is
      encountered.  */
-  required = MAX_MULTIBYTE_LENGTH * src_len;
+  if (min (PTRDIFF_MAX, SIZE_MAX) / MAX_MULTIBYTE_LENGTH < src_len)
+    memory_full (SIZE_MAX);
+  required = src_len;
+  required *= MAX_MULTIBYTE_LENGTH;
   if (encode_terminal_src_size < required)
     {
-      if (encode_terminal_src)
-       encode_terminal_src = xrealloc (encode_terminal_src, required);
-      else
-       encode_terminal_src = xmalloc (required);
+      encode_terminal_src = xrealloc (encode_terminal_src, required);
       encode_terminal_src_size = required;
     }
 
@@ -544,19 +540,23 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
          if (src->u.cmp.automatic)
            {
              gstring = composition_gstring_from_id (src->u.cmp.id);
-             required = src->slice.cmp.to + 1 - src->slice.cmp.from;
+             required = src->slice.cmp.to - src->slice.cmp.from + 1;
            }
          else
            {
              cmp = composition_table[src->u.cmp.id];
-             required = MAX_MULTIBYTE_LENGTH * cmp->glyph_len;
+             required = cmp->glyph_len;
+             required *= MAX_MULTIBYTE_LENGTH;
            }
 
-         if (encode_terminal_src_size < nbytes + required)
+         if (encode_terminal_src_size - nbytes < required)
            {
-             encode_terminal_src_size = nbytes + required;
-             encode_terminal_src = xrealloc (encode_terminal_src,
-                                             encode_terminal_src_size);
+             ptrdiff_t size;
+             if (min (PTRDIFF_MAX, SIZE_MAX) - nbytes < required)
+               memory_full (SIZE_MAX);
+             size = nbytes + required;
+             encode_terminal_src = xrealloc (encode_terminal_src, size);
+             encode_terminal_src_size = size;
              buf = encode_terminal_src + nbytes;
            }
 
@@ -627,11 +627,15 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
          if (NILP (string))
            {
              nbytes = buf - encode_terminal_src;
-             if (encode_terminal_src_size < nbytes + MAX_MULTIBYTE_LENGTH)
+             if (encode_terminal_src_size - nbytes < MAX_MULTIBYTE_LENGTH)
                {
-                 encode_terminal_src_size = nbytes + MAX_MULTIBYTE_LENGTH;
-                 encode_terminal_src = xrealloc (encode_terminal_src,
-                                                 encode_terminal_src_size);
+                 ptrdiff_t size;
+                 if (min (PTRDIFF_MAX, SIZE_MAX) - MAX_MULTIBYTE_LENGTH
+                     < nbytes)
+                   memory_full (SIZE_MAX);
+                 size = nbytes + MAX_MULTIBYTE_LENGTH;
+                 encode_terminal_src = xrealloc (encode_terminal_src, size);
+                 encode_terminal_src_size = size;
                  buf = encode_terminal_src + nbytes;
                }
              if (CHAR_BYTE8_P (c)
@@ -659,11 +663,14 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
              if (! STRING_MULTIBYTE (string))
                string = string_to_multibyte (string);
              nbytes = buf - encode_terminal_src;
-             if (encode_terminal_src_size < nbytes + SBYTES (string))
+             if (encode_terminal_src_size - nbytes < SBYTES (string))
                {
-                 encode_terminal_src_size = nbytes + SBYTES (string);
-                 encode_terminal_src = xrealloc (encode_terminal_src,
-                                                 encode_terminal_src_size);
+                 ptrdiff_t size;
+                 if (min (PTRDIFF_MAX, SIZE_MAX) - SBYTES (string) < nbytes)
+                   memory_full (SIZE_MAX);
+                 size = nbytes + SBYTES (string);
+                 encode_terminal_src = xrealloc (encode_terminal_src, size);
+                 encode_terminal_src_size = size;
                  buf = encode_terminal_src + nbytes;
                }
              memcpy (buf, SDATA (string), SBYTES (string));
@@ -684,12 +691,9 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi
   coding->source = encode_terminal_src;
   if (encode_terminal_dst_size == 0)
     {
+      encode_terminal_dst = xrealloc (encode_terminal_dst,
+                                     encode_terminal_src_size);
       encode_terminal_dst_size = encode_terminal_src_size;
-      if (encode_terminal_dst)
-       encode_terminal_dst = xrealloc (encode_terminal_dst,
-                                       encode_terminal_dst_size);
-      else
-       encode_terminal_dst = xmalloc (encode_terminal_dst_size);
     }
   coding->destination = encode_terminal_dst;
   coding->dst_bytes = encode_terminal_dst_size;
@@ -1156,18 +1160,14 @@ calculate_costs (struct frame *frame)
          char_ins_del_vector (i.e., char_ins_del_cost) isn't used because
          X turns off char_ins_del_ok. */
 
-      max_frame_lines = max (max_frame_lines, FRAME_LINES (frame));
       max_frame_cols = max (max_frame_cols, FRAME_COLS (frame));
+      if ((min (PTRDIFF_MAX, SIZE_MAX) / sizeof (int) - 1) / 2 < max_frame_cols)
+       memory_full (SIZE_MAX);
 
-      if (char_ins_del_vector != 0)
-        char_ins_del_vector
-          = (int *) xrealloc (char_ins_del_vector,
-                              (sizeof (int)
-                               + 2 * max_frame_cols * sizeof (int)));
-      else
-        char_ins_del_vector
-          = (int *) xmalloc (sizeof (int)
-                             + 2 * max_frame_cols * sizeof (int));
+      char_ins_del_vector
+       = (int *) xrealloc (char_ins_del_vector,
+                           (sizeof (int)
+                            + 2 * max_frame_cols * sizeof (int)));
 
       memset (char_ins_del_vector, 0,
              (sizeof (int) + 2 * max_frame_cols * sizeof (int)));