* coding.c: Integer and memory overflow fixes.

(produce_chars): Redo buffer-overflow calculations to avoid
unnecessary integer overflow.  Check for size overflow.
(encode_coding_object): Don't update size until xmalloc succeeds.

diff --git a/src/ChangeLog b/src/ChangeLog
index 9f50e928fac9997c7176d70549ff1016a6c4bc68..d86ae3602756ba3211943ac42d8ba5a558b498db 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,10 @@
 2011-07-28  Paul Eggert  <eggert@cs.ucla.edu>
 
+       * coding.c: Integer and memory overflow fixes.
+       (produce_chars): Redo buffer-overflow calculations to avoid
+       unnecessary integer overflow.  Check for size overflow.
+       (encode_coding_object): Don't update size until xmalloc succeeds.
+
        * character.c (Fstring): Check for size-calculation overflow.
 
        * ccl.c: Integer and memory overflow fixes.

diff --git a/src/coding.c b/src/coding.c
index 73a4bbc5e258ebb83bafbf664ef26783d9cd099b..5fd59d394d9f7913a90f77308903526f5a74811d 100644 (file)
--- a/src/coding.c
+++ b/src/coding.c
@@ -6683,8 +6683,12 @@ produce_chars (struct coding_system *coding, Lisp_Object translation_table,
                    break;
                }
 
-             if (dst + MAX_MULTIBYTE_LENGTH * to_nchars > dst_end)
+             if ((dst_end - dst) / MAX_MULTIBYTE_LENGTH < to_nchars)
                {
+                 if (((min (PTRDIFF_MAX, SIZE_MAX) - (buf_end - buf))
+                      / MAX_MULTIBYTE_LENGTH)
+                     < to_nchars)
+                   memory_full (SIZE_MAX);
                  dst = alloc_destination (coding,
                                           buf_end - buf
                                           + MAX_MULTIBYTE_LENGTH * to_nchars,
@@ -7888,11 +7892,10 @@ encode_coding_object (struct coding_system *coding,
     }
   else if (EQ (dst_object, Qt))
     {
+      ptrdiff_t dst_bytes = max (1, coding->src_chars);
       coding->dst_object = Qnil;
-      coding->dst_bytes = coding->src_chars;
-      if (coding->dst_bytes == 0)
-       coding->dst_bytes = 1;
-      coding->destination = (unsigned char *) xmalloc (coding->dst_bytes);
+      coding->destination = (unsigned char *) xmalloc (dst_bytes);
+      coding->dst_bytes = dst_bytes;
       coding->dst_multibyte = 0;
     }
   else