Fix coding-related core dumps with gcc -ftrapv.

The code was computing A - B, where A and B are pointers, and B is
random garbage.  This can lead to core dumps on platforms that
have special pointer registers, and it also leads to core dumps on
x86-64 when compiled with gcc -ftrapv.  The fix is to compute
A - B only when B is initialized properly.
* coding.c (coding_set_source, coding_set_destination): Return void.
(coding_change_source, coding_change_destinations): New functions,
with the old behaviors of coding_set_source and coding_set_destination.
All callers that need an offset changed to use these new functions.

diff --git a/src/ChangeLog b/src/ChangeLog
index bea9b4fbc476f7b0c22e7a5f8b0fccc970ac976c..1ade46911847c9d8eda82845aa003300dc9f4158 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,16 @@
+2012-05-26  Paul Eggert  <eggert@cs.ucla.edu>
+
+       Fix coding-related core dumps with gcc -ftrapv.
+       The code was computing A - B, where A and B are pointers, and B is
+       random garbage.  This can lead to core dumps on platforms that
+       have special pointer registers, and it also leads to core dumps on
+       x86-64 when compiled with gcc -ftrapv.  The fix is to compute
+       A - B only when B is initialized properly.
+       * coding.c (coding_set_source, coding_set_destination): Return void.
+       (coding_change_source, coding_change_destinations): New functions,
+       with the old behaviors of coding_set_source and coding_set_destination.
+       All callers that need an offset changed to use these new functions.
+
 2012-05-26  Glenn Morris  <rgm@gnu.org>
 
        * nsterm.m (ns_init_paths): Don't mess with INFOPATH.  (Bug#2791)

diff --git a/src/coding.c b/src/coding.c
index 42f342c390fe6252a74ef239c8e6f81e0af7fac9..17e342298b98f3ac4071e61b7b0a54fe7a220fdd 100644 (file)
--- a/src/coding.c
+++ b/src/coding.c
@@ -847,8 +847,10 @@ static int encode_coding_ccl (struct coding_system *);
 static void decode_coding_raw_text (struct coding_system *);
 static int encode_coding_raw_text (struct coding_system *);
 
-static ptrdiff_t coding_set_source (struct coding_system *);
-static ptrdiff_t coding_set_destination (struct coding_system *);
+static void coding_set_source (struct coding_system *);
+static ptrdiff_t coding_change_source (struct coding_system *);
+static void coding_set_destination (struct coding_system *);
+static ptrdiff_t coding_change_destination (struct coding_system *);
 static void coding_alloc_by_realloc (struct coding_system *, ptrdiff_t);
 static void coding_alloc_by_making_gap (struct coding_system *,
                                         ptrdiff_t, ptrdiff_t);
@@ -927,7 +929,7 @@ record_conversion_result (struct coding_system *coding,
     charset_map_loaded = 0;                                                 \
     c = DECODE_CHAR (charset, code);                                        \
     if (charset_map_loaded                                                  \
-       && (offset = coding_set_source (coding)))                            \
+       && (offset = coding_change_source (coding)))                         \
       {                                                                             \
        src += offset;                                                       \
        src_base += offset;                                                  \
@@ -942,7 +944,7 @@ record_conversion_result (struct coding_system *coding,
     charset_map_loaded = 0;                                            \
     code = ENCODE_CHAR (charset, c);                                   \
     if (charset_map_loaded                                             \
-       && (offset = coding_set_destination (coding)))                  \
+       && (offset = coding_change_destination (coding)))               \
       {                                                                        \
        dst += offset;                                                  \
        dst_end += offset;                                              \
@@ -956,7 +958,7 @@ record_conversion_result (struct coding_system *coding,
     charset_map_loaded = 0;                                            \
     charset = char_charset (c, charset_list, code_return);             \
     if (charset_map_loaded                                             \
-       && (offset = coding_set_destination (coding)))                  \
+       && (offset = coding_change_destination (coding)))               \
       {                                                                        \
        dst += offset;                                                  \
        dst_end += offset;                                              \
@@ -970,7 +972,7 @@ record_conversion_result (struct coding_system *coding,
     charset_map_loaded = 0;                                            \
     result = CHAR_CHARSET_P (c, charset);                              \
     if (charset_map_loaded                                             \
-       && (offset = coding_set_destination (coding)))                  \
+       && (offset = coding_change_destination (coding)))               \
       {                                                                        \
        dst += offset;                                                  \
        dst_end += offset;                                              \
@@ -1056,14 +1058,11 @@ record_conversion_result (struct coding_system *coding,
        | ((p)[-1] & 0x3F))))
 
 
-/* Update coding->source from coding->src_object, and return how many
-   bytes coding->source was changed.  */
+/* Set coding->source from coding->src_object.  */
 
-static ptrdiff_t
+static void
 coding_set_source (struct coding_system *coding)
 {
-  const unsigned char *orig = coding->source;
-
   if (BUFFERP (coding->src_object))
     {
       struct buffer *buf = XBUFFER (coding->src_object);
@@ -1082,18 +1081,26 @@ coding_set_source (struct coding_system *coding)
       /* Otherwise, the source is C string and is never relocated
         automatically.  Thus we don't have to update anything.  */
     }
-  return coding->source - orig;
 }
 
 
-/* Update coding->destination from coding->dst_object, and return how
-   many bytes coding->destination was changed.  */
+/* Set coding->source from coding->src_object, and return how many
+   bytes coding->source was changed.  */
 
 static ptrdiff_t
-coding_set_destination (struct coding_system *coding)
+coding_change_source (struct coding_system *coding)
 {
-  const unsigned char *orig = coding->destination;
+  const unsigned char *orig = coding->source;
+  coding_set_source (coding);
+  return coding->source - orig;
+}
 
+
+/* Set coding->destination from coding->dst_object.  */
+
+static void
+coding_set_destination (struct coding_system *coding)
+{
   if (BUFFERP (coding->dst_object))
     {
       if (BUFFERP (coding->src_object) && coding->src_pos < 0)
@@ -1118,6 +1125,17 @@ coding_set_destination (struct coding_system *coding)
       /* Otherwise, the destination is C string and is never relocated
         automatically.  Thus we don't have to update anything.  */
     }
+}
+
+
+/* Set coding->destination from coding->dst_object, and return how
+   many bytes coding->destination was changed.  */
+
+static ptrdiff_t
+coding_change_destination (struct coding_system *coding)
+{
+  const unsigned char *orig = coding->destination;
+  coding_set_destination (coding);
   return coding->destination - orig;
 }
 
@@ -4452,7 +4470,7 @@ encode_coding_iso_2022 (struct coding_system *coding)
          nbytes = encode_designation_at_bol (coding, charbuf, charbuf_end,
                                              desig_buf);
          if (charset_map_loaded
-             && (offset = coding_set_destination (coding)))
+             && (offset = coding_change_destination (coding)))
            {
              dst += offset;
              dst_end += offset;