From: Paul Eggert Date: Fri, 29 Jul 2011 01:22:19 +0000 (-0700) Subject: * term.c: Integer and memory overflow issues. X-Git-Tag: emacs-pretest-24.0.90~104^2~152^2~101 X-Git-Url: http://git.eshelyaron.com/gitweb/?a=commitdiff_plain;h=fee31f82d5279a6faeb2d4cef808e9d7fce2f210;p=emacs.git * term.c: Integer and memory overflow issues. (max_frame_lines): Remove; unused. (encode_terminal_src_size, encode_terminal_dst_size): Now ptrdiff_t, not int. (encode_terminal_code, calculate_costs): Check for size calculation overflow. (encode_terminal_code): Use ptrdiff_t, not int, to record glyph table lengths and related sizes. Don't update size until alloc done. Redo calculations to avoid overflow. (calculate_costs): Don't bother calling xmalloc when xrealloc will do. --- diff --git a/src/ChangeLog b/src/ChangeLog index d1db5e48daf..7485afb00f4 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,16 @@ 2011-07-29 Paul Eggert + * term.c: Integer and memory overflow issues. + (max_frame_lines): Remove; unused. + (encode_terminal_src_size, encode_terminal_dst_size): Now ptrdiff_t, + not int. + (encode_terminal_code, calculate_costs): Check for size + calculation overflow. + (encode_terminal_code): Use ptrdiff_t, not int, to record glyph + table lengths and related sizes. Don't update size until alloc + done. Redo calculations to avoid overflow. + (calculate_costs): Don't bother calling xmalloc when xrealloc will do. + * sysdep.c: Integer and memory overflow issues. (system_process_attributes): Use ptrdiff_t, not int, for command line length. Do not attempt to address one before the beginning diff --git a/src/term.c b/src/term.c index 22056451cb9..bc6fa8f80f9 100644 --- a/src/term.c +++ b/src/term.c @@ -136,10 +136,6 @@ enum no_color_bit static int max_frame_cols; -/* The largest frame height in any call to calculate_costs. */ - -static int max_frame_lines; - /* Non-zero if we have dropped our controlling tty and therefore should not open a frame on stdout. */ static int no_controlling_tty; @@ -497,8 +493,8 @@ tty_clear_end_of_line (struct frame *f, int first_unused_hpos) static unsigned char *encode_terminal_src; static unsigned char *encode_terminal_dst; /* Allocated sizes of the above buffers. */ -static int encode_terminal_src_size; -static int encode_terminal_dst_size; +static ptrdiff_t encode_terminal_src_size; +static ptrdiff_t encode_terminal_dst_size; /* Encode SRC_LEN glyphs starting at SRC to terminal output codes. Set CODING->produced to the byte-length of the resulting byte @@ -509,8 +505,8 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi { struct glyph *src_end = src + src_len; unsigned char *buf; - int nchars, nbytes, required; - register int tlen = GLYPH_TABLE_LENGTH; + ptrdiff_t nchars, nbytes, required; + ptrdiff_t tlen = GLYPH_TABLE_LENGTH; register Lisp_Object *tbase = GLYPH_TABLE_BASE; Lisp_Object charset_list; @@ -518,13 +514,13 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi multibyte-form. But, it may be enlarged on demand if Vglyph_table contains a string or a composite glyph is encountered. */ - required = MAX_MULTIBYTE_LENGTH * src_len; + if (min (PTRDIFF_MAX, SIZE_MAX) / MAX_MULTIBYTE_LENGTH < src_len) + memory_full (SIZE_MAX); + required = src_len; + required *= MAX_MULTIBYTE_LENGTH; if (encode_terminal_src_size < required) { - if (encode_terminal_src) - encode_terminal_src = xrealloc (encode_terminal_src, required); - else - encode_terminal_src = xmalloc (required); + encode_terminal_src = xrealloc (encode_terminal_src, required); encode_terminal_src_size = required; } @@ -544,19 +540,23 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi if (src->u.cmp.automatic) { gstring = composition_gstring_from_id (src->u.cmp.id); - required = src->slice.cmp.to + 1 - src->slice.cmp.from; + required = src->slice.cmp.to - src->slice.cmp.from + 1; } else { cmp = composition_table[src->u.cmp.id]; - required = MAX_MULTIBYTE_LENGTH * cmp->glyph_len; + required = cmp->glyph_len; + required *= MAX_MULTIBYTE_LENGTH; } - if (encode_terminal_src_size < nbytes + required) + if (encode_terminal_src_size - nbytes < required) { - encode_terminal_src_size = nbytes + required; - encode_terminal_src = xrealloc (encode_terminal_src, - encode_terminal_src_size); + ptrdiff_t size; + if (min (PTRDIFF_MAX, SIZE_MAX) - nbytes < required) + memory_full (SIZE_MAX); + size = nbytes + required; + encode_terminal_src = xrealloc (encode_terminal_src, size); + encode_terminal_src_size = size; buf = encode_terminal_src + nbytes; } @@ -627,11 +627,15 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi if (NILP (string)) { nbytes = buf - encode_terminal_src; - if (encode_terminal_src_size < nbytes + MAX_MULTIBYTE_LENGTH) + if (encode_terminal_src_size - nbytes < MAX_MULTIBYTE_LENGTH) { - encode_terminal_src_size = nbytes + MAX_MULTIBYTE_LENGTH; - encode_terminal_src = xrealloc (encode_terminal_src, - encode_terminal_src_size); + ptrdiff_t size; + if (min (PTRDIFF_MAX, SIZE_MAX) - MAX_MULTIBYTE_LENGTH + < nbytes) + memory_full (SIZE_MAX); + size = nbytes + MAX_MULTIBYTE_LENGTH; + encode_terminal_src = xrealloc (encode_terminal_src, size); + encode_terminal_src_size = size; buf = encode_terminal_src + nbytes; } if (CHAR_BYTE8_P (c) @@ -659,11 +663,14 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi if (! STRING_MULTIBYTE (string)) string = string_to_multibyte (string); nbytes = buf - encode_terminal_src; - if (encode_terminal_src_size < nbytes + SBYTES (string)) + if (encode_terminal_src_size - nbytes < SBYTES (string)) { - encode_terminal_src_size = nbytes + SBYTES (string); - encode_terminal_src = xrealloc (encode_terminal_src, - encode_terminal_src_size); + ptrdiff_t size; + if (min (PTRDIFF_MAX, SIZE_MAX) - SBYTES (string) < nbytes) + memory_full (SIZE_MAX); + size = nbytes + SBYTES (string); + encode_terminal_src = xrealloc (encode_terminal_src, size); + encode_terminal_src_size = size; buf = encode_terminal_src + nbytes; } memcpy (buf, SDATA (string), SBYTES (string)); @@ -684,12 +691,9 @@ encode_terminal_code (struct glyph *src, int src_len, struct coding_system *codi coding->source = encode_terminal_src; if (encode_terminal_dst_size == 0) { + encode_terminal_dst = xrealloc (encode_terminal_dst, + encode_terminal_src_size); encode_terminal_dst_size = encode_terminal_src_size; - if (encode_terminal_dst) - encode_terminal_dst = xrealloc (encode_terminal_dst, - encode_terminal_dst_size); - else - encode_terminal_dst = xmalloc (encode_terminal_dst_size); } coding->destination = encode_terminal_dst; coding->dst_bytes = encode_terminal_dst_size; @@ -1156,18 +1160,14 @@ calculate_costs (struct frame *frame) char_ins_del_vector (i.e., char_ins_del_cost) isn't used because X turns off char_ins_del_ok. */ - max_frame_lines = max (max_frame_lines, FRAME_LINES (frame)); max_frame_cols = max (max_frame_cols, FRAME_COLS (frame)); + if ((min (PTRDIFF_MAX, SIZE_MAX) / sizeof (int) - 1) / 2 < max_frame_cols) + memory_full (SIZE_MAX); - if (char_ins_del_vector != 0) - char_ins_del_vector - = (int *) xrealloc (char_ins_del_vector, - (sizeof (int) - + 2 * max_frame_cols * sizeof (int))); - else - char_ins_del_vector - = (int *) xmalloc (sizeof (int) - + 2 * max_frame_cols * sizeof (int)); + char_ins_del_vector + = (int *) xrealloc (char_ins_del_vector, + (sizeof (int) + + 2 * max_frame_cols * sizeof (int))); memset (char_ins_del_vector, 0, (sizeof (int) + 2 * max_frame_cols * sizeof (int)));