-;;; utf-8.el --- limited UTF-8 decoding/encoding support -*- coding: iso-2022-7bit -*-
+;;; utf-8.el --- UTF-8 decoding/encoding support -*- coding: iso-2022-7bit -*-
;; Copyright (C) 2001 Electrotechnical Laboratory, JAPAN.
;; Licensed to the Free Software Foundation.
-;; Copyright (C) 2001 Free Software Foundation, Inc.
+;; Copyright (C) 2001, 2002 Free Software Foundation, Inc.
;; Author: TAKAHASHI Naoto <ntakahas@m17n.org>
+;; Maintainer: FSF
;; Keywords: multilingual, Unicode, UTF-8, i18n
;; This file is part of GNU Emacs.
;; On decoding, Unicode characters that do not fit into the above
;; character sets are handled as `eight-bit-control' or
;; `eight-bit-graphic' characters to retain the information about the
-;; original byte sequence.
+;; original byte sequence and text properties record the corresponding
+;; unicode.
+;;
+;; Fixme: note that reading and writing invalid utf-8 may not be
+;; idempotent -- to represent the bytes to fix that needs a new charset.
;;
;; Characters from other character sets can be encoded with
;; mule-utf-8 by populating the table `ucs-mule-to-mule-unicode' and
-;; registering the translation with `register-char-codings'.
+;; registering the translation with `register-char-codings'. Hash
+;; tables `utf-8-subst-table' and `utf-8-subst-rev-table' are used to
+;; support encoding and decoding of about a quarter of the CJK space
+;; between U+3400 and U+DFFF.
;; UTF-8 is defined in RFC 2279. A sketch of the encoding is:
"Translation table for encoding to `mule-utf-8'.")
;; Could have been done by ucs-tables loaded before.
(unless (get 'ucs-mule-to-mule-unicode 'translation-table)
- (define-translation-table 'ucs-mule-to-mule-unicode ucs-mule-to-mule-unicode))
+ (define-translation-table 'ucs-mule-to-mule-unicode
+ ucs-mule-to-mule-unicode))
+
+(defvar utf-8-subst-table (make-hash-table :test 'eq))
+(defvar utf-8-subst-rev-table (make-hash-table :test 'eq))
+(define-translation-hash-table 'utf-8-subst-table utf-8-subst-table)
+(define-translation-hash-table 'utf-8-subst-rev-table utf-8-subst-rev-table)
+
+(defvar utf-8-translation-table-for-decode (make-translation-table)
+ "Translation table applied after decoding utf-8 to mule-unicode.
+This is only actually applied to characters which would normally be
+decoded into mule-unicode-0100-24ff.")
+(define-translation-table 'utf-8-translation-table-for-decode
+ utf-8-translation-table-for-decode)
+
+;; Map Cyrillic and Greek to iso-8859 charsets, which take half the
+;; space of mule-unicode. For Latin scripts this isn't very
+;; important. Hebrew and Arabic might go here too when there's proper
+;; support for them.
+(mapc
+ (lambda (pair)
+ (aset utf-8-translation-table-for-decode (car pair) (cdr pair)))
+ '((?\e$,1&d\e(B . ?\e,F4\e(B) (?\e$,1&e\e(B . ?\e,F5\e(B) (?\e$,1&f\e(B . ?\e,F6\e(B) (?\e$,1&h\e(B . ?\e,F8\e(B) (?\e$,1&i\e(B . ?\e,F9\e(B)
+ (?\e$,1&j\e(B . ?\e,F:\e(B) (?\e$,1&l\e(B . ?\e,F<\e(B) (?\e$,1&n\e(B . ?\e,F>\e(B) (?\e$,1&o\e(B . ?\e,F?\e(B) (?\e$,1&p\e(B . ?\e,F@\e(B)
+ (?\e$,1&q\e(B . ?\e,FA\e(B) (?\e$,1&r\e(B . ?\e,FB\e(B) (?\e$,1&s\e(B . ?\e,FC\e(B) (?\e$,1&t\e(B . ?\e,FD\e(B) (?\e$,1&u\e(B . ?\e,FE\e(B)
+ (?\e$,1&v\e(B . ?\e,FF\e(B) (?\e$,1&w\e(B . ?\e,FG\e(B) (?\e$,1&x\e(B . ?\e,FH\e(B) (?\e$,1&y\e(B . ?\e,FI\e(B) (?\e$,1&z\e(B . ?\e,FJ\e(B)
+ (?\e$,1&{\e(B . ?\e,FK\e(B) (?\e$,1&|\e(B . ?\e,FL\e(B) (?\e$,1&}\e(B . ?\e,FM\e(B) (?\e$,1&~\e(B . ?\e,FN\e(B) (?\e$,1&\7f\e(B . ?\e,FO\e(B)
+ (?\e$,1' \e(B . ?\e,FP\e(B) (?\e$,1'!\e(B . ?\e,FQ\e(B) (?\e$,1'#\e(B . ?\e,FS\e(B) (?\e$,1'$\e(B . ?\e,FT\e(B) (?\e$,1'%\e(B . ?\e,FU\e(B)
+ (?\e$,1'&\e(B . ?\e,FV\e(B) (?\e$,1''\e(B . ?\e,FW\e(B) (?\e$,1'(\e(B . ?\e,FX\e(B) (?\e$,1')\e(B . ?\e,FY\e(B) (?\e$,1'*\e(B . ?\e,FZ\e(B)
+ (?\e$,1'+\e(B . ?\e,F[\e(B) (?\e$,1',\e(B . ?\e,F\\e(B) (?\e$,1'-\e(B . ?\e,F]\e(B) (?\e$,1'.\e(B . ?\e,F^\e(B) (?\e$,1'/\e(B . ?\e,F_\e(B)
+ (?\e$,1'0\e(B . ?\e,F`\e(B) (?\e$,1'1\e(B . ?\e,Fa\e(B) (?\e$,1'2\e(B . ?\e,Fb\e(B) (?\e$,1'3\e(B . ?\e,Fc\e(B) (?\e$,1'4\e(B . ?\e,Fd\e(B)
+ (?\e$,1'5\e(B . ?\e,Fe\e(B) (?\e$,1'6\e(B . ?\e,Ff\e(B) (?\e$,1'7\e(B . ?\e,Fg\e(B) (?\e$,1'8\e(B . ?\e,Fh\e(B) (?\e$,1'9\e(B . ?\e,Fi\e(B)
+ (?\e$,1':\e(B . ?\e,Fj\e(B) (?\e$,1';\e(B . ?\e,Fk\e(B) (?\e$,1'<\e(B . ?\e,Fl\e(B) (?\e$,1'=\e(B . ?\e,Fm\e(B) (?\e$,1'>\e(B . ?\e,Fn\e(B)
+ (?\e$,1'?\e(B . ?\e,Fo\e(B) (?\e$,1'@\e(B . ?\e,Fp\e(B) (?\e$,1'A\e(B . ?\e,Fq\e(B) (?\e$,1'B\e(B . ?\e,Fr\e(B) (?\e$,1'C\e(B . ?\e,Fs\e(B)
+ (?\e$,1'D\e(B . ?\e,Ft\e(B) (?\e$,1'E\e(B . ?\e,Fu\e(B) (?\e$,1'F\e(B . ?\e,Fv\e(B) (?\e$,1'G\e(B . ?\e,Fw\e(B) (?\e$,1'H\e(B . ?\e,Fx\e(B)
+ (?\e$,1'I\e(B . ?\e,Fy\e(B) (?\e$,1'J\e(B . ?\e,Fz\e(B) (?\e$,1'K\e(B . ?\e,F{\e(B) (?\e$,1'L\e(B . ?\e,F|\e(B) (?\e$,1'M\e(B . ?\e,F}\e(B)
+ (?\e$,1'N\e(B . ?\e,F~\e(B)
+
+ (?\e$,1(!\e(B . ?\e,L!\e(B) (?\e$,1("\e(B . ?\e,L"\e(B) (?\e$,1(#\e(B . ?\e,L#\e(B) (?\e$,1($\e(B . ?\e,L$\e(B)
+ (?\e$,1(%\e(B . ?\e,L%\e(B) (?\e$,1(&\e(B . ?\e,L&\e(B) (?\e$,1('\e(B . ?\e,L'\e(B) (?\e$,1((\e(B . ?\e,L(\e(B) (?\e$,1()\e(B . ?\e,L)\e(B)
+ (?\e$,1(*\e(B . ?\e,L*\e(B) (?\e$,1(+\e(B . ?\e,L+\e(B) (?\e$,1(,\e(B . ?\e,L,\e(B) (?\e$,1(.\e(B . ?\e,L.\e(B) (?\e$,1(/\e(B . ?\e,L/\e(B)
+ (?\e$,1(0\e(B . ?\e,L0\e(B) (?\e$,1(1\e(B . ?\e,L1\e(B) (?\e$,1(2\e(B . ?\e,L2\e(B) (?\e$,1(3\e(B . ?\e,L3\e(B) (?\e$,1(4\e(B . ?\e,L4\e(B)
+ (?\e$,1(5\e(B . ?\e,L5\e(B) (?\e$,1(6\e(B . ?\e,L6\e(B) (?\e$,1(7\e(B . ?\e,L7\e(B) (?\e$,1(8\e(B . ?\e,L8\e(B) (?\e$,1(9\e(B . ?\e,L9\e(B)
+ (?\e$,1(:\e(B . ?\e,L:\e(B) (?\e$,1(;\e(B . ?\e,L;\e(B) (?\e$,1(<\e(B . ?\e,L<\e(B) (?\e$,1(=\e(B . ?\e,L=\e(B) (?\e$,1(>\e(B . ?\e,L>\e(B)
+ (?\e$,1(?\e(B . ?\e,L?\e(B) (?\e$,1(@\e(B . ?\e,L@\e(B) (?\e$,1(A\e(B . ?\e,LA\e(B) (?\e$,1(B\e(B . ?\e,LB\e(B) (?\e$,1(C\e(B . ?\e,LC\e(B)
+ (?\e$,1(D\e(B . ?\e,LD\e(B) (?\e$,1(E\e(B . ?\e,LE\e(B) (?\e$,1(F\e(B . ?\e,LF\e(B) (?\e$,1(G\e(B . ?\e,LG\e(B) (?\e$,1(H\e(B . ?\e,LH\e(B)
+ (?\e$,1(I\e(B . ?\e,LI\e(B) (?\e$,1(J\e(B . ?\e,LJ\e(B) (?\e$,1(K\e(B . ?\e,LK\e(B) (?\e$,1(L\e(B . ?\e,LL\e(B) (?\e$,1(M\e(B . ?\e,LM\e(B)
+ (?\e$,1(N\e(B . ?\e,LN\e(B) (?\e$,1(O\e(B . ?\e,LO\e(B) (?\e$,1(P\e(B . ?\e,LP\e(B) (?\e$,1(Q\e(B . ?\e,LQ\e(B) (?\e$,1(R\e(B . ?\e,LR\e(B)
+ (?\e$,1(S\e(B . ?\e,LS\e(B) (?\e$,1(T\e(B . ?\e,LT\e(B) (?\e$,1(U\e(B . ?\e,LU\e(B) (?\e$,1(V\e(B . ?\e,LV\e(B) (?\e$,1(W\e(B . ?\e,LW\e(B)
+ (?\e$,1(X\e(B . ?\e,LX\e(B) (?\e$,1(Y\e(B . ?\e,LY\e(B) (?\e$,1(Z\e(B . ?\e,LZ\e(B) (?\e$,1([\e(B . ?\e,L[\e(B) (?\e$,1(\\e(B . ?\e,L\\e(B)
+ (?\e$,1(]\e(B . ?\e,L]\e(B) (?\e$,1(^\e(B . ?\e,L^\e(B) (?\e$,1(_\e(B . ?\e,L_\e(B) (?\e$,1(`\e(B . ?\e,L`\e(B) (?\e$,1(a\e(B . ?\e,La\e(B)
+ (?\e$,1(b\e(B . ?\e,Lb\e(B) (?\e$,1(c\e(B . ?\e,Lc\e(B) (?\e$,1(d\e(B . ?\e,Ld\e(B) (?\e$,1(e\e(B . ?\e,Le\e(B) (?\e$,1(f\e(B . ?\e,Lf\e(B)
+ (?\e$,1(g\e(B . ?\e,Lg\e(B) (?\e$,1(h\e(B . ?\e,Lh\e(B) (?\e$,1(i\e(B . ?\e,Li\e(B) (?\e$,1(j\e(B . ?\e,Lj\e(B) (?\e$,1(k\e(B . ?\e,Lk\e(B)
+ (?\e$,1(l\e(B . ?\e,Ll\e(B) (?\e$,1(m\e(B . ?\e,Lm\e(B) (?\e$,1(n\e(B . ?\e,Ln\e(B) (?\e$,1(o\e(B . ?\e,Lo\e(B) (?\e$,1(q\e(B . ?\e,Lq\e(B)
+ (?\e$,1(r\e(B . ?\e,Lr\e(B) (?\e$,1(s\e(B . ?\e,Ls\e(B) (?\e$,1(t\e(B . ?\e,Lt\e(B) (?\e$,1(u\e(B . ?\e,Lu\e(B) (?\e$,1(v\e(B . ?\e,Lv\e(B)
+ (?\e$,1(w\e(B . ?\e,Lw\e(B) (?\e$,1(x\e(B . ?\e,Lx\e(B) (?\e$,1(y\e(B . ?\e,Ly\e(B) (?\e$,1(z\e(B . ?\e,Lz\e(B) (?\e$,1({\e(B . ?\e,L{\e(B)
+ (?\e$,1(|\e(B . ?\e,L|\e(B) (?\e$,1(~\e(B . ?\e,L~\e(B) (?\e$,1(\7f\e(B . ?\e,L\7f\e(B)))
+
+(defcustom utf-8-fragment-on-decoding nil
+ "Whether or not to decode some scripts in UTF-8 text into 8-bit characters.
+Setting this means that the relevant Cyrillic and Greek characters are
+decoded into the iso8859 charsets rather than into
+mule-unicode-0100-24ff. The 8-bit characters take half as much space
+in the buffer, but using them may affect how the buffer can be re-encoded
+and may require a different input method to search for them, for instance.
+See `unify-8859-on-decoding-mode' and `unify-8859-on-encoding-mode'
+for mechanisms to make this largely transparent."
+ :set (lambda (s v)
+ (if v
+ (define-translation-table 'utf-8-translation-table-for-decode
+ utf-8-translation-table-for-decode)
+ (define-translation-table 'utf-8-translation-table-for-decode))
+ (set-default s v))
+ :version "21.4"
+ :type 'boolean
+ :group 'mule)
+
+(defcustom utf-8-translate-cjk nil
+ "Whether the `mule-utf-8' coding system should encode many CJK characters.
+
+Enabling this loads tables which enable the coding system to encode
+characters in the charsets `korean-ksc5601', `chinese-gb2312' and
+`japanese-jisx0208', and to decode the corresponding unicodes into
+such characters. This works by loading the library `utf-8-subst'; see
+its commentary. The tables are fairly large (about 33000 entries), so this
+option is not the default."
+ :link '(emacs-commentary-link "utf-8-subst")
+ :set (lambda (s v)
+ (when v
+ (require 'utf-8-subst)
+ (let ((table (make-char-table 'translation-table)))
+ (coding-system-put 'mule-utf-8 'safe-charsets
+ (append (coding-system-get 'mule-utf-8
+ 'safe-charsets)
+ '(korean-ksc5601 chinese-gb2312
+ japanese-jisx0208)))
+ (maphash (lambda (k v)
+ (aset table k v))
+ utf-8-subst-rev-table)
+ (register-char-codings 'mule-utf-8 table)))
+ (set-default s v))
+ :version "21.4"
+ :type 'boolean
+ :group 'mule)
+
(define-ccl-program ccl-decode-mule-utf-8
;;
;; charset | bytes in utf-8 | bytes in emacs
;; 1byte encoding, i.e., ascii
(if (r0 < #x80)
(write r0)
-
- ;; 2 byte encoding 00000yyyyyxxxxxx = 110yyyyy 10xxxxxx
- (if (r0 < #xe0)
- ((read r1)
-
- (if ((r1 & #b11000000) != #b10000000)
- ;; Invalid 2-byte sequence
- ((if (r0 < #xa0)
- (write-multibyte-character r5 r0)
- (write-multibyte-character r6 r0))
- (if (r1 < #x80)
- (write r1)
- (if (r1 < #xa0)
- (write-multibyte-character r5 r1)
- (write-multibyte-character r6 r1))))
-
- ((r0 &= #x1f)
- (r0 <<= 6)
- (r1 &= #x3f)
- (r1 += r0)
- ;; Now r1 holds scalar value
-
- ;; eight-bit-control
- (if (r1 < 160)
- ((write-multibyte-character r5 r1))
-
- ;; latin-iso8859-1
- (if (r1 < 256)
- ((r0 = ,(charset-id 'latin-iso8859-1))
- (r1 -= 128)
- (write-multibyte-character r0 r1))
-
- ;; mule-unicode-0100-24ff (< 0800)
- ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
- (r1 -= #x0100)
- (r2 = (((r1 / 96) + 32) << 7))
- (r1 %= 96)
- (r1 += (r2 + 32))
- (write-multibyte-character r0 r1)))))))
-
- ;; 3byte encoding
- ;; zzzzyyyyyyxxxxxx = 1110zzzz 10yyyyyy 10xxxxxx
- (if (r0 < #xf0)
- ((read r1 r2)
-
- ;; This is set to 1 if the encoding is invalid.
- (r4 = 0)
-
- (r3 = (r1 & #b11000000))
- (r3 |= ((r2 >> 2) & #b00110000))
- (if (r3 != #b10100000)
- (r4 = 1)
- ((r3 = ((r0 & #x0f) << 12))
- (r3 += ((r1 & #x3f) << 6))
- (r3 += (r2 & #x3f))
- (if (r3 < #x0800)
- (r4 = 1))))
-
- (if (r4 != 0)
- ;; Invalid 3-byte sequence
+ (if (r0 < #xc0) ; continuation byte (invalid here)
+ (if (r0 < #xa0)
+ (write-multibyte-character r5 r0)
+ (write-multibyte-character r6 r0))
+ ;; 2 byte encoding 00000yyyyyxxxxxx = 110yyyyy 10xxxxxx
+ (if (r0 < #xe0)
+ ((read r1)
+
+ (if ((r1 & #b11000000) != #b10000000)
+ ;; Invalid 2-byte sequence
((if (r0 < #xa0)
(write-multibyte-character r5 r0)
(write-multibyte-character r6 r0))
(write r1)
(if (r1 < #xa0)
(write-multibyte-character r5 r1)
- (write-multibyte-character r6 r1)))
- (if (r2 < #x80)
- (write r2)
- (if (r2 < #xa0)
- (write-multibyte-character r5 r2)
- (write-multibyte-character r6 r2))))
+ (write-multibyte-character r6 r1))))
+
+ ((r3 = r0) ; save in case of overlong sequence
+ (r2 = r1)
+ (r0 &= #x1f)
+ (r0 <<= 6)
+ (r2 = r1) ; save in case of overlong sequence
+ (r1 &= #x3f)
+ (r1 += r0)
+ ;; Now r1 holds scalar value
+
+ (if (r1 < 128) ; `overlong sequence'
+ ((if (r3 < #xa0)
+ (write-multibyte-character r5 r3)
+ (write-multibyte-character r6 r3))
+ (if (r2 < #x80)
+ (write r2)
+ (if (r2 < #xa0)
+ (write-multibyte-character r5 r2)
+ (write-multibyte-character r6 r2))))
+
+ ;; eight-bit-control
+ (if (r1 < 160)
+ ((write-multibyte-character r5 r1))
+
+ ;; latin-iso8859-1
+ (if (r1 < 256)
+ ((r0 = ,(charset-id 'latin-iso8859-1))
+ (r1 -= 128)
+ (write-multibyte-character r0 r1))
+
+ ;; mule-unicode-0100-24ff (< 0800)
+ ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
+ (r1 -= #x0100)
+ (r2 = (((r1 / 96) + 32) << 7))
+ (r1 %= 96)
+ (r1 += (r2 + 32))
+ (translate-character
+ utf-8-translation-table-for-decode r0 r1)
+ (write-multibyte-character r0 r1))))))))
+
+ ;; 3byte encoding
+ ;; zzzzyyyyyyxxxxxx = 1110zzzz 10yyyyyy 10xxxxxx
+ (if (r0 < #xf0)
+ ((read r1 r2)
+
+ ;; This is set to 1 if the encoding is invalid.
+ (r4 = 0)
+
+ (r3 = (r1 & #b11000000))
+ (r3 |= ((r2 >> 2) & #b00110000))
+ (if (r3 != #b10100000)
+ (r4 = 1)
+ ((r3 = ((r0 & #x0f) << 12))
+ (r3 += ((r1 & #x3f) << 6))
+ (r3 += (r2 & #x3f))
+ (if (r3 < #x0800)
+ (r4 = 1))))
+
+ (if (r4 != 0)
+ ;; Invalid 3-byte sequence
+ ((if (r0 < #xa0)
+ (write-multibyte-character r5 r0)
+ (write-multibyte-character r6 r0))
+ (if (r1 < #x80)
+ (write r1)
+ (if (r1 < #xa0)
+ (write-multibyte-character r5 r1)
+ (write-multibyte-character r6 r1)))
+ (if (r2 < #x80)
+ (write r2)
+ (if (r2 < #xa0)
+ (write-multibyte-character r5 r2)
+ (write-multibyte-character r6 r2))))
- ;; mule-unicode-0100-24ff (>= 0800)
- ((if (r3 < #x2500)
- ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
- (r3 -= #x0100)
- (r3 //= 96)
- (r1 = (r7 + 32))
- (r1 += ((r3 + 32) << 7))
- (write-multibyte-character r0 r1))
-
- ;; mule-unicode-2500-33ff
- (if (r3 < #x3400)
- ((r0 = ,(charset-id 'mule-unicode-2500-33ff))
- (r3 -= #x2500)
+ ;; mule-unicode-0100-24ff (>= 0800)
+ ((if (r3 < #x2500)
+ ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
+ (r3 -= #x0100)
(r3 //= 96)
(r1 = (r7 + 32))
(r1 += ((r3 + 32) << 7))
+ (translate-character
+ utf-8-translation-table-for-decode r0 r1)
(write-multibyte-character r0 r1))
-
- ;; U+3400 .. U+DFFF
- ;; keep those bytes as eight-bit-{control|graphic}
- (if (r3 < #xe000)
- ( ;; #xe0 <= r0 < #xf0, so r0 is eight-bit-graphic
- (r3 = r6)
- (write-multibyte-character r3 r0)
- (if (r1 < #xa0)
- (r3 = r5))
- (write-multibyte-character r3 r1)
- (if (r2 < #xa0)
- (r3 = r5)
- (r3 = r6))
- (write-multibyte-character r3 r2))
+
+ ;; mule-unicode-2500-33ff
+ ;; Fixme: Perhaps allow translation via
+ ;; utf-8-subst-table for #x2e80 up, so that we use
+ ;; consistent charsets for all of CJK. Would need
+ ;; corresponding change to encoding tables.
+ (if (r3 < #x3400)
+ ((r0 = ,(charset-id 'mule-unicode-2500-33ff))
+ (r3 -= #x2500)
+ (r3 //= 96)
+ (r1 = (r7 + 32))
+ (r1 += ((r3 + 32) << 7))
+ (write-multibyte-character r0 r1))
+
+ ;; U+3400 .. U+D7FF
+ ;; Try to convert to CJK chars, else keep
+ ;; them as eight-bit-{control|graphic}.
+ (if (r3 < #xd800)
+ ((r4 = r3) ; don't zap r3
+ (lookup-integer utf-8-subst-table r4 r5)
+ (if r7
+ ;; got a translation
+ ((write-multibyte-character r4 r5)
+ ;; Zapped through register starvation.
+ (r5 = ,(charset-id 'eight-bit-control)))
+ ;; #xe0 <= r0 < #xf0, so r0 is eight-bit-graphic
+ ((r3 = r6)
+ (write-multibyte-character r3 r0)
+ (if (r1 < #xa0)
+ (r3 = r5))
+ (write-multibyte-character r3 r1)
+ (if (r2 < #xa0)
+ (r3 = r5)
+ (r3 = r6))
+ (write-multibyte-character r3 r2))))
+
+ ;; Surrogates, U+D800 .. U+DFFF
+ ;; Fixme: process them properly.
+ (if (r3 < #xe000)
+ ((r3 = r6)
+ (write-multibyte-character r3 r0) ; eight-bit-graphic
+ (if (r1 < #xa0)
+ (r3 = r5))
+ (write-multibyte-character r3 r1)
+ (if (r2 < #xa0)
+ (r3 = r5)
+ (r3 = r6))
+ (write-multibyte-character r3 r2))
- ;; mule-unicode-e000-ffff
- ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
- (r3 -= #xe000)
- (r3 //= 96)
- (r1 = (r7 + 32))
- (r1 += ((r3 + 32) << 7))
- (write-multibyte-character r0 r1))))))))
-
- ;; 4byte encoding
- ;; keep those bytes as eight-bit-{control|graphic}
- ((read r1 r2 r3)
- ;; r0 > #xf0, thus eight-bit-graphic
- (write-multibyte-character r6 r0)
- (if (r1 < #xa0)
- (write-multibyte-character r5 r1)
- (write-multibyte-character r6 r1))
- (if (r2 < #xa0)
- (write-multibyte-character r5 r2)
- (write-multibyte-character r6 r2))
- (if (r3 < #xa0)
- (write-multibyte-character r5 r3)
- (write-multibyte-character r6 r3))))))
-
+ ;; mule-unicode-e000-ffff
+ ;; Fixme: fffe and ffff are invalid.
+ ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
+ (r3 -= #xe000)
+ (r3 //= 96)
+ (r1 = (r7 + 32))
+ (r1 += ((r3 + 32) << 7))
+ (write-multibyte-character r0 r1)))))))))
+
+ (if (r0 < #xfe)
+ ;; 4byte encoding
+ ;; keep those bytes as eight-bit-{control|graphic}
+ ;; Fixme: allow lookup in utf-8-subst-table.
+ ((read r1 r2 r3)
+ ;; r0 > #xf0, thus eight-bit-graphic
+ (write-multibyte-character r6 r0)
+ (if (r1 < #xa0)
+ (if (r1 < #x80) ; invalid byte
+ (write r1)
+ (write-multibyte-character r5 r1))
+ (write-multibyte-character r6 r1))
+ (if (r2 < #xa0)
+ (if (r2 < #x80) ; invalid byte
+ (write r2)
+ (write-multibyte-character r5 r2))
+ (write-multibyte-character r6 r2))
+ (if (r3 < #xa0)
+ (if (r3 < #x80) ; invalid byte
+ (write r3)
+ (write-multibyte-character r5 r3))
+ (write-multibyte-character r6 r3))
+ (if (r0 >= #xf8) ; 5- or 6-byte encoding
+ ((read r1)
+ (if (r1 < #xa0)
+ (if (r1 < #x80) ; invalid byte
+ (write r1)
+ (write-multibyte-character r5 r1))
+ (write-multibyte-character r6 r1))
+ (if (r0 >= #xfc) ; 6-byte
+ ((read r1)
+ (if (r1 < #xa0)
+ (if (r1 < #x80) ; invalid byte
+ (write r1)
+ (write-multibyte-character r5 r1))
+ (write-multibyte-character r6 r1)))))))
+ ;; else invalid byte >= #xfe
+ (write-multibyte-character r6 r0))))))
(repeat))))
"CCL program to decode UTF-8.
Basic decoding is done into the charsets ascii, latin-iso8859-1 and
-mule-unicode-*. Encodings of un-representable Unicode characters are
-decoded asis into eight-bit-control and eight-bit-graphic
-characters.")
+mule-unicode-*, but see also `utf-8-translation-table-for-decode' and
+`utf-8-subst-table'.
+Encodings of un-representable Unicode characters are decoded asis into
+eight-bit-control and eight-bit-graphic characters.")
(define-ccl-program ccl-encode-mule-utf-8
`(1
(if (r0 == ,(charset-id 'mule-unicode-e000-ffff))
((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96))
(r1 &= #x7f)
- (r1 += (r0 + 57312)) ; 57312 == -160 + #xe000
+ (r1 += (r0 + 57312)) ; 57312 == -32 + #xe000
(r0 = (((r1 & #xf000) >> 12) | #xe0))
(r2 = ((r1 & #x3f) | #x80))
(r1 &= #x0fc0)
((write #xc2)
(write r1)))))))
- ;; Unsupported character.
- ;; Output U+FFFD, which is `ef bf bd' in UTF-8.
- ((write #xef)
- (write #xbf)
- (write #xbd)))))))))
+ ((lookup-character utf-8-subst-rev-table r0 r1)
+ (if r7 ; lookup succeeded
+ ((r1 = (((r0 & #xf000) >> 12) | #xe0))
+ (r2 = ((r0 & #x3f) | #x80))
+ (r0 &= #x0fc0)
+ (r0 >>= 6)
+ (r0 |= #x80)
+ (write r1 r0 r2))
+ ;; Unsupported character.
+ ;; Output U+FFFD, which is `ef bf bd' in UTF-8.
+ ((write #xef)
+ (write #xbf)
+ (write #xbd)))))))))))
(repeat)))
(if (r1 >= #xa0)
(write r1)
((write #xc2)
(write r1)))))
- "CCL program to encode into UTF-8.
-Only characters from the charsets ascii, eight-bit-control,
-eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are recognized.
-Others are encoded as U+FFFD.")
+ "CCL program to encode into UTF-8.")
;; Dummy definition so that the CCL can be checked correctly; the
;; actual data are loaded on demand.
(unless (boundp 'ucs-mule-8859-to-mule-unicode) ; don't zap it
(define-translation-table 'ucs-mule-8859-to-mule-unicode))
+(define-ccl-program ccl-untranslated-to-ucs
+ `(0
+ (if (r0 < #xf0) ; 3-byte encoding, as above
+ ((r4 = 0)
+ (r3 = (r1 & #b11000000))
+ (r3 |= ((r2 >> 2) & #b00110000))
+ (if (r3 != #b10100000)
+ (r4 = 1)
+ ((r3 = ((r0 & #x0f) << 12))
+ (r3 += ((r1 & #x3f) << 6))
+ (r3 += (r2 & #x3f))
+ (if (r3 < #x0800)
+ (r4 = 1))))
+ (if (r4 != 0)
+ (r0 = 0)
+ (r0 = r3)))
+ (if (r0 < #xf8) ; 4-byte (Mule-UCS recipe)
+ ((r4 = (r1 >> 6))
+ (if (r4 != #b10)
+ (r0 = 0)
+ ((r4 = (r2 >> 6))
+ (if (r4 != #b10)
+ (r0 = 0)
+ ((r4 = (r3 >> 6))
+ (if (r4 != #b10)
+ (r0 = 0)
+ ((r1 = ((r1 & #x3F) << 12))
+ (r2 = ((r2 & #x3F) << 6))
+ (r3 &= #x3F)
+ (r0 = (((((r0 & #x07) << 18) | r1) | r2) | r3)))))))))
+ (r0 = 0))))
+ "Decode 3- or 4-byte sequences in r0, r1, r2 [,r3] to unicodes in r0.
+r0 == 0 for invalid sequence.")
+
+(defvar utf-8-ccl-regs (make-vector 8 0))
+
(defsubst utf-8-untranslated-to-ucs ()
- (let ((b1 (char-after))
- (b2 (char-after (1+ (point))))
- (b3 (char-after (+ 2 (point))))
- (b4 (char-after (+ 4 (point)))))
- (if (and b1 b2 b3)
- (cond ((< b1 ?\xf0)
- (setq b2 (lsh (logand b2 ?\x3f) 6))
- (setq b3 (logand b3 ?\x3f))
- (logior b3 (logior b2 (lsh (logand b1 ?\x0f) 12))))
- (b4
- (setq b2 (lsh (logand b2 ?\x3f) 12))
- (setq b3 (lsh (logand b3 ?\x3f) 6))
- (setq b4 (logand b4 ?\x3f))
- (logior b4 (logior b3 (logior b2 (lsh (logand b1 ?\x07)
- 18)))))))))
+ "Return the UCS code for an untranslated sequence of raw bytes t point.
+Only for 3- or 4-byte sequences."
+ (aset utf-8-ccl-regs 0 (or (char-after) 0))
+ (aset utf-8-ccl-regs 1 (or (char-after (1+ (point))) 0))
+ (aset utf-8-ccl-regs 2 (or (char-after (+ 2 (point))) 0))
+ (aset utf-8-ccl-regs 3 (or (char-after (+ 3 (point))) 0))
+ (ccl-execute 'ccl-untranslated-to-ucs utf-8-ccl-regs)
+ (aref utf-8-ccl-regs 0))
(defun utf-8-help-echo (window object position)
(format "Untranslated Unicode U+%04X"
(get-char-property position 'untranslated-utf-8 object)))
-(defvar utf-8-subst-table nil
- "If non-nil, a hash table mapping `untranslatable utf-8' to Emacs characters.")
-
;; We compose the untranslatable sequences into a single character.
;; This is infelicitous for editing, because there's currently no
;; mechanism for treating compositions as atomic, but is OK for
-;; display. We try to compose an appropriate character from a hash
-;; table of CJK characters to display correctly. Otherwise we use
-;; U+FFFD. What we really should have is hash table lookup from CCL
-;; so that we could do this properly. This function GCs too much.
+;; display. They are composed to U+FFFD with help-echo which
+;; indicates the unicodes they represent. This function GCs too much.
(defsubst utf-8-compose ()
"Put a suitable composition on an untranslatable sequence.
Return the sequence's length."
(let* ((u (utf-8-untranslated-to-ucs))
- (l (and u (if (>= u ?\x10000)
+ (l (unless (zerop u)
+ (if (>= u #x10000)
4
- 3)))
- (subst (and utf-8-subst-table (gethash u utf-8-subst-table))))
- (when u
+ 3))))
+ (when l
(put-text-property (point) (min (point-max) (+ l (point)))
'untranslated-utf-8 u)
- (unless subst
- (put-text-property (point) (min (point-max) (+ l (point)))
- 'help-echo 'utf-8-help-echo)
- (setq subst ?\e$,3u=\e(B))
- (compose-region (point) (+ l (point)) subst)
+ (put-text-property (point) (min (point-max) (+ l (point)))
+ 'help-echo 'utf-8-help-echo)
+ (compose-region (point) (+ l (point)) ?\e$,3u=\e(B)
l)))
(defcustom utf-8-compose-scripts nil
- "*Non-nil means compose various scipts on decoding utf-8 text."
+ "*Non-nil means compose various scripts on decoding utf-8 text."
:group 'mule
- :type 'boolean) ; omitted in Emacs 21.1
+ :version "21.4"
+ :type 'boolean)
(defun utf-8-post-read-conversion (length)
"Compose untranslated utf-8 sequences into single characters.
;; Can't do eval-when-compile to insert a multibyte constant
;; version of the string in the loop, since it's always loaded as
;; unibyte from a byte-compiled file.
- (let ((range (string-as-multibyte "^\341-\377")))
- (while (and (skip-chars-forward
- range)
+ (let ((range (string-as-multibyte "^\xe1-\xf7")))
+ (while (and (skip-chars-forward range)
(not (eobp)))
(forward-char (utf-8-compose)))))
- ;; Fixme: Takahashi-san implies it may not work this easily -- needs
- ;; checking with him.
+ ;; Fixme: Takahashi-san implies it may not work this easily. I
+ ;; asked why but didn't get a reply. -- fx
(when (and utf-8-compose-scripts (> length 1))
;; These currently have definitions which cover the relevant
- ;; Unicodes. We could avoid loading thai-util &c by checking
+ ;; unicodes. We could avoid loading thai-util &c by checking
;; whether the region contains any characters with the appropriate
;; categories. There aren't yet Unicode-based rules for Tibetan.
(save-excursion (setq length (diacritic-post-read-conversion length)))
(save-excursion (setq length (thai-post-read-conversion length)))
(save-excursion (setq length (lao-post-read-conversion length)))
- (save-excursion (setq length (devanagari-post-read-conversion length))))
+ (save-excursion
+ (setq length (in-is13194-devanagari-post-read-conversion length))))
length)
-(defun utf-8-pre-write-conversion (beg end)
- "Semi-dummy pre-write function effectively to autoload ucs-tables."
- ;; Ensure translation table is loaded.
- (require 'ucs-tables)
- ;; Don't do this again.
- (coding-system-put 'mule-utf-8 'pre-write-conversion nil)
- nil)
+;; ucs-tables is preloaded
+;; (defun utf-8-pre-write-conversion (beg end)
+;; "Semi-dummy pre-write function effectively to autoload ucs-tables."
+;; ;; Ensure translation table is loaded.
+;; (require 'ucs-tables)
+;; ;; Don't do this again.
+;; (coding-system-put 'mule-utf-8 'pre-write-conversion nil)
+;; nil)
(make-coding-system
'mule-utf-8 4 ?u
"UTF-8 encoding for Emacs-supported Unicode characters.
-The supported Emacs character sets are the following, plus others
-which may be included in the translation table
-`ucs-mule-to-mule-unicode':
+The supported Emacs character sets are the following, plus any other
+characters included in the tables `ucs-mule-to-mule-unicode' and
+`utf-8-subst-rev-table':
ascii
eight-bit-control
eight-bit-graphic
mule-unicode-e000-ffff
Unicode characters out of the ranges U+0000-U+33FF and U+E200-U+FFFF
-are decoded into sequences of eight-bit-control and eight-bit-graphic
-characters to preserve their byte sequences and composed to display as
-a single character. Emacs characters that can't be encoded to these
-ranges are encoded as U+FFFD."
+may be decoded into korean-ksc5601, chinese-gb2312, japanese-jisx0208
+\(see user option `utf-8-translate-cjk'); otherwise, sequences of
+eight-bit-control and eight-bit-graphic characters are used to
+preserve their byte sequences, and these are composed to display as a
+single character. Emacs characters that otherwise can't be encoded
+are encoded as U+FFFD."
'(ccl-decode-mule-utf-8 . ccl-encode-mule-utf-8)
'((safe-charsets
(mime-charset . utf-8)
(coding-category . coding-category-utf-8)
(valid-codes (0 . 255))
- (pre-write-conversion . utf-8-pre-write-conversion)
+;; (pre-write-conversion . utf-8-pre-write-conversion)
(post-read-conversion . utf-8-post-read-conversion)))
(define-coding-system-alias 'utf-8 'mule-utf-8)
projects / emacs.git / commitdiff