1 |
;;; utf-8.el --- limited UTF-8 decoding/encoding support -*- coding: iso-2022-7bit -*- |
;;; utf-8.el --- UTF-8 decoding/encoding support -*- coding: iso-2022-7bit -*- |
2 |
|
|
3 |
;; Copyright (C) 2001 Electrotechnical Laboratory, JAPAN. |
;; Copyright (C) 2001 Electrotechnical Laboratory, JAPAN. |
4 |
;; Licensed to the Free Software Foundation. |
;; Licensed to the Free Software Foundation. |
5 |
;; Copyright (C) 2001 Free Software Foundation, Inc. |
;; Copyright (C) 2001, 2002 Free Software Foundation, Inc. |
6 |
|
|
7 |
;; Author: TAKAHASHI Naoto <ntakahas@m17n.org> |
;; Author: TAKAHASHI Naoto <ntakahas@m17n.org> |
8 |
|
;; Maintainer: FSF |
9 |
;; Keywords: multilingual, Unicode, UTF-8, i18n |
;; Keywords: multilingual, Unicode, UTF-8, i18n |
10 |
|
|
11 |
;; This file is part of GNU Emacs. |
;; This file is part of GNU Emacs. |
40 |
;; On decoding, Unicode characters that do not fit into the above |
;; On decoding, Unicode characters that do not fit into the above |
41 |
;; character sets are handled as `eight-bit-control' or |
;; character sets are handled as `eight-bit-control' or |
42 |
;; `eight-bit-graphic' characters to retain the information about the |
;; `eight-bit-graphic' characters to retain the information about the |
43 |
;; original byte sequence. |
;; original byte sequence and text properties record the corresponding |
44 |
|
;; unicode. |
45 |
;; |
;; |
46 |
;; Characters from other character sets can be encoded with |
;; Fixme: note that reading and writing invalid utf-8 may not be |
47 |
;; mule-utf-8 by populating the table `ucs-mule-to-mule-unicode' and |
;; idempotent -- to represent the bytes to fix that needs a new charset. |
48 |
;; registering the translation with `register-char-codings'. |
;; |
49 |
|
;; Characters from other character sets can be encoded with mule-utf-8 |
50 |
|
;; by populating the translation table |
51 |
|
;; `utf-translation-table-for-encode'. Hash tables |
52 |
|
;; `utf-subst-table-for-decode' and `utf-subst-table-for-encode' are |
53 |
|
;; used to support encoding and decoding of about a quarter of the CJK |
54 |
|
;; space between U+3400 and U+DFFF. |
55 |
|
|
56 |
;; UTF-8 is defined in RFC 2279. A sketch of the encoding is: |
;; UTF-8 is defined in RFC 2279. A sketch of the encoding is: |
57 |
|
|
64 |
|
|
65 |
;;; Code: |
;;; Code: |
66 |
|
|
67 |
(defvar ucs-mule-to-mule-unicode (make-translation-table) |
(defvar ucs-mule-to-mule-unicode (make-char-table 'translation-table nil) |
68 |
"Translation table for encoding to `mule-utf-8'.") |
"Char table mapping characters to latin-iso8859-1 or mule-unicode-*. |
69 |
;; Could have been done by ucs-tables loaded before. |
|
70 |
(unless (get 'ucs-mule-to-mule-unicode 'translation-table) |
If `unify-8859-on-encoding-mode' is non-nil, this table populates the |
71 |
(define-translation-table 'ucs-mule-to-mule-unicode ucs-mule-to-mule-unicode)) |
translation-table named `utf-translation-table-for-encode'.") |
72 |
|
|
73 |
|
(define-translation-table 'utf-translation-table-for-encode) |
74 |
|
|
75 |
|
|
76 |
|
;; Map Cyrillic and Greek to iso-8859 charsets, which take half the |
77 |
|
;; space of mule-unicode. For Latin scripts this isn't very |
78 |
|
;; important. Hebrew and Arabic might go here too when there's proper |
79 |
|
;; support for them. |
80 |
|
|
81 |
|
(defvar utf-fragmentation-table (make-char-table 'translation-table nil) |
82 |
|
"Char-table normally mapping non-Latin mule-unicode-* chars to iso-8859-*. |
83 |
|
|
84 |
|
If `utf-fragment-on-decoding' is non-nil, this table populates the |
85 |
|
translation-table named `utf-translation-table-for-decode'") |
86 |
|
|
87 |
|
(defvar utf-defragmentation-table (make-char-table 'translation-table nil) |
88 |
|
"Char-table for reverse mapping of `utf-fragmentation-table'. |
89 |
|
|
90 |
|
If `utf-fragment-on-decoding' is non-nil and |
91 |
|
`unify-8859-on-encoding-mode' is nil, this table populates the |
92 |
|
translation-table named `utf-translation-table-for-encode'") |
93 |
|
|
94 |
|
(define-translation-table 'utf-translation-table-for-decode) |
95 |
|
|
96 |
|
|
97 |
|
(defvar ucs-mule-cjk-to-unicode (make-hash-table :test 'eq) |
98 |
|
"Hash table mapping Emacs CJK character sets to Unicode code points. |
99 |
|
|
100 |
|
If `utf-translate-cjk' is non-nil, this table populates the |
101 |
|
translation-hash-table named `utf-subst-table-for-encode'.") |
102 |
|
|
103 |
|
(define-translation-hash-table 'utf-subst-table-for-encode |
104 |
|
ucs-mule-cjk-to-unicode) |
105 |
|
|
106 |
|
(defvar ucs-unicode-to-mule-cjk (make-hash-table :test 'eq) |
107 |
|
"Hash table mapping Unicode code points to Emacs CJK character sets. |
108 |
|
|
109 |
|
If `utf-translate-cjk' is non-nil, this table populates the |
110 |
|
translation-hash-table named `utf-subst-table-for-decode'.") |
111 |
|
|
112 |
|
(define-translation-hash-table 'utf-subst-table-for-decode |
113 |
|
ucs-unicode-to-mule-cjk) |
114 |
|
|
115 |
|
(mapc |
116 |
|
(lambda (pair) |
117 |
|
(aset utf-fragmentation-table (car pair) (cdr pair)) |
118 |
|
(aset utf-defragmentation-table (cdr pair) (car pair))) |
119 |
|
'((?$,1&d(B . ?,F4(B) (?$,1&e(B . ?,F5(B) (?$,1&f(B . ?,F6(B) (?$,1&h(B . ?,F8(B) (?$,1&i(B . ?,F9(B) |
120 |
|
(?$,1&j(B . ?,F:(B) (?$,1&l(B . ?,F<(B) (?$,1&n(B . ?,F>(B) (?$,1&o(B . ?,F?(B) (?$,1&p(B . ?,F@(B) |
121 |
|
(?$,1&q(B . ?,FA(B) (?$,1&r(B . ?,FB(B) (?$,1&s(B . ?,FC(B) (?$,1&t(B . ?,FD(B) (?$,1&u(B . ?,FE(B) |
122 |
|
(?$,1&v(B . ?,FF(B) (?$,1&w(B . ?,FG(B) (?$,1&x(B . ?,FH(B) (?$,1&y(B . ?,FI(B) (?$,1&z(B . ?,FJ(B) |
123 |
|
(?$,1&{(B . ?,FK(B) (?$,1&|(B . ?,FL(B) (?$,1&}(B . ?,FM(B) (?$,1&~(B . ?,FN(B) (?$,1&(B . ?,FO(B) |
124 |
|
(?$,1' (B . ?,FP(B) (?$,1'!(B . ?,FQ(B) (?$,1'#(B . ?,FS(B) (?$,1'$(B . ?,FT(B) (?$,1'%(B . ?,FU(B) |
125 |
|
(?$,1'&(B . ?,FV(B) (?$,1''(B . ?,FW(B) (?$,1'((B . ?,FX(B) (?$,1')(B . ?,FY(B) (?$,1'*(B . ?,FZ(B) |
126 |
|
(?$,1'+(B . ?,F[(B) (?$,1',(B . ?,F\(B) (?$,1'-(B . ?,F](B) (?$,1'.(B . ?,F^(B) (?$,1'/(B . ?,F_(B) |
127 |
|
(?$,1'0(B . ?,F`(B) (?$,1'1(B . ?,Fa(B) (?$,1'2(B . ?,Fb(B) (?$,1'3(B . ?,Fc(B) (?$,1'4(B . ?,Fd(B) |
128 |
|
(?$,1'5(B . ?,Fe(B) (?$,1'6(B . ?,Ff(B) (?$,1'7(B . ?,Fg(B) (?$,1'8(B . ?,Fh(B) (?$,1'9(B . ?,Fi(B) |
129 |
|
(?$,1':(B . ?,Fj(B) (?$,1';(B . ?,Fk(B) (?$,1'<(B . ?,Fl(B) (?$,1'=(B . ?,Fm(B) (?$,1'>(B . ?,Fn(B) |
130 |
|
(?$,1'?(B . ?,Fo(B) (?$,1'@(B . ?,Fp(B) (?$,1'A(B . ?,Fq(B) (?$,1'B(B . ?,Fr(B) (?$,1'C(B . ?,Fs(B) |
131 |
|
(?$,1'D(B . ?,Ft(B) (?$,1'E(B . ?,Fu(B) (?$,1'F(B . ?,Fv(B) (?$,1'G(B . ?,Fw(B) (?$,1'H(B . ?,Fx(B) |
132 |
|
(?$,1'I(B . ?,Fy(B) (?$,1'J(B . ?,Fz(B) (?$,1'K(B . ?,F{(B) (?$,1'L(B . ?,F|(B) (?$,1'M(B . ?,F}(B) |
133 |
|
(?$,1'N(B . ?,F~(B) |
134 |
|
|
135 |
|
(?$,1(!(B . ?,L!(B) (?$,1("(B . ?,L"(B) (?$,1(#(B . ?,L#(B) (?$,1($(B . ?,L$(B) |
136 |
|
(?$,1(%(B . ?,L%(B) (?$,1(&(B . ?,L&(B) (?$,1('(B . ?,L'(B) (?$,1(((B . ?,L((B) (?$,1()(B . ?,L)(B) |
137 |
|
(?$,1(*(B . ?,L*(B) (?$,1(+(B . ?,L+(B) (?$,1(,(B . ?,L,(B) (?$,1(.(B . ?,L.(B) (?$,1(/(B . ?,L/(B) |
138 |
|
(?$,1(0(B . ?,L0(B) (?$,1(1(B . ?,L1(B) (?$,1(2(B . ?,L2(B) (?$,1(3(B . ?,L3(B) (?$,1(4(B . ?,L4(B) |
139 |
|
(?$,1(5(B . ?,L5(B) (?$,1(6(B . ?,L6(B) (?$,1(7(B . ?,L7(B) (?$,1(8(B . ?,L8(B) (?$,1(9(B . ?,L9(B) |
140 |
|
(?$,1(:(B . ?,L:(B) (?$,1(;(B . ?,L;(B) (?$,1(<(B . ?,L<(B) (?$,1(=(B . ?,L=(B) (?$,1(>(B . ?,L>(B) |
141 |
|
(?$,1(?(B . ?,L?(B) (?$,1(@(B . ?,L@(B) (?$,1(A(B . ?,LA(B) (?$,1(B(B . ?,LB(B) (?$,1(C(B . ?,LC(B) |
142 |
|
(?$,1(D(B . ?,LD(B) (?$,1(E(B . ?,LE(B) (?$,1(F(B . ?,LF(B) (?$,1(G(B . ?,LG(B) (?$,1(H(B . ?,LH(B) |
143 |
|
(?$,1(I(B . ?,LI(B) (?$,1(J(B . ?,LJ(B) (?$,1(K(B . ?,LK(B) (?$,1(L(B . ?,LL(B) (?$,1(M(B . ?,LM(B) |
144 |
|
(?$,1(N(B . ?,LN(B) (?$,1(O(B . ?,LO(B) (?$,1(P(B . ?,LP(B) (?$,1(Q(B . ?,LQ(B) (?$,1(R(B . ?,LR(B) |
145 |
|
(?$,1(S(B . ?,LS(B) (?$,1(T(B . ?,LT(B) (?$,1(U(B . ?,LU(B) (?$,1(V(B . ?,LV(B) (?$,1(W(B . ?,LW(B) |
146 |
|
(?$,1(X(B . ?,LX(B) (?$,1(Y(B . ?,LY(B) (?$,1(Z(B . ?,LZ(B) (?$,1([(B . ?,L[(B) (?$,1(\(B . ?,L\(B) |
147 |
|
(?$,1(](B . ?,L](B) (?$,1(^(B . ?,L^(B) (?$,1(_(B . ?,L_(B) (?$,1(`(B . ?,L`(B) (?$,1(a(B . ?,La(B) |
148 |
|
(?$,1(b(B . ?,Lb(B) (?$,1(c(B . ?,Lc(B) (?$,1(d(B . ?,Ld(B) (?$,1(e(B . ?,Le(B) (?$,1(f(B . ?,Lf(B) |
149 |
|
(?$,1(g(B . ?,Lg(B) (?$,1(h(B . ?,Lh(B) (?$,1(i(B . ?,Li(B) (?$,1(j(B . ?,Lj(B) (?$,1(k(B . ?,Lk(B) |
150 |
|
(?$,1(l(B . ?,Ll(B) (?$,1(m(B . ?,Lm(B) (?$,1(n(B . ?,Ln(B) (?$,1(o(B . ?,Lo(B) (?$,1(q(B . ?,Lq(B) |
151 |
|
(?$,1(r(B . ?,Lr(B) (?$,1(s(B . ?,Ls(B) (?$,1(t(B . ?,Lt(B) (?$,1(u(B . ?,Lu(B) (?$,1(v(B . ?,Lv(B) |
152 |
|
(?$,1(w(B . ?,Lw(B) (?$,1(x(B . ?,Lx(B) (?$,1(y(B . ?,Ly(B) (?$,1(z(B . ?,Lz(B) (?$,1({(B . ?,L{(B) |
153 |
|
(?$,1(|(B . ?,L|(B) (?$,1(~(B . ?,L~(B) (?$,1((B . ?,L(B))) |
154 |
|
|
155 |
|
|
156 |
|
(defcustom utf-fragment-on-decoding nil |
157 |
|
"Whether or not to decode some chars in UTF-8/16 text into iso8859 charsets. |
158 |
|
Setting this means that the relevant Cyrillic and Greek characters are |
159 |
|
decoded into the iso8859 charsets rather than into |
160 |
|
mule-unicode-0100-24ff. The iso8859 charsets take half as much space |
161 |
|
in the buffer, but using them may affect how the buffer can be re-encoded |
162 |
|
and may require a different input method to search for them, for instance. |
163 |
|
See `unify-8859-on-decoding-mode' and `unify-8859-on-encoding-mode' |
164 |
|
for mechanisms to make this largely transparent. |
165 |
|
|
166 |
|
Setting this variable outside customize has no effect." |
167 |
|
:set (lambda (s v) |
168 |
|
(if v |
169 |
|
(progn |
170 |
|
(define-translation-table 'utf-translation-table-for-decode |
171 |
|
utf-fragmentation-table) |
172 |
|
;; Even if unify-8859-on-encoding-mode is off, make |
173 |
|
;; mule-utf-* encode characters in |
174 |
|
;; utf-fragmentation-table. |
175 |
|
(unless (eq (get 'utf-translation-table-for-encode |
176 |
|
'translation-table) |
177 |
|
ucs-mule-to-mule-unicode) |
178 |
|
(define-translation-table 'utf-translation-table-for-encode |
179 |
|
utf-defragmentation-table))) |
180 |
|
(define-translation-table 'utf-translation-table-for-decode) |
181 |
|
;; When unify-8859-on-encoding-mode is off, be sure to make |
182 |
|
;; mule-utf-* disabled for characters in |
183 |
|
;; utf-fragmentation-table. |
184 |
|
(unless (eq (get 'utf-translation-table-for-encode |
185 |
|
'translation-table) |
186 |
|
ucs-mule-to-mule-unicode) |
187 |
|
(define-translation-table 'utf-translation-table-for-encode))) |
188 |
|
(set-default s v)) |
189 |
|
:version "21.4" |
190 |
|
:type 'boolean |
191 |
|
:group 'mule) |
192 |
|
|
193 |
|
(define-minor-mode utf-translate-cjk-mode |
194 |
|
"Whether the UTF based coding systems should decode/encode CJK characters. |
195 |
|
Enabling this loads tables which allow the coding systems mule-utf-8, |
196 |
|
mule-utf-16-le and mule-utf-16-be to encode characters in the charsets |
197 |
|
`korean-ksc5601', `chinese-gb2312', `chinese-big5-1', |
198 |
|
`chinese-big5-2', `japanese-jisx0208' and `japanese-jisx0212', and to |
199 |
|
decode the corresponding unicodes into such characters. |
200 |
|
|
201 |
|
Where the charsets overlap, the one preferred for decoding is chosen |
202 |
|
according to the language environment in effect when this option is |
203 |
|
turned on: ksc5601 for Korean, gb2312 for Chinese-GB, big5 for |
204 |
|
Chinese-Big5 and jisx for other environments. |
205 |
|
|
206 |
|
The tables are large (over 40000 entries), so this option is not the |
207 |
|
default. Also, installing them may be rather slow." |
208 |
|
:init-value nil |
209 |
|
:version "21.4" |
210 |
|
:type 'boolean |
211 |
|
:set-after '(current-language-environment) |
212 |
|
:group 'mule |
213 |
|
:global t |
214 |
|
(if utf-translate-cjk-mode |
215 |
|
;; Fixme: Allow the use of the CJK charsets to be |
216 |
|
;; customized by reordering and possible omission. |
217 |
|
(progn |
218 |
|
;; Redefine them with realistic initial sizes and a |
219 |
|
;; smallish rehash size to avoid wasting significant |
220 |
|
;; space after they're built. |
221 |
|
(setq ucs-mule-cjk-to-unicode |
222 |
|
(make-hash-table :test 'eq :size 43000 :rehash-size 1000) |
223 |
|
ucs-unicode-to-mule-cjk |
224 |
|
(make-hash-table :test 'eq :size 43000 :rehash-size 1000)) |
225 |
|
;; Load the files explicitly, to avoid having to keep |
226 |
|
;; around the large tables they contain (as well as the |
227 |
|
;; ones which get built). |
228 |
|
(cond |
229 |
|
((string= "Korean" current-language-environment) |
230 |
|
(load "subst-jis") |
231 |
|
(load "subst-big5") |
232 |
|
(load "subst-gb2312") |
233 |
|
(load "subst-ksc")) |
234 |
|
((string= "Chinese-BIG5" current-language-environment) |
235 |
|
(load "subst-jis") |
236 |
|
(load "subst-ksc") |
237 |
|
(load "subst-gb2312") |
238 |
|
(load "subst-big5")) |
239 |
|
((string= "Chinese-GB" current-language-environment) |
240 |
|
(load "subst-jis") |
241 |
|
(load "subst-ksc") |
242 |
|
(load "subst-big5") |
243 |
|
(load "subst-gb2312")) |
244 |
|
(t |
245 |
|
(load "subst-ksc") |
246 |
|
(load "subst-gb2312") |
247 |
|
(load "subst-big5") |
248 |
|
(load "subst-jis"))) ; jis covers as much as big5, gb2312 |
249 |
|
(let ((table (make-char-table 'translation-table))) |
250 |
|
(maphash (lambda (k v) |
251 |
|
(aset table k t)) |
252 |
|
ucs-mule-cjk-to-unicode) |
253 |
|
(define-translation-hash-table 'utf-subst-table-for-decode |
254 |
|
ucs-unicode-to-mule-cjk) |
255 |
|
(define-translation-hash-table 'utf-subst-table-for-encode |
256 |
|
ucs-mule-cjk-to-unicode)) |
257 |
|
(define-translation-hash-table 'utf-subst-table-for-decode |
258 |
|
(make-hash-table :test 'eq)) |
259 |
|
(define-translation-hash-table 'utf-subst-table-for-encode |
260 |
|
(make-hash-table :test 'eq))))) |
261 |
|
|
262 |
(define-ccl-program ccl-decode-mule-utf-8 |
(define-ccl-program ccl-decode-mule-utf-8 |
263 |
;; |
;; |
264 |
;; charset | bytes in utf-8 | bytes in emacs |
;; charset | bytes in utf-8 | bytes in emacs |
283 |
((r5 = ,(charset-id 'eight-bit-control)) |
((r5 = ,(charset-id 'eight-bit-control)) |
284 |
(r6 = ,(charset-id 'eight-bit-graphic)) |
(r6 = ,(charset-id 'eight-bit-graphic)) |
285 |
(loop |
(loop |
286 |
|
(r0 = -1) |
287 |
(read r0) |
(read r0) |
288 |
|
|
289 |
;; 1byte encoding, i.e., ascii |
;; 1byte encoding, i.e., ascii |
290 |
(if (r0 < #x80) |
(if (r0 < #x80) |
291 |
(write r0) |
((write r0)) |
292 |
|
(if (r0 < #xc0) ; continuation byte (invalid here) |
293 |
;; 2 byte encoding 00000yyyyyxxxxxx = 110yyyyy 10xxxxxx |
((if (r0 < #xa0) |
294 |
(if (r0 < #xe0) |
(write-multibyte-character r5 r0) |
295 |
((read r1) |
(write-multibyte-character r6 r0))) |
296 |
|
;; 2 byte encoding 00000yyyyyxxxxxx = 110yyyyy 10xxxxxx |
297 |
(if ((r1 & #b11000000) != #b10000000) |
(if (r0 < #xe0) |
298 |
;; Invalid 2-byte sequence |
((r1 = -1) |
299 |
((if (r0 < #xa0) |
(read r1) |
|
(write-multibyte-character r5 r0) |
|
|
(write-multibyte-character r6 r0)) |
|
|
(if (r1 < #x80) |
|
|
(write r1) |
|
|
(if (r1 < #xa0) |
|
|
(write-multibyte-character r5 r1) |
|
|
(write-multibyte-character r6 r1)))) |
|
|
|
|
|
((r0 &= #x1f) |
|
|
(r0 <<= 6) |
|
|
(r1 &= #x3f) |
|
|
(r1 += r0) |
|
|
;; Now r1 holds scalar value |
|
|
|
|
|
;; eight-bit-control |
|
|
(if (r1 < 160) |
|
|
((write-multibyte-character r5 r1)) |
|
|
|
|
|
;; latin-iso8859-1 |
|
|
(if (r1 < 256) |
|
|
((r0 = ,(charset-id 'latin-iso8859-1)) |
|
|
(r1 -= 128) |
|
|
(write-multibyte-character r0 r1)) |
|
|
|
|
|
;; mule-unicode-0100-24ff (< 0800) |
|
|
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
|
|
(r1 -= #x0100) |
|
|
(r2 = (((r1 / 96) + 32) << 7)) |
|
|
(r1 %= 96) |
|
|
(r1 += (r2 + 32)) |
|
|
(write-multibyte-character r0 r1))))))) |
|
|
|
|
|
;; 3byte encoding |
|
|
;; zzzzyyyyyyxxxxxx = 1110zzzz 10yyyyyy 10xxxxxx |
|
|
(if (r0 < #xf0) |
|
|
((read r1 r2) |
|
|
|
|
|
;; This is set to 1 if the encoding is invalid. |
|
|
(r4 = 0) |
|
|
|
|
|
(r3 = (r1 & #b11000000)) |
|
|
(r3 |= ((r2 >> 2) & #b00110000)) |
|
|
(if (r3 != #b10100000) |
|
|
(r4 = 1) |
|
|
((r3 = ((r0 & #x0f) << 12)) |
|
|
(r3 += ((r1 & #x3f) << 6)) |
|
|
(r3 += (r2 & #x3f)) |
|
|
(if (r3 < #x0800) |
|
|
(r4 = 1)))) |
|
300 |
|
|
301 |
(if (r4 != 0) |
(if ((r1 & #b11000000) != #b10000000) |
302 |
;; Invalid 3-byte sequence |
;; Invalid 2-byte sequence |
303 |
((if (r0 < #xa0) |
((if (r0 < #xa0) |
304 |
(write-multibyte-character r5 r0) |
(write-multibyte-character r5 r0) |
305 |
(write-multibyte-character r6 r0)) |
(write-multibyte-character r6 r0)) |
307 |
(write r1) |
(write r1) |
308 |
(if (r1 < #xa0) |
(if (r1 < #xa0) |
309 |
(write-multibyte-character r5 r1) |
(write-multibyte-character r5 r1) |
310 |
(write-multibyte-character r6 r1))) |
(write-multibyte-character r6 r1)))) |
311 |
(if (r2 < #x80) |
|
312 |
(write r2) |
((r3 = r0) ; save in case of overlong sequence |
313 |
(if (r2 < #xa0) |
(r2 = r1) |
314 |
(write-multibyte-character r5 r2) |
(r0 &= #x1f) |
315 |
(write-multibyte-character r6 r2)))) |
(r0 <<= 6) |
316 |
|
(r1 &= #x3f) |
317 |
;; mule-unicode-0100-24ff (>= 0800) |
(r1 += r0) |
318 |
((if (r3 < #x2500) |
;; Now r1 holds scalar value |
319 |
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
|
320 |
(r3 -= #x0100) |
(if (r1 < 128) ; `overlong sequence' |
321 |
(r3 //= 96) |
((if (r3 < #xa0) |
322 |
(r1 = (r7 + 32)) |
(write-multibyte-character r5 r3) |
323 |
(r1 += ((r3 + 32) << 7)) |
(write-multibyte-character r6 r3)) |
324 |
(write-multibyte-character r0 r1)) |
(if (r2 < #x80) |
325 |
|
(write r2) |
326 |
;; mule-unicode-2500-33ff |
(if (r2 < #xa0) |
327 |
(if (r3 < #x3400) |
(write-multibyte-character r5 r2) |
328 |
((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
(write-multibyte-character r6 r2)))) |
329 |
(r3 -= #x2500) |
|
330 |
|
;; eight-bit-control |
331 |
|
(if (r1 < 160) |
332 |
|
((write-multibyte-character r5 r1)) |
333 |
|
|
334 |
|
;; latin-iso8859-1 |
335 |
|
(if (r1 < 256) |
336 |
|
((r0 = ,(charset-id 'latin-iso8859-1)) |
337 |
|
(r1 -= 128) |
338 |
|
(write-multibyte-character r0 r1)) |
339 |
|
|
340 |
|
;; mule-unicode-0100-24ff (< 0800) |
341 |
|
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
342 |
|
(r1 -= #x0100) |
343 |
|
(r2 = (((r1 / 96) + 32) << 7)) |
344 |
|
(r1 %= 96) |
345 |
|
(r1 += (r2 + 32)) |
346 |
|
(translate-character |
347 |
|
utf-translation-table-for-decode r0 r1) |
348 |
|
(write-multibyte-character r0 r1)))))))) |
349 |
|
|
350 |
|
;; 3byte encoding |
351 |
|
;; zzzzyyyyyyxxxxxx = 1110zzzz 10yyyyyy 10xxxxxx |
352 |
|
(if (r0 < #xf0) |
353 |
|
((r1 = -1) |
354 |
|
(r2 = -1) |
355 |
|
(read r1 r2) |
356 |
|
|
357 |
|
;; This is set to 1 if the encoding is invalid. |
358 |
|
(r4 = 0) |
359 |
|
|
360 |
|
(r3 = (r1 & #b11000000)) |
361 |
|
(r3 |= ((r2 >> 2) & #b00110000)) |
362 |
|
(if (r3 != #b10100000) |
363 |
|
(r4 = 1) |
364 |
|
((r3 = ((r0 & #x0f) << 12)) |
365 |
|
(r3 += ((r1 & #x3f) << 6)) |
366 |
|
(r3 += (r2 & #x3f)) |
367 |
|
(if (r3 < #x0800) |
368 |
|
(r4 = 1)))) |
369 |
|
|
370 |
|
(if (r4 != 0) |
371 |
|
;; Invalid 3-byte sequence |
372 |
|
((if (r0 < #xa0) |
373 |
|
(write-multibyte-character r5 r0) |
374 |
|
(write-multibyte-character r6 r0)) |
375 |
|
(if (r1 < #x80) |
376 |
|
(write r1) |
377 |
|
(if (r1 < #xa0) |
378 |
|
(write-multibyte-character r5 r1) |
379 |
|
(write-multibyte-character r6 r1))) |
380 |
|
(if (r2 < #x80) |
381 |
|
(write r2) |
382 |
|
(if (r2 < #xa0) |
383 |
|
(write-multibyte-character r5 r2) |
384 |
|
(write-multibyte-character r6 r2)))) |
385 |
|
|
386 |
|
;; mule-unicode-0100-24ff (>= 0800) |
387 |
|
((if (r3 < #x2500) |
388 |
|
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
389 |
|
(r3 -= #x0100) |
390 |
(r3 //= 96) |
(r3 //= 96) |
391 |
(r1 = (r7 + 32)) |
(r1 = (r7 + 32)) |
392 |
(r1 += ((r3 + 32) << 7)) |
(r1 += ((r3 + 32) << 7)) |
393 |
|
(translate-character |
394 |
|
utf-translation-table-for-decode r0 r1) |
395 |
(write-multibyte-character r0 r1)) |
(write-multibyte-character r0 r1)) |
|
|
|
|
;; U+3400 .. U+DFFF |
|
|
;; keep those bytes as eight-bit-{control|graphic} |
|
|
(if (r3 < #xe000) |
|
|
( ;; #xe0 <= r0 < #xf0, so r0 is eight-bit-graphic |
|
|
(r3 = r6) |
|
|
(write-multibyte-character r3 r0) |
|
|
(if (r1 < #xa0) |
|
|
(r3 = r5)) |
|
|
(write-multibyte-character r3 r1) |
|
|
(if (r2 < #xa0) |
|
|
(r3 = r5) |
|
|
(r3 = r6)) |
|
|
(write-multibyte-character r3 r2)) |
|
|
|
|
|
;; mule-unicode-e000-ffff |
|
|
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
|
|
(r3 -= #xe000) |
|
|
(r3 //= 96) |
|
|
(r1 = (r7 + 32)) |
|
|
(r1 += ((r3 + 32) << 7)) |
|
|
(write-multibyte-character r0 r1)))))))) |
|
396 |
|
|
397 |
;; 4byte encoding |
;; mule-unicode-2500-33ff |
398 |
;; keep those bytes as eight-bit-{control|graphic} |
(if (r3 < #x3400) |
399 |
((read r1 r2 r3) |
((r4 = r3) ; don't zap r3 |
400 |
;; r0 > #xf0, thus eight-bit-graphic |
(lookup-integer utf-subst-table-for-decode r4 r5) |
401 |
(write-multibyte-character r6 r0) |
(if r7 |
402 |
(if (r1 < #xa0) |
;; got a translation |
403 |
(write-multibyte-character r5 r1) |
((write-multibyte-character r4 r5) |
404 |
(write-multibyte-character r6 r1)) |
;; Zapped through register starvation. |
405 |
(if (r2 < #xa0) |
(r5 = ,(charset-id 'eight-bit-control))) |
406 |
(write-multibyte-character r5 r2) |
((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
407 |
(write-multibyte-character r6 r2)) |
(r3 -= #x2500) |
408 |
(if (r3 < #xa0) |
(r3 //= 96) |
409 |
(write-multibyte-character r5 r3) |
(r1 = (r7 + 32)) |
410 |
(write-multibyte-character r6 r3)))))) |
(r1 += ((r3 + 32) << 7)) |
411 |
|
(write-multibyte-character r0 r1)))) |
412 |
|
|
413 |
|
;; U+3400 .. U+D7FF |
414 |
|
;; Try to convert to CJK chars, else keep |
415 |
|
;; them as eight-bit-{control|graphic}. |
416 |
|
(if (r3 < #xd800) |
417 |
|
((r4 = r3) ; don't zap r3 |
418 |
|
(lookup-integer utf-subst-table-for-decode r4 r5) |
419 |
|
(if r7 |
420 |
|
;; got a translation |
421 |
|
((write-multibyte-character r4 r5) |
422 |
|
;; Zapped through register starvation. |
423 |
|
(r5 = ,(charset-id 'eight-bit-control))) |
424 |
|
;; #xe0 <= r0 < #xf0, so r0 is eight-bit-graphic |
425 |
|
((r3 = r6) |
426 |
|
(write-multibyte-character r3 r0) |
427 |
|
(if (r1 < #xa0) |
428 |
|
(r3 = r5)) |
429 |
|
(write-multibyte-character r3 r1) |
430 |
|
(if (r2 < #xa0) |
431 |
|
(r3 = r5) |
432 |
|
(r3 = r6)) |
433 |
|
(write-multibyte-character r3 r2)))) |
434 |
|
|
435 |
|
;; Surrogates, U+D800 .. U+DFFF |
436 |
|
(if (r3 < #xe000) |
437 |
|
((r3 = r6) |
438 |
|
(write-multibyte-character r3 r0) ; eight-bit-graphic |
439 |
|
(if (r1 < #xa0) |
440 |
|
(r3 = r5)) |
441 |
|
(write-multibyte-character r3 r1) |
442 |
|
(if (r2 < #xa0) |
443 |
|
(r3 = r5) |
444 |
|
(r3 = r6)) |
445 |
|
(write-multibyte-character r3 r2)) |
446 |
|
|
447 |
|
;; mule-unicode-e000-ffff |
448 |
|
;; Fixme: fffe and ffff are invalid. |
449 |
|
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
450 |
|
(r3 -= #xe000) |
451 |
|
(r3 //= 96) |
452 |
|
(r1 = (r7 + 32)) |
453 |
|
(r1 += ((r3 + 32) << 7)) |
454 |
|
(write-multibyte-character r0 r1))))))))) |
455 |
|
|
456 |
|
(if (r0 < #xfe) |
457 |
|
;; 4byte encoding |
458 |
|
;; keep those bytes as eight-bit-{control|graphic} |
459 |
|
;; Fixme: allow lookup in utf-subst-table-for-decode. |
460 |
|
((r1 = -1) |
461 |
|
(r2 = -1) |
462 |
|
(r3 = -1) |
463 |
|
(read r1 r2 r3) |
464 |
|
;; r0 > #xf0, thus eight-bit-graphic |
465 |
|
(write-multibyte-character r6 r0) |
466 |
|
(if (r1 < #xa0) |
467 |
|
(if (r1 < #x80) ; invalid byte |
468 |
|
(write r1) |
469 |
|
(write-multibyte-character r5 r1)) |
470 |
|
(write-multibyte-character r6 r1)) |
471 |
|
(if (r2 < #xa0) |
472 |
|
(if (r2 < #x80) ; invalid byte |
473 |
|
(write r2) |
474 |
|
(write-multibyte-character r5 r2)) |
475 |
|
(write-multibyte-character r6 r2)) |
476 |
|
(if (r3 < #xa0) |
477 |
|
(if (r3 < #x80) ; invalid byte |
478 |
|
(write r3) |
479 |
|
(write-multibyte-character r5 r3)) |
480 |
|
(write-multibyte-character r6 r3)) |
481 |
|
(if (r0 >= #xf8) ; 5- or 6-byte encoding |
482 |
|
((r0 = -1) |
483 |
|
(read r0) |
484 |
|
(if (r0 < #xa0) |
485 |
|
(if (r0 < #x80) ; invalid byte |
486 |
|
(write r0) |
487 |
|
(write-multibyte-character r5 r0)) |
488 |
|
(write-multibyte-character r6 r0)) |
489 |
|
(if (r0 >= #xfc) ; 6-byte |
490 |
|
((r0 = -1) |
491 |
|
(read r0) |
492 |
|
(if (r0 < #xa0) |
493 |
|
(if (r0 < #x80) ; invalid byte |
494 |
|
(write r0) |
495 |
|
(write-multibyte-character r5 r0)) |
496 |
|
(write-multibyte-character r6 r0))))))) |
497 |
|
;; else invalid byte >= #xfe |
498 |
|
(write-multibyte-character r6 r0)))))) |
499 |
|
(repeat))) |
500 |
|
|
501 |
(repeat)))) |
;; At EOF... |
502 |
|
(if (r0 >= 0) |
503 |
|
((if (r0 < #x80) |
504 |
|
(write r0) |
505 |
|
(if (r0 < #xa0) |
506 |
|
(write-multibyte-character r5 r0) |
507 |
|
((write-multibyte-character r6 r0)))) |
508 |
|
(if (r1 >= 0) |
509 |
|
((if (r1 < #x80) |
510 |
|
(write r1) |
511 |
|
(if (r1 < #xa0) |
512 |
|
(write-multibyte-character r5 r1) |
513 |
|
((write-multibyte-character r6 r1)))) |
514 |
|
(if (r2 >= 0) |
515 |
|
((if (r2 < #x80) |
516 |
|
(write r2) |
517 |
|
(if (r2 < #xa0) |
518 |
|
(write-multibyte-character r5 r2) |
519 |
|
((write-multibyte-character r6 r2)))) |
520 |
|
(if (r3 >= 0) |
521 |
|
(if (r3 < #x80) |
522 |
|
(write r3) |
523 |
|
(if (r3 < #xa0) |
524 |
|
(write-multibyte-character r5 r3) |
525 |
|
((write-multibyte-character r6 r3)))))))))))) |
526 |
|
|
527 |
"CCL program to decode UTF-8. |
"CCL program to decode UTF-8. |
528 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
529 |
mule-unicode-*. Encodings of un-representable Unicode characters are |
mule-unicode-*, but see also `utf-fragmentation-table' and |
530 |
decoded asis into eight-bit-control and eight-bit-graphic |
`ucs-mule-cjk-to-unicode'. |
531 |
characters.") |
Encodings of un-representable Unicode characters are decoded asis into |
532 |
|
eight-bit-control and eight-bit-graphic characters.") |
533 |
|
|
534 |
(define-ccl-program ccl-encode-mule-utf-8 |
(define-ccl-program ccl-encode-mule-utf-8 |
535 |
`(1 |
`(1 |
538 |
(if (r5 < 0) |
(if (r5 < 0) |
539 |
((r1 = -1) |
((r1 = -1) |
540 |
(read-multibyte-character r0 r1) |
(read-multibyte-character r0 r1) |
541 |
(translate-character ucs-mule-to-mule-unicode r0 r1)) |
(translate-character utf-translation-table-for-encode r0 r1)) |
542 |
(;; We have already done read-multibyte-character. |
(;; We have already done read-multibyte-character. |
543 |
(r0 = r5) |
(r0 = r5) |
544 |
(r1 = r6) |
(r1 = r6) |
592 |
(if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
(if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
593 |
((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96)) |
((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96)) |
594 |
(r1 &= #x7f) |
(r1 &= #x7f) |
595 |
(r1 += (r0 + 57312)) ; 57312 == -160 + #xe000 |
(r1 += (r0 + 57312)) ; 57312 == -32 + #xe000 |
596 |
(r0 = (((r1 & #xf000) >> 12) | #xe0)) |
(r0 = (((r1 & #xf000) >> 12) | #xe0)) |
597 |
(r2 = ((r1 & #x3f) | #x80)) |
(r2 = ((r1 & #x3f) | #x80)) |
598 |
(r1 &= #x0fc0) |
(r1 &= #x0fc0) |
633 |
((write #xc2) |
((write #xc2) |
634 |
(write r1))))))) |
(write r1))))))) |
635 |
|
|
636 |
;; Unsupported character. |
((lookup-character utf-subst-table-for-encode r0 r1) |
637 |
;; Output U+FFFD, which is `ef bf bd' in UTF-8. |
(if r7 ; lookup succeeded |
638 |
((write #xef) |
((r1 = (((r0 & #xf000) >> 12) | #xe0)) |
639 |
(write #xbf) |
(r2 = ((r0 & #x3f) | #x80)) |
640 |
(write #xbd))))))))) |
(r0 &= #x0fc0) |
641 |
|
(r0 >>= 6) |
642 |
|
(r0 |= #x80) |
643 |
|
(write r1 r0 r2)) |
644 |
|
;; Unsupported character. |
645 |
|
;; Output U+FFFD, which is `ef bf bd' in UTF-8. |
646 |
|
((write #xef) |
647 |
|
(write #xbf) |
648 |
|
(write #xbd))))))))))) |
649 |
(repeat))) |
(repeat))) |
650 |
(if (r1 >= #xa0) |
(if (r1 >= #xa0) |
651 |
(write r1) |
(write r1) |
653 |
((write #xc2) |
((write #xc2) |
654 |
(write r1))))) |
(write r1))))) |
655 |
|
|
656 |
"CCL program to encode into UTF-8. |
"CCL program to encode into UTF-8.") |
657 |
Only characters from the charsets ascii, eight-bit-control, |
|
658 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are recognized. |
|
659 |
Others are encoded as U+FFFD.") |
(define-ccl-program ccl-untranslated-to-ucs |
660 |
|
`(0 |
661 |
;; Dummy definition so that the CCL can be checked correctly; the |
(if (r0 < #xf0) ; 3-byte encoding, as above |
662 |
;; actual data are loaded on demand. |
((r4 = 0) |
663 |
(unless (boundp 'ucs-mule-8859-to-mule-unicode) ; don't zap it |
(r3 = (r1 & #b11000000)) |
664 |
(define-translation-table 'ucs-mule-8859-to-mule-unicode)) |
(r3 |= ((r2 >> 2) & #b00110000)) |
665 |
|
(if (r3 != #b10100000) |
666 |
|
(r4 = 1) |
667 |
|
((r3 = ((r0 & #x0f) << 12)) |
668 |
|
(r3 += ((r1 & #x3f) << 6)) |
669 |
|
(r3 += (r2 & #x3f)) |
670 |
|
(if (r3 < #x0800) |
671 |
|
(r4 = 1)))) |
672 |
|
(if (r4 != 0) |
673 |
|
(r0 = 0) |
674 |
|
(r0 = r3))) |
675 |
|
(if (r0 < #xf8) ; 4-byte (Mule-UCS recipe) |
676 |
|
((r4 = (r1 >> 6)) |
677 |
|
(if (r4 != #b10) |
678 |
|
(r0 = 0) |
679 |
|
((r4 = (r2 >> 6)) |
680 |
|
(if (r4 != #b10) |
681 |
|
(r0 = 0) |
682 |
|
((r4 = (r3 >> 6)) |
683 |
|
(if (r4 != #b10) |
684 |
|
(r0 = 0) |
685 |
|
((r1 = ((r1 & #x3F) << 12)) |
686 |
|
(r2 = ((r2 & #x3F) << 6)) |
687 |
|
(r3 &= #x3F) |
688 |
|
(r0 = (((((r0 & #x07) << 18) | r1) | r2) | r3))))))))) |
689 |
|
(r0 = 0)))) |
690 |
|
"Decode 3- or 4-byte sequences in r0, r1, r2 [,r3] to unicodes in r0. |
691 |
|
r0 == 0 for invalid sequence.") |
692 |
|
|
693 |
|
(defvar utf-8-ccl-regs (make-vector 8 0)) |
694 |
|
|
695 |
(defsubst utf-8-untranslated-to-ucs () |
(defsubst utf-8-untranslated-to-ucs () |
696 |
(let ((b1 (char-after)) |
"Return the UCS code for an untranslated sequence of raw bytes t point. |
697 |
(b2 (char-after (1+ (point)))) |
Only for 3- or 4-byte sequences." |
698 |
(b3 (char-after (+ 2 (point)))) |
(aset utf-8-ccl-regs 0 (or (char-after) 0)) |
699 |
(b4 (char-after (+ 4 (point))))) |
(aset utf-8-ccl-regs 1 (or (char-after (1+ (point))) 0)) |
700 |
(if (and b1 b2 b3) |
(aset utf-8-ccl-regs 2 (or (char-after (+ 2 (point))) 0)) |
701 |
(cond ((< b1 ?\xf0) |
(aset utf-8-ccl-regs 3 (or (char-after (+ 3 (point))) 0)) |
702 |
(setq b2 (lsh (logand b2 ?\x3f) 6)) |
(ccl-execute 'ccl-untranslated-to-ucs utf-8-ccl-regs) |
703 |
(setq b3 (logand b3 ?\x3f)) |
(aref utf-8-ccl-regs 0)) |
|
(logior b3 (logior b2 (lsh (logand b1 ?\x0f) 12)))) |
|
|
(b4 |
|
|
(setq b2 (lsh (logand b2 ?\x3f) 12)) |
|
|
(setq b3 (lsh (logand b3 ?\x3f) 6)) |
|
|
(setq b4 (logand b4 ?\x3f)) |
|
|
(logior b4 (logior b3 (logior b2 (lsh (logand b1 ?\x07) |
|
|
18))))))))) |
|
704 |
|
|
705 |
(defun utf-8-help-echo (window object position) |
(defun utf-8-help-echo (window object position) |
706 |
(format "Untranslated Unicode U+%04X" |
(format "Untranslated Unicode U+%04X" |
707 |
(get-char-property position 'untranslated-utf-8 object))) |
(get-char-property position 'untranslated-utf-8 object))) |
708 |
|
|
|
(defvar utf-8-subst-table nil |
|
|
"If non-nil, a hash table mapping `untranslatable utf-8' to Emacs characters.") |
|
|
|
|
709 |
;; We compose the untranslatable sequences into a single character. |
;; We compose the untranslatable sequences into a single character. |
710 |
;; This is infelicitous for editing, because there's currently no |
;; This is infelicitous for editing, because there's currently no |
711 |
;; mechanism for treating compositions as atomic, but is OK for |
;; mechanism for treating compositions as atomic, but is OK for |
712 |
;; display. We try to compose an appropriate character from a hash |
;; display. They are composed to U+FFFD with help-echo which |
713 |
;; table of CJK characters to display correctly. Otherwise we use |
;; indicates the unicodes they represent. This function GCs too much. |
|
;; U+FFFD. What we really should have is hash table lookup from CCL |
|
|
;; so that we could do this properly. This function GCs too much. |
|
714 |
(defsubst utf-8-compose () |
(defsubst utf-8-compose () |
715 |
"Put a suitable composition on an untranslatable sequence. |
"Put a suitable composition on an untranslatable sequence. |
716 |
Return the sequence's length." |
Return the sequence's length." |
717 |
(let* ((u (utf-8-untranslated-to-ucs)) |
(let* ((u (utf-8-untranslated-to-ucs)) |
718 |
(l (and u (if (>= u ?\x10000) |
(l (unless (zerop u) |
719 |
|
(if (>= u #x10000) |
720 |
4 |
4 |
721 |
3))) |
3)))) |
722 |
(subst (and utf-8-subst-table (gethash u utf-8-subst-table)))) |
(when l |
|
(when u |
|
723 |
(put-text-property (point) (min (point-max) (+ l (point))) |
(put-text-property (point) (min (point-max) (+ l (point))) |
724 |
'untranslated-utf-8 u) |
'untranslated-utf-8 u) |
725 |
(unless subst |
(put-text-property (point) (min (point-max) (+ l (point))) |
726 |
(put-text-property (point) (min (point-max) (+ l (point))) |
'help-echo 'utf-8-help-echo) |
727 |
'help-echo 'utf-8-help-echo) |
(compose-region (point) (+ l (point)) ?$,3u=(B) |
|
(setq subst ?$,3u=(B)) |
|
|
(compose-region (point) (+ l (point)) subst) |
|
728 |
l))) |
l))) |
729 |
|
|
730 |
(defcustom utf-8-compose-scripts nil |
(defcustom utf-8-compose-scripts nil |
731 |
"*Non-nil means compose various scipts on decoding utf-8 text." |
"*Non-nil means compose various scripts on decoding utf-8 text." |
732 |
:group 'mule |
:group 'mule |
733 |
:type 'boolean) ; omitted in Emacs 21.1 |
:version "21.4" |
734 |
|
:type 'boolean) |
735 |
|
|
736 |
(defun utf-8-post-read-conversion (length) |
(defun utf-8-post-read-conversion (length) |
737 |
"Compose untranslated utf-8 sequences into single characters. |
"Compose untranslated utf-8 sequences into single characters. |
740 |
;; Can't do eval-when-compile to insert a multibyte constant |
;; Can't do eval-when-compile to insert a multibyte constant |
741 |
;; version of the string in the loop, since it's always loaded as |
;; version of the string in the loop, since it's always loaded as |
742 |
;; unibyte from a byte-compiled file. |
;; unibyte from a byte-compiled file. |
743 |
(let ((range (string-as-multibyte "^\341-\377"))) |
(let ((range (string-as-multibyte "^\xe1-\xf7"))) |
744 |
(while (and (skip-chars-forward |
(while (and (skip-chars-forward range) |
|
range) |
|
745 |
(not (eobp))) |
(not (eobp))) |
746 |
(forward-char (utf-8-compose))))) |
(forward-char (utf-8-compose))))) |
747 |
;; Fixme: Takahashi-san implies it may not work this easily -- needs |
;; Fixme: Takahashi-san implies it may not work this easily. I |
748 |
;; checking with him. |
;; asked why but didn't get a reply. -- fx |
749 |
(when (and utf-8-compose-scripts (> length 1)) |
(when (and utf-8-compose-scripts (> length 1)) |
750 |
;; These currently have definitions which cover the relevant |
;; These currently have definitions which cover the relevant |
751 |
;; Unicodes. We could avoid loading thai-util &c by checking |
;; unicodes. We could avoid loading thai-util &c by checking |
752 |
;; whether the region contains any characters with the appropriate |
;; whether the region contains any characters with the appropriate |
753 |
;; categories. There aren't yet Unicode-based rules for Tibetan. |
;; categories. There aren't yet Unicode-based rules for Tibetan. |
754 |
(save-excursion (setq length (diacritic-post-read-conversion length))) |
(save-excursion (setq length (diacritic-post-read-conversion length))) |
755 |
(save-excursion (setq length (thai-post-read-conversion length))) |
(save-excursion (setq length (thai-post-read-conversion length))) |
756 |
(save-excursion (setq length (lao-post-read-conversion length))) |
(save-excursion (setq length (lao-post-read-conversion length))) |
757 |
(save-excursion (setq length (devanagari-post-read-conversion length)))) |
(save-excursion |
758 |
|
(setq length (in-is13194-devanagari-post-read-conversion length)))) |
759 |
length) |
length) |
760 |
|
|
761 |
(defun utf-8-pre-write-conversion (beg end) |
;; ucs-tables is preloaded |
762 |
"Semi-dummy pre-write function effectively to autoload ucs-tables." |
;; (defun utf-8-pre-write-conversion (beg end) |
763 |
;; Ensure translation table is loaded. |
;; "Semi-dummy pre-write function effectively to autoload ucs-tables." |
764 |
(require 'ucs-tables) |
;; ;; Ensure translation-table is loaded. |
765 |
;; Don't do this again. |
;; (require 'ucs-tables) |
766 |
(coding-system-put 'mule-utf-8 'pre-write-conversion nil) |
;; ;; Don't do this again. |
767 |
nil) |
;; (coding-system-put 'mule-utf-8 'pre-write-conversion nil) |
768 |
|
;; nil) |
769 |
|
|
770 |
(make-coding-system |
(make-coding-system |
771 |
'mule-utf-8 4 ?u |
'mule-utf-8 4 ?u |
772 |
"UTF-8 encoding for Emacs-supported Unicode characters. |
"UTF-8 encoding for Emacs-supported Unicode characters. |
773 |
The supported Emacs character sets are the following, plus others |
It supports Unicode characters of these ranges: |
774 |
which may be included in the translation table |
U+0000..U+33FF, U+E000..U+FFFF. |
775 |
`ucs-mule-to-mule-unicode': |
They correspond to these Emacs character sets: |
776 |
ascii |
ascii, latin-iso8859-1, mule-unicode-0100-24ff, |
777 |
eight-bit-control |
mule-unicode-2500-33ff, mule-unicode-e000-ffff |
778 |
eight-bit-graphic |
|
779 |
latin-iso8859-1 |
On decoding (e.g. reading a file), Unicode characters not in the above |
780 |
latin-iso8859-2 |
ranges are decoded into sequences of eight-bit-control and |
781 |
latin-iso8859-3 |
eight-bit-graphic characters to preserve their byte sequences. The |
782 |
latin-iso8859-4 |
byte sequence is preserved on i/o for valid utf-8, but not necessarily |
783 |
cyrillic-iso8859-5 |
for invalid utf-8. |
784 |
greek-iso8859-7 |
|
785 |
hebrew-iso8859-8 |
On encoding (e.g. writing a file), Emacs characters not belonging to |
786 |
latin-iso8859-9 |
any of the character sets listed above are encoded into the UTF-8 byte |
787 |
latin-iso8859-14 |
sequence representing U+FFFD (REPLACEMENT CHARACTER)." |
|
latin-iso8859-15 |
|
|
mule-unicode-0100-24ff |
|
|
mule-unicode-2500-33ff |
|
|
mule-unicode-e000-ffff |
|
|
|
|
|
Unicode characters out of the ranges U+0000-U+33FF and U+E200-U+FFFF |
|
|
are decoded into sequences of eight-bit-control and eight-bit-graphic |
|
|
characters to preserve their byte sequences and composed to display as |
|
|
a single character. Emacs characters that can't be encoded to these |
|
|
ranges are encoded as U+FFFD." |
|
788 |
|
|
789 |
'(ccl-decode-mule-utf-8 . ccl-encode-mule-utf-8) |
'(ccl-decode-mule-utf-8 . ccl-encode-mule-utf-8) |
790 |
'((safe-charsets |
'((safe-charsets |
792 |
eight-bit-control |
eight-bit-control |
793 |
eight-bit-graphic |
eight-bit-graphic |
794 |
latin-iso8859-1 |
latin-iso8859-1 |
|
latin-iso8859-15 |
|
|
latin-iso8859-14 |
|
|
latin-iso8859-9 |
|
|
hebrew-iso8859-8 |
|
|
greek-iso8859-7 |
|
|
cyrillic-iso8859-5 |
|
|
latin-iso8859-4 |
|
|
latin-iso8859-3 |
|
|
latin-iso8859-2 |
|
|
vietnamese-viscii-lower |
|
|
vietnamese-viscii-upper |
|
|
thai-tis620 |
|
|
ipa |
|
|
ethiopic |
|
|
indian-is13194 |
|
|
katakana-jisx0201 |
|
|
chinese-sisheng |
|
|
lao |
|
795 |
mule-unicode-0100-24ff |
mule-unicode-0100-24ff |
796 |
mule-unicode-2500-33ff |
mule-unicode-2500-33ff |
797 |
mule-unicode-e000-ffff) |
mule-unicode-e000-ffff) |
798 |
(mime-charset . utf-8) |
(mime-charset . utf-8) |
799 |
(coding-category . coding-category-utf-8) |
(coding-category . coding-category-utf-8) |
800 |
(valid-codes (0 . 255)) |
(valid-codes (0 . 255)) |
801 |
(pre-write-conversion . utf-8-pre-write-conversion) |
;; (pre-write-conversion . utf-8-pre-write-conversion) |
802 |
(post-read-conversion . utf-8-post-read-conversion))) |
(post-read-conversion . utf-8-post-read-conversion) |
803 |
|
(dependency unify-8859-on-encoding-mode |
804 |
|
unify-8859-on-decoding-mode |
805 |
|
utf-fragment-on-decoding |
806 |
|
utf-translate-cjk))) |
807 |
|
|
808 |
(define-coding-system-alias 'utf-8 'mule-utf-8) |
(define-coding-system-alias 'utf-8 'mule-utf-8) |
809 |
|
|