68 |
;; Needed in macro expansion, so can't be let-bound. Zapped after use. |
;; Needed in macro expansion, so can't be let-bound. Zapped after use. |
69 |
(eval-and-compile |
(eval-and-compile |
70 |
(defconst utf-16-decode-ucs |
(defconst utf-16-decode-ucs |
71 |
;; We have the unicode in r1. Output is character codes in r0, r1, |
;; We have the unicode in r1. Output is charset ID in r0, code point |
72 |
;; and r2 if appropriate. |
;; in r1. |
73 |
`((lookup-integer utf-8-subst-table r0 r3) |
`((lookup-integer utf-subst-table-for-decode r1 r3) |
74 |
(if r7 (r1 = r3)) ; got a translation |
(if r7 ; got a translation |
75 |
(if (r1 < 128) |
((r0 = r1) (r1 = r3)) |
76 |
(r0 = ,(charset-id 'ascii)) |
(if (r1 < 128) |
77 |
(if (r1 < 160) |
(r0 = ,(charset-id 'ascii)) |
78 |
(r0 = ,(charset-id 'eight-bit-control)) |
(if (r1 < 160) |
79 |
(if (r1 < 256) |
(r0 = ,(charset-id 'eight-bit-control)) |
80 |
((r0 = ,(charset-id 'latin-iso8859-1)) |
(if (r1 < 256) |
81 |
(r1 -= 128)) |
((r0 = ,(charset-id 'latin-iso8859-1)) |
82 |
(if (r1 < #x2500) |
(r1 -= 128)) |
83 |
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
(if (r1 < #x2500) |
84 |
(r1 -= #x100) |
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
85 |
(r2 = (((r1 / 96) + 32) << 7)) |
(r1 -= #x100) |
86 |
(r1 %= 96) |
(r2 = (((r1 / 96) + 32) << 7)) |
87 |
(r1 += (r2 + 32))) |
(r1 %= 96) |
88 |
(if (r1 < #x3400) |
(r1 += (r2 + 32))) |
89 |
((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
(if (r1 < #x3400) |
90 |
(r1 -= #x2500) |
((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
91 |
(r2 = (((r1 / 96) + 32) << 7)) |
(r1 -= #x2500) |
92 |
(r1 %= 96) |
(r2 = (((r1 / 96) + 32) << 7)) |
93 |
(r1 += (r2 + 32))) |
(r1 %= 96) |
94 |
(if (r1 < #xd800) ; 2 untranslated bytes |
(r1 += (r2 + 32))) |
95 |
;; ;; Assume this is rare, so don't worry about the |
(if (r1 < #xd800) ; 2 untranslated bytes |
96 |
;; ;; overhead of the call. |
;; ;; Assume this is rare, so don't worry about the |
97 |
;; (call mule-utf-16-untrans) |
;; ;; overhead of the call. |
98 |
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
;; (call mule-utf-16-untrans) |
99 |
(r1 = 15037)) ; U+fffd |
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
100 |
(if (r1 < #xe000) ; surrogate |
(r1 = 15037)) ; U+fffd |
101 |
;; ((call mule-utf-16-untrans) |
(if (r1 < #xe000) ; surrogate |
102 |
;; (write-multibyte-character r0 r1) |
;; ((call mule-utf-16-untrans) |
103 |
;; (read r3 r4) |
;; (write-multibyte-character r0 r1) |
104 |
;; (call mule-utf-16-untrans)) |
;; (read r3 r4) |
105 |
((read r3 r4) |
;; (call mule-utf-16-untrans)) |
106 |
(r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
((read r3 r4) |
107 |
(r1 = 15037)) |
(r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
108 |
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
(r1 = 15037)) |
109 |
(r1 -= #xe000) |
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
110 |
(r2 = (((r1 / 96) + 32) << 7)) |
(r1 -= #xe000) |
111 |
(r1 %= 96) |
(r2 = (((r1 / 96) + 32) << 7)) |
112 |
(r1 += (r2 + 32))))))))))))) |
(r1 %= 96) |
113 |
|
(r1 += (r2 + 32)))))))))))))) |
114 |
|
|
115 |
(define-ccl-program ccl-decode-mule-utf-16-le |
(define-ccl-program ccl-decode-mule-utf-16-le |
116 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
119 |
(read r3 r4) |
(read r3 r4) |
120 |
(r1 = (r4 <8 r3)) |
(r1 = (r4 <8 r3)) |
121 |
,utf-16-decode-ucs |
,utf-16-decode-ucs |
122 |
(translate-character utf-8-translation-table-for-decode r0 r1) |
(translate-character utf-translation-table-for-decode r0 r1) |
123 |
(write-multibyte-character r0 r1) |
(write-multibyte-character r0 r1) |
124 |
(repeat)))) |
(repeat)))) |
125 |
"Decode little endian UTF-16 (ignoring signature bytes). |
"Decode little endian UTF-16 (ignoring signature bytes). |
126 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
127 |
mule-unicode-*. Un-representable Unicode characters are |
mule-unicode-*. Un-representable Unicode characters are decoded as |
128 |
decoded as U+fffd. The result is run through translation table |
U+fffd. The result is run through the translation-table named |
129 |
`utf-8-translation-table-for-decode' if that is defined.") |
`utf-translation-table-for-decode'.") |
130 |
|
|
131 |
(define-ccl-program ccl-decode-mule-utf-16-be |
(define-ccl-program ccl-decode-mule-utf-16-be |
132 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
135 |
(read r3 r4) |
(read r3 r4) |
136 |
(r1 = (r3 <8 r4)) |
(r1 = (r3 <8 r4)) |
137 |
,utf-16-decode-ucs |
,utf-16-decode-ucs |
138 |
(translate-character utf-8-translation-table-for-decode r0 r1) |
(translate-character utf-translation-table-for-decode r0 r1) |
139 |
(write-multibyte-character r0 r1) |
(write-multibyte-character r0 r1) |
140 |
(repeat)))) |
(repeat)))) |
141 |
"Decode big endian UTF-16 (ignoring signature bytes). |
"Decode big endian UTF-16 (ignoring signature bytes). |
142 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
143 |
mule-unicode-*. Un-representable Unicode characters are |
mule-unicode-*. Un-representable Unicode characters are |
144 |
decoded as U+fffd. The result is run through translation table |
decoded as U+fffd. The result is run through the translation-table of |
145 |
`utf-8-non-latin-8859-table'.") |
name `utf-translation-table-for-decode'.") |
146 |
|
|
147 |
(makunbound 'utf-16-decode-ucs) ; done with it |
(makunbound 'utf-16-decode-ucs) ; done with it |
148 |
|
|
177 |
(write #xfe) |
(write #xfe) |
178 |
(loop |
(loop |
179 |
(read-multibyte-character r0 r1) |
(read-multibyte-character r0 r1) |
180 |
(translate-character ucs-mule-to-mule-unicode r0 r1) |
(lookup-character utf-subst-table-for-encode r0 r1) |
181 |
,utf-16-decode-to-ucs |
(if (r7 == 0) |
182 |
|
((translate-character utf-translation-table-for-encode r0 r1) |
183 |
|
,utf-16-decode-to-ucs)) |
184 |
(write (r0 & 255)) |
(write (r0 & 255)) |
185 |
(write (r0 >> 8)) |
(write (r0 >> 8)) |
186 |
(repeat)))) |
(repeat)))) |
187 |
"Encode to little endian UTF-16 with signature. |
"Encode to little endian UTF-16 with signature. |
188 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
189 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
190 |
after translation through the table `ucs-mule-to-mule-unicode'. |
after translation through the translation-table of name |
191 |
|
`utf-translation-table-for-encode'. |
192 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
193 |
|
|
194 |
(define-ccl-program ccl-encode-mule-utf-16-be |
(define-ccl-program ccl-encode-mule-utf-16-be |
197 |
(write #xff) |
(write #xff) |
198 |
(loop |
(loop |
199 |
(read-multibyte-character r0 r1) |
(read-multibyte-character r0 r1) |
200 |
(translate-character ucs-mule-to-mule-unicode r0 r1) |
(lookup-character utf-subst-table-for-encode r0 r1) |
201 |
,utf-16-decode-to-ucs |
(if (r7 == 0) |
202 |
|
((translate-character utf-translation-table-for-encode r0 r1) |
203 |
|
,utf-16-decode-to-ucs)) |
204 |
(write (r0 >> 8)) |
(write (r0 >> 8)) |
205 |
(write (r0 & 255)) |
(write (r0 & 255)) |
206 |
(repeat)))) |
(repeat)))) |
207 |
"Encode to big endian UTF-16 with signature. |
"Encode to big endian UTF-16 with signature. |
208 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
209 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
210 |
after translation through the table `ucs-mule-to-mule-unicode'. |
after translation through the translation-table named |
211 |
|
`utf-translation-table-for-encode'. |
212 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
213 |
|
|
214 |
(makunbound 'utf-16-decode-to-ucs) |
(makunbound 'utf-16-decode-to-ucs) |
217 |
|
|
218 |
Assumes and ignores the leading two-byte signature. |
Assumes and ignores the leading two-byte signature. |
219 |
|
|
220 |
The supported Emacs character sets are the following, plus others |
It supports Unicode characters of these ranges: |
221 |
which may be included in the translation table |
U+0000..U+33FF, U+E000..U+FFFF. |
222 |
`ucs-mule-to-mule-unicode': |
They correspond to these Emacs character sets: |
223 |
ascii |
ascii, latin-iso8859-1, mule-unicode-0100-24ff, |
224 |
eight-bit-control |
mule-unicode-2500-33ff, mule-unicode-e000-ffff |
225 |
latin-iso8859-1 |
|
226 |
mule-unicode-0100-24ff |
On decoding (e.g. reading a file), Unicode characters not in the above |
227 |
mule-unicode-2500-33ff |
ranges are decoded as U+FFFD, effectively corrupting the data |
228 |
mule-unicode-e000-ffff |
if they are re-encoded. |
229 |
|
|
230 |
Note that Unicode characters out of the ranges U+0000-U+33FF and |
On encoding (e.g. writing a file), Emacs characters not belonging to |
231 |
U+E200-U+FFFF are decoded as U+FFFD, effectively corrupting the data |
any of the character sets listed above are encoded into the byte |
232 |
if they are re-encoded. Emacs characters without Unicode conversions |
sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
|
are encoded as U+FFFD.")) |
|
233 |
(make-coding-system |
(make-coding-system |
234 |
'mule-utf-16-le 4 |
'mule-utf-16-le 4 |
235 |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
248 |
(mime-charset . utf-16le) |
(mime-charset . utf-16le) |
249 |
(coding-category . coding-category-utf-16-le) |
(coding-category . coding-category-utf-16-le) |
250 |
(valid-codes (0 . 255)) |
(valid-codes (0 . 255)) |
251 |
(pre-write-conversion . utf-16-le-pre-write-conversion))) |
(pre-write-conversion . utf-16-le-pre-write-conversion) |
252 |
|
(dependency unify-8859-on-encoding-mode |
253 |
|
unify-8859-on-decoding-mode |
254 |
|
utf-fragment-on-decoding |
255 |
|
utf-translate-cjk))) |
256 |
|
|
257 |
(make-coding-system |
(make-coding-system |
258 |
'mule-utf-16-be 4 ?u |
'mule-utf-16-be 4 ?u |
271 |
(mime-charset . utf-16be) |
(mime-charset . utf-16be) |
272 |
(coding-category . coding-category-utf-16-be) |
(coding-category . coding-category-utf-16-be) |
273 |
(valid-codes (0 . 255)) |
(valid-codes (0 . 255)) |
274 |
(pre-write-conversion . utf-16-be-pre-write-conversion))) |
(pre-write-conversion . utf-16-be-pre-write-conversion) |
275 |
|
(dependency unify-8859-on-encoding-mode |
276 |
(register-char-codings 'mule-utf-16-le ucs-mule-to-mule-unicode) |
unify-8859-on-decoding-mode |
277 |
(register-char-codings 'mule-utf-16-be ucs-mule-to-mule-unicode)) |
utf-fragment-on-decoding |
278 |
|
utf-translate-cjk)))) |
279 |
|
|
280 |
(define-coding-system-alias 'utf-16-le 'mule-utf-16-le) |
(define-coding-system-alias 'utf-16-le 'mule-utf-16-le) |
281 |
(define-coding-system-alias 'utf-16-be 'mule-utf-16-be) |
(define-coding-system-alias 'utf-16-be 'mule-utf-16-be) |