26 |
|
|
27 |
;; Support for UTF-16, which is a two-byte encoding (modulo |
;; Support for UTF-16, which is a two-byte encoding (modulo |
28 |
;; surrogates) of Unicode, written either in little or big endian |
;; surrogates) of Unicode, written either in little or big endian |
29 |
;; order: coding-systems `mule-utf-16-le' and `mule-utf-16-be'. |
;; order and either with or without the leading BOM (a two-byte |
30 |
;; (utf-16-le is used by the DozeN'T clipboard, for instance.) The |
;; signature which identifies their byte sex)a. |
31 |
;; data are preceeded by a two-byte signature which identifies their |
;; |
32 |
;; byte sex. These are used by the coding-category-utf-16-{b,l}e code |
;; We provides these base coding systems. |
33 |
;; to identify the coding, but ignored on decoding. |
;; name endian BOM |
34 |
|
;; ---- ------ --- |
35 |
|
;; mule-utf-16le little no |
36 |
|
;; mule-utf-16be big no |
37 |
|
;; mule-utf-16le-with-signature little yes |
38 |
|
;; mule-utf-16be-with-signature big yes |
39 |
|
;; mule-utf-16 both yes |
40 |
|
;; |
41 |
;; Note that un-decodable sequences aren't (yet?) preserved as raw |
;; Note that un-decodable sequences aren't (yet?) preserved as raw |
42 |
;; bytes, as they are with utf-8, so reading and writing as utf-16 can |
;; bytes, as they are with utf-8, so reading and writing as utf-16 can |
43 |
;; corrupt data. |
;; corrupt data. |
118 |
(r1 %= 96) |
(r1 %= 96) |
119 |
(r1 += (r2 + 32))))))))))))) |
(r1 += (r2 + 32))))))))))))) |
120 |
|
|
121 |
(defconst utf-16-le-decode-loop |
(defconst utf-16le-decode-loop |
122 |
`(loop |
`(loop |
123 |
(read r3 r4) |
(read r3 r4) |
124 |
(r1 = (r4 <8 r3)) |
(r1 = (r4 <8 r3)) |
127 |
(write-multibyte-character r0 r1) |
(write-multibyte-character r0 r1) |
128 |
(repeat))) |
(repeat))) |
129 |
|
|
130 |
(defconst utf-16-be-decode-loop |
(defconst utf-16be-decode-loop |
131 |
`(loop |
`(loop |
132 |
(read r3 r4) |
(read r3 r4) |
133 |
(r1 = (r3 <8 r4)) |
(r1 = (r3 <8 r4)) |
138 |
|
|
139 |
) |
) |
140 |
|
|
141 |
(define-ccl-program ccl-decode-mule-utf-16-le |
(define-ccl-program ccl-decode-mule-utf-16le |
142 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
143 |
,utf-16-le-decode-loop) |
,utf-16le-decode-loop) |
144 |
"Decode UTF-16LE (little endian without signature bytes). |
"Decode UTF-16LE (little endian without signature bytes). |
145 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
146 |
mule-unicode-*. Un-representable Unicode characters are decoded as |
mule-unicode-*. Un-representable Unicode characters are decoded as |
147 |
U+fffd. The result is run through the translation-table named |
U+fffd. The result is run through the translation-table named |
148 |
`utf-translation-table-for-decode'.") |
`utf-translation-table-for-decode'.") |
149 |
|
|
150 |
(define-ccl-program ccl-decode-mule-utf-16-be |
(define-ccl-program ccl-decode-mule-utf-16be |
151 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
152 |
,utf-16-be-decode-loop) |
,utf-16be-decode-loop) |
153 |
"Decode UTF-16BE (big endian without signature bytes). |
"Decode UTF-16BE (big endian without signature bytes). |
154 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
155 |
mule-unicode-*. Un-representable Unicode characters are |
mule-unicode-*. Un-representable Unicode characters are |
156 |
decoded as U+fffd. The result is run through the translation-table of |
decoded as U+fffd. The result is run through the translation-table of |
157 |
name `utf-translation-table-for-decode'.") |
name `utf-translation-table-for-decode'.") |
158 |
|
|
159 |
(define-ccl-program ccl-decode-mule-utf-16-le-with-signature |
(define-ccl-program ccl-decode-mule-utf-16le-with-signature |
160 |
`(2 |
`(2 |
161 |
((read r3 r4) |
((read r3 r4) |
162 |
,utf-16-le-decode-loop)) |
,utf-16le-decode-loop)) |
163 |
"Like ccl-decode-utf-16-le but skip the first 2-byte BOM.") |
"Like ccl-decode-utf-16le but skip the first 2-byte BOM.") |
164 |
|
|
165 |
(define-ccl-program ccl-decode-mule-utf-16-be-with-signature |
(define-ccl-program ccl-decode-mule-utf-16be-with-signature |
166 |
`(2 |
`(2 |
167 |
((read r3 r4) |
((read r3 r4) |
168 |
,utf-16-be-decode-loop)) |
,utf-16be-decode-loop)) |
169 |
"Like ccl-decode-utf-16-be but skip the first 2-byte BOM.") |
"Like ccl-decode-utf-16be but skip the first 2-byte BOM.") |
170 |
|
|
171 |
(define-ccl-program ccl-decode-mule-utf-16 |
(define-ccl-program ccl-decode-mule-utf-16 |
172 |
`(2 |
`(2 |
178 |
;; function. |
;; function. |
179 |
(,@utf-16-decode-ucs |
(,@utf-16-decode-ucs |
180 |
(write-multibyte-character r0 r1) |
(write-multibyte-character r0 r1) |
181 |
,utf-16-le-decode-loop) |
,utf-16le-decode-loop) |
182 |
((if (r1 == #xFEFF) |
((if (r1 == #xFEFF) |
183 |
;; R1 is a BOM for big endian, but we can't keep that |
;; R1 is a BOM for big endian, but we can't keep that |
184 |
;; character in the output because it can't be |
;; character in the output because it can't be |
190 |
(,@utf-16-decode-ucs |
(,@utf-16-decode-ucs |
191 |
(translate-character utf-translation-table-for-decode r0 r1))) |
(translate-character utf-translation-table-for-decode r0 r1))) |
192 |
(write-multibyte-character r0 r1) |
(write-multibyte-character r0 r1) |
193 |
,utf-16-be-decode-loop)))) |
,utf-16be-decode-loop)))) |
194 |
"Like ccl-decode-utf-16-be/le but check the first BOM.") |
"Like ccl-decode-utf-16be/le but check the first BOM.") |
195 |
|
|
196 |
(makunbound 'utf-16-decode-ucs) ; done with it |
(makunbound 'utf-16-decode-ucs) ; done with it |
197 |
(makunbound 'utf-16-le-decode-loop) |
(makunbound 'utf-16le-decode-loop) |
198 |
(makunbound 'utf-16-be-decode-loop) |
(makunbound 'utf-16be-decode-loop) |
199 |
|
|
200 |
(eval-and-compile |
(eval-and-compile |
201 |
(defconst utf-16-decode-to-ucs |
(defconst utf-16-decode-to-ucs |
222 |
(r0 = (r3 + #xe000)) |
(r0 = (r3 + #xe000)) |
223 |
(r0 = #xfffd)))))))))) |
(r0 = #xfffd)))))))))) |
224 |
|
|
225 |
(defconst utf-16-le-encode-loop |
(defconst utf-16le-encode-loop |
226 |
`(loop |
`(loop |
227 |
(read-multibyte-character r0 r1) |
(read-multibyte-character r0 r1) |
228 |
(lookup-character utf-subst-table-for-encode r0 r1) |
(lookup-character utf-subst-table-for-encode r0 r1) |
233 |
(write (r0 >> 8)) |
(write (r0 >> 8)) |
234 |
(repeat))) |
(repeat))) |
235 |
|
|
236 |
(defconst utf-16-be-encode-loop |
(defconst utf-16be-encode-loop |
237 |
`(loop |
`(loop |
238 |
(read-multibyte-character r0 r1) |
(read-multibyte-character r0 r1) |
239 |
(lookup-character utf-subst-table-for-encode r0 r1) |
(lookup-character utf-subst-table-for-encode r0 r1) |
245 |
(repeat))) |
(repeat))) |
246 |
) |
) |
247 |
|
|
248 |
(define-ccl-program ccl-encode-mule-utf-16-le |
|
249 |
|
(define-ccl-program ccl-encode-mule-utf-16le |
250 |
`(1 |
`(1 |
251 |
,utf-16-le-encode-loop) |
,utf-16le-encode-loop) |
252 |
"Encode to UTF-16LE (little endian without signature). |
"Encode to UTF-16LE (little endian without signature). |
253 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
254 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
256 |
`utf-translation-table-for-encode'. |
`utf-translation-table-for-encode'. |
257 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
258 |
|
|
259 |
(define-ccl-program ccl-encode-mule-utf-16-be |
(define-ccl-program ccl-encode-mule-utf-16be |
260 |
`(1 |
`(1 |
261 |
,utf-16-be-encode-loop) |
,utf-16be-encode-loop) |
262 |
"Encode to UTF-16BE (big endian without signature). |
"Encode to UTF-16BE (big endian without signature). |
263 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
264 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
266 |
`utf-translation-table-for-encode'. |
`utf-translation-table-for-encode'. |
267 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
268 |
|
|
269 |
(define-ccl-program ccl-encode-mule-utf-16-le-with-signature |
(define-ccl-program ccl-encode-mule-utf-16le-with-signature |
270 |
`(1 |
`(1 |
271 |
((write #xFF) |
((write #xFF) |
272 |
(write #xFE) |
(write #xFE) |
273 |
,utf-16-le-encode-loop)) |
,utf-16le-encode-loop)) |
274 |
"Encode to UTF-16 (little endian with signature). |
"Encode to UTF-16 (little endian with signature). |
275 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
276 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
278 |
`utf-translation-table-for-encode'. |
`utf-translation-table-for-encode'. |
279 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
280 |
|
|
281 |
(define-ccl-program ccl-encode-mule-utf-16-be-with-signature |
(define-ccl-program ccl-encode-mule-utf-16be-with-signature |
282 |
`(1 |
`(1 |
283 |
((write #xFE) |
((write #xFE) |
284 |
(write #xFF) |
(write #xFF) |
285 |
,utf-16-be-encode-loop)) |
,utf-16be-encode-loop)) |
286 |
"Encode to UTF-16 (big endian with signature). |
"Encode to UTF-16 (big endian with signature). |
287 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
288 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
291 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
292 |
|
|
293 |
(makunbound 'utf-16-decode-to-ucs) |
(makunbound 'utf-16-decode-to-ucs) |
294 |
(makunbound 'utf-16-le-encode-loop) |
(makunbound 'utf-16le-encode-loop) |
295 |
(makunbound 'utf-16-be-encode-loop) |
(makunbound 'utf-16be-encode-loop) |
296 |
|
|
297 |
(defun mule-utf-16-post-read-conversion (length) |
(defun mule-utf-16-post-read-conversion (length) |
298 |
(when (> length 0) |
(when (> length 0) |
302 |
(setq last-coding-system-used |
(setq last-coding-system-used |
303 |
(coding-system-change-text-conversion |
(coding-system-change-text-conversion |
304 |
last-coding-system-used |
last-coding-system-used |
305 |
'mule-utf-16-le-with-signature)) |
'mule-utf-16le-with-signature)) |
306 |
(setq length (1- length))) |
(setq length (1- length))) |
307 |
((= char (decode-char 'ucs #xFFFF)) |
((= char (decode-char 'ucs #xFFFF)) |
308 |
(delete-char 1) |
(delete-char 1) |
309 |
(setq last-coding-system-used |
(setq last-coding-system-used |
310 |
(coding-system-change-text-conversion |
(coding-system-change-text-conversion |
311 |
last-coding-system-used |
last-coding-system-used |
312 |
'mule-utf-16-be-with-signature)) |
'mule-utf-16be-with-signature)) |
313 |
(setq length (1- length))) |
(setq length (1- length))) |
314 |
(t |
(t |
315 |
(setq last-coding-system-used 'mule-utf-16-be))))) |
(setq last-coding-system-used 'mule-utf-16be))))) |
316 |
length) |
length) |
317 |
|
|
318 |
(let ((doc " |
(let ((doc " |
331 |
any of the character sets listed above are encoded into the byte |
any of the character sets listed above are encoded into the byte |
332 |
sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
333 |
(make-coding-system |
(make-coding-system |
334 |
'mule-utf-16-le 4 |
'mule-utf-16le 4 |
335 |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
336 |
(concat |
(concat |
337 |
"Little endian UTF-16 encoding for Emacs-supported Unicode characters." |
"UTF-16LE encoding for Emacs-supported Unicode characters." |
338 |
doc) |
doc) |
339 |
|
|
340 |
'(ccl-decode-mule-utf-16-le . ccl-encode-mule-utf-16-le) |
'(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le) |
341 |
'((safe-charsets |
'((safe-charsets |
342 |
ascii |
ascii |
343 |
eight-bit-control |
eight-bit-control |
353 |
utf-translate-cjk))) |
utf-translate-cjk))) |
354 |
|
|
355 |
(make-coding-system |
(make-coding-system |
356 |
'mule-utf-16-be 4 ?u |
'mule-utf-16be 4 ?u |
357 |
(concat |
(concat |
358 |
"Big endian UTF-16 encoding for Emacs-supported Unicode characters." |
"UTF-16BE encoding for Emacs-supported Unicode characters." |
359 |
doc) |
doc) |
360 |
|
|
361 |
'(ccl-decode-mule-utf-16-be . ccl-encode-mule-utf-16-be) |
'(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be) |
362 |
'((safe-charsets |
'((safe-charsets |
363 |
ascii |
ascii |
364 |
eight-bit-control |
eight-bit-control |
374 |
utf-translate-cjk))) |
utf-translate-cjk))) |
375 |
|
|
376 |
(make-coding-system |
(make-coding-system |
377 |
'mule-utf-16-le-with-signature 4 ?u |
'mule-utf-16le-with-signature 4 ?u |
378 |
(concat |
(concat |
379 |
"Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
"Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
380 |
doc) |
doc) |
381 |
|
|
382 |
'(ccl-decode-mule-utf-16-le-with-signature |
'(ccl-decode-mule-utf-16le-with-signature |
383 |
. ccl-encode-mule-utf-16-le-with-signature) |
. ccl-encode-mule-utf-16le-with-signature) |
384 |
'((safe-charsets |
'((safe-charsets |
385 |
ascii |
ascii |
386 |
eight-bit-control |
eight-bit-control |
397 |
utf-translate-cjk))) |
utf-translate-cjk))) |
398 |
|
|
399 |
(make-coding-system |
(make-coding-system |
400 |
'mule-utf-16-be-with-signature 4 ?u |
'mule-utf-16be-with-signature 4 ?u |
401 |
(concat |
(concat |
402 |
"Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
"Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
403 |
doc) |
doc) |
404 |
|
|
405 |
'(ccl-decode-mule-utf-16-be-with-signature |
'(ccl-decode-mule-utf-16be-with-signature |
406 |
. ccl-encode-mule-utf-16-be-with-signature) |
. ccl-encode-mule-utf-16be-with-signature) |
407 |
'((safe-charsets |
'((safe-charsets |
408 |
ascii |
ascii |
409 |
eight-bit-control |
eight-bit-control |
425 |
"UTF-16 (with or without BOM) for Emacs-supported Unicode characters." |
"UTF-16 (with or without BOM) for Emacs-supported Unicode characters." |
426 |
doc) |
doc) |
427 |
|
|
428 |
'(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16-be-with-signature) |
'(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature) |
429 |
'((safe-charsets |
'((safe-charsets |
430 |
ascii |
ascii |
431 |
eight-bit-control |
eight-bit-control |
443 |
(post-read-conversion . mule-utf-16-post-read-conversion))) |
(post-read-conversion . mule-utf-16-post-read-conversion))) |
444 |
) |
) |
445 |
|
|
446 |
(define-coding-system-alias 'utf-16-le 'mule-utf-16-le) |
(define-coding-system-alias 'utf-16le 'mule-utf-16le) |
447 |
(define-coding-system-alias 'utf-16-be 'mule-utf-16-be) |
(define-coding-system-alias 'utf-16be 'mule-utf-16be) |
448 |
(define-coding-system-alias 'utf-16-le-with-signature |
(define-coding-system-alias 'utf-16le-with-signature |
449 |
'mule-utf-16-le-with-signature) |
'mule-utf-16le-with-signature) |
450 |
(define-coding-system-alias 'utf-16-be-with-signature |
(define-coding-system-alias 'utf-16be-with-signature |
451 |
'mule-utf-16-be-with-signature) |
'mule-utf-16be-with-signature) |
452 |
(define-coding-system-alias 'utf-16 'mule-utf-16) |
(define-coding-system-alias 'utf-16 'mule-utf-16) |
453 |
|
|
454 |
|
;; For backward compatibility. |
455 |
|
(define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature) |
456 |
|
(define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature) |
457 |
|
(define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature) |
458 |
|
(define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature) |
459 |
|
|
460 |
;;; utf-16.el ends here |
;;; utf-16.el ends here |