1 |
;;; utf-16.el --- UTF-16 encoding/decoding |
;;; utf-16.el --- UTF-16 encoding/decoding |
2 |
|
|
3 |
;; Copyright (C) 2001, 2002 Free Software Foundation, Inc. |
;; Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc. |
4 |
|
|
5 |
;; Author: Dave Love <fx@gnu.org> |
;; Author: Dave Love <fx@gnu.org> |
6 |
;; Keywords: Unicode, UTF-16, i18n |
;; Keywords: Unicode, UTF-16, i18n |
25 |
;;; Commentary: |
;;; Commentary: |
26 |
|
|
27 |
;; Support for UTF-16, which is a two-byte encoding (modulo |
;; Support for UTF-16, which is a two-byte encoding (modulo |
28 |
;; surrogates) of Unicode, written either in little or big endian |
;; surrogates) of Unicode, defined in RFC 2781. It is written either |
29 |
;; order: coding-systems `mule-utf-16-le' and `mule-utf-16-be'. |
;; in little or big endian order and either with or without the |
30 |
;; (utf-16-le is used by the DozeN'T clipboard, for instance.) The |
;; leading BOM (a two-byte signature which identifies their byte sex). |
31 |
;; data are preceeded by a two-byte signature which identifies their |
;; |
32 |
;; byte sex. These are used by the coding-category-utf-16-{b,l}e code |
;; We provide these base coding systems. |
33 |
;; to identify the coding, but ignored on decoding. |
;; name endian BOM |
34 |
|
;; ---- ------ --- |
35 |
|
;; mule-utf-16le little no |
36 |
|
;; mule-utf-16be big no |
37 |
|
;; mule-utf-16le-with-signature little yes |
38 |
|
;; mule-utf-16be-with-signature big yes |
39 |
|
;; mule-utf-16 both yes |
40 |
|
;; |
41 |
;; Note that un-decodable sequences aren't (yet?) preserved as raw |
;; Note that un-decodable sequences aren't (yet?) preserved as raw |
42 |
;; bytes, as they are with utf-8, so reading and writing as utf-16 can |
;; bytes, as they are with utf-8, so reading and writing as utf-16 can |
43 |
;; corrupt data. |
;; corrupt data. |
75 |
(eval-and-compile |
(eval-and-compile |
76 |
(defconst utf-16-decode-ucs |
(defconst utf-16-decode-ucs |
77 |
;; We have the unicode in r1. Output is charset ID in r0, code |
;; We have the unicode in r1. Output is charset ID in r0, code |
78 |
;; point in r1. As r6 keeps endian information, the value should |
;; point in r1. |
|
;; not be changed. |
|
79 |
`((lookup-integer utf-subst-table-for-decode r1 r3) |
`((lookup-integer utf-subst-table-for-decode r1 r3) |
80 |
(if r7 ; got a translation |
(if r7 ; got a translation |
81 |
((r0 = r1) (r1 = r3)) |
((r0 = r1) (r1 = r3)) |
116 |
(r1 -= #xe000) |
(r1 -= #xe000) |
117 |
(r2 = (((r1 / 96) + 32) << 7)) |
(r2 = (((r1 / 96) + 32) << 7)) |
118 |
(r1 %= 96) |
(r1 %= 96) |
119 |
(r1 += (r2 + 32)))))))))))))) |
(r1 += (r2 + 32))))))))))))) |
120 |
|
|
121 |
|
(defconst utf-16le-decode-loop |
122 |
|
`(loop |
123 |
|
(read r3 r4) |
124 |
|
(r1 = (r4 <8 r3)) |
125 |
|
,utf-16-decode-ucs |
126 |
|
(translate-character utf-translation-table-for-decode r0 r1) |
127 |
|
(write-multibyte-character r0 r1) |
128 |
|
(repeat))) |
129 |
|
|
130 |
|
(defconst utf-16be-decode-loop |
131 |
|
`(loop |
132 |
|
(read r3 r4) |
133 |
|
(r1 = (r3 <8 r4)) |
134 |
|
,@utf-16-decode-ucs |
135 |
|
(translate-character utf-translation-table-for-decode r0 r1) |
136 |
|
(write-multibyte-character r0 r1) |
137 |
|
(repeat))) |
138 |
|
|
139 |
|
) |
140 |
|
|
141 |
(define-ccl-program ccl-decode-mule-utf-16-le |
(define-ccl-program ccl-decode-mule-utf-16le |
142 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
143 |
((loop |
,utf-16le-decode-loop) |
|
(read r3 r4) |
|
|
(r1 = (r4 <8 r3)) |
|
|
,utf-16-decode-ucs |
|
|
(translate-character utf-translation-table-for-decode r0 r1) |
|
|
(write-multibyte-character r0 r1) |
|
|
(repeat)))) |
|
144 |
"Decode UTF-16LE (little endian without signature bytes). |
"Decode UTF-16LE (little endian without signature bytes). |
145 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
146 |
mule-unicode-*. Un-representable Unicode characters are decoded as |
mule-unicode-*. Un-representable Unicode characters are decoded as |
147 |
U+fffd. The result is run through the translation-table named |
U+fffd. The result is run through the translation-table named |
148 |
`utf-translation-table-for-decode'.") |
`utf-translation-table-for-decode'.") |
149 |
|
|
150 |
(define-ccl-program ccl-decode-mule-utf-16-be |
(define-ccl-program ccl-decode-mule-utf-16be |
151 |
`(2 ; 2 bytes -> 1 to 4 bytes |
`(2 ; 2 bytes -> 1 to 4 bytes |
152 |
((loop |
,utf-16be-decode-loop) |
|
(read r3 r4) |
|
|
(r1 = (r3 <8 r4)) |
|
|
,utf-16-decode-ucs |
|
|
(translate-character utf-translation-table-for-decode r0 r1) |
|
|
(write-multibyte-character r0 r1) |
|
|
(repeat)))) |
|
153 |
"Decode UTF-16BE (big endian without signature bytes). |
"Decode UTF-16BE (big endian without signature bytes). |
154 |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
155 |
mule-unicode-*. Un-representable Unicode characters are |
mule-unicode-*. Un-representable Unicode characters are |
156 |
decoded as U+fffd. The result is run through the translation-table of |
decoded as U+fffd. The result is run through the translation-table of |
157 |
name `utf-translation-table-for-decode'.") |
name `utf-translation-table-for-decode'.") |
158 |
|
|
159 |
|
(define-ccl-program ccl-decode-mule-utf-16le-with-signature |
160 |
|
`(2 |
161 |
|
((read r3 r4) |
162 |
|
,utf-16le-decode-loop)) |
163 |
|
"Like ccl-decode-utf-16le but skip the first 2-byte BOM.") |
164 |
|
|
165 |
|
(define-ccl-program ccl-decode-mule-utf-16be-with-signature |
166 |
|
`(2 |
167 |
|
((read r3 r4) |
168 |
|
,utf-16be-decode-loop)) |
169 |
|
"Like ccl-decode-utf-16be but skip the first 2-byte BOM.") |
170 |
|
|
171 |
|
(define-ccl-program ccl-decode-mule-utf-16 |
172 |
|
`(2 |
173 |
|
((read r3 r4) |
174 |
|
(r1 = (r3 <8 r4)) |
175 |
|
(if (r1 == #xFFFE) |
176 |
|
;; R1 is a BOM for little endian. We keep this character as |
177 |
|
;; is temporarily. It is removed by post-read-conversion |
178 |
|
;; function. |
179 |
|
(,@utf-16-decode-ucs |
180 |
|
(write-multibyte-character r0 r1) |
181 |
|
,utf-16le-decode-loop) |
182 |
|
((if (r1 == #xFEFF) |
183 |
|
;; R1 is a BOM for big endian, but we can't keep that |
184 |
|
;; character in the output because it can't be |
185 |
|
;; distinguished with the normal U+FEFF. So, we keep |
186 |
|
;; #xFFFF instead. |
187 |
|
((r1 = #xFFFF) |
188 |
|
,@utf-16-decode-ucs) |
189 |
|
;; R1 a normal Unicode character. |
190 |
|
(,@utf-16-decode-ucs |
191 |
|
(translate-character utf-translation-table-for-decode r0 r1))) |
192 |
|
(write-multibyte-character r0 r1) |
193 |
|
,utf-16be-decode-loop)))) |
194 |
|
"Like ccl-decode-utf-16be/le but check the first BOM.") |
195 |
|
|
196 |
(makunbound 'utf-16-decode-ucs) ; done with it |
(makunbound 'utf-16-decode-ucs) ; done with it |
197 |
|
(makunbound 'utf-16le-decode-loop) |
198 |
|
(makunbound 'utf-16be-decode-loop) |
199 |
|
|
200 |
(eval-and-compile |
(eval-and-compile |
201 |
(defconst utf-16-decode-to-ucs |
(defconst utf-16-decode-to-ucs |
220 |
(r0 = (r3 + #x2500)) |
(r0 = (r3 + #x2500)) |
221 |
(if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
(if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
222 |
(r0 = (r3 + #xe000)) |
(r0 = (r3 + #xe000)) |
223 |
(r0 = #xfffd))))))))))) |
(r0 = #xfffd)))))))))) |
224 |
|
|
225 |
|
(defconst utf-16le-encode-loop |
226 |
|
`(loop |
227 |
|
(read-multibyte-character r0 r1) |
228 |
|
(lookup-character utf-subst-table-for-encode r0 r1) |
229 |
|
(if (r7 == 0) |
230 |
|
((translate-character utf-translation-table-for-encode r0 r1) |
231 |
|
,utf-16-decode-to-ucs)) |
232 |
|
(write (r0 & 255)) |
233 |
|
(write (r0 >> 8)) |
234 |
|
(repeat))) |
235 |
|
|
236 |
|
(defconst utf-16be-encode-loop |
237 |
|
`(loop |
238 |
|
(read-multibyte-character r0 r1) |
239 |
|
(lookup-character utf-subst-table-for-encode r0 r1) |
240 |
|
(if (r7 == 0) |
241 |
|
((translate-character utf-translation-table-for-encode r0 r1) |
242 |
|
,utf-16-decode-to-ucs)) |
243 |
|
(write (r0 >> 8)) |
244 |
|
(write (r0 & 255)) |
245 |
|
(repeat))) |
246 |
|
) |
247 |
|
|
248 |
(define-ccl-program ccl-encode-mule-utf-16-le |
|
249 |
|
(define-ccl-program ccl-encode-mule-utf-16le |
250 |
`(1 |
`(1 |
251 |
((loop |
,utf-16le-encode-loop) |
|
(read-multibyte-character r0 r1) |
|
|
(lookup-character utf-subst-table-for-encode r0 r1) |
|
|
(if (r7 == 0) |
|
|
((translate-character utf-translation-table-for-encode r0 r1) |
|
|
,utf-16-decode-to-ucs)) |
|
|
(write (r0 & 255)) |
|
|
(write (r0 >> 8)) |
|
|
(repeat)))) |
|
252 |
"Encode to UTF-16LE (little endian without signature). |
"Encode to UTF-16LE (little endian without signature). |
253 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
254 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
256 |
`utf-translation-table-for-encode'. |
`utf-translation-table-for-encode'. |
257 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
258 |
|
|
259 |
(define-ccl-program ccl-encode-mule-utf-16-be |
(define-ccl-program ccl-encode-mule-utf-16be |
260 |
`(1 |
`(1 |
261 |
((loop |
,utf-16be-encode-loop) |
|
(read-multibyte-character r0 r1) |
|
|
(lookup-character utf-subst-table-for-encode r0 r1) |
|
|
(if (r7 == 0) |
|
|
((translate-character utf-translation-table-for-encode r0 r1) |
|
|
,utf-16-decode-to-ucs)) |
|
|
(write (r0 >> 8)) |
|
|
(write (r0 & 255)) |
|
|
(repeat)))) |
|
262 |
"Encode to UTF-16BE (big endian without signature). |
"Encode to UTF-16BE (big endian without signature). |
263 |
Characters from the charsets ascii, eight-bit-control, |
Characters from the charsets ascii, eight-bit-control, |
264 |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
266 |
`utf-translation-table-for-encode'. |
`utf-translation-table-for-encode'. |
267 |
Others are encoded as U+FFFD.") |
Others are encoded as U+FFFD.") |
268 |
|
|
269 |
|
(define-ccl-program ccl-encode-mule-utf-16le-with-signature |
270 |
|
`(1 |
271 |
|
((write #xFF) |
272 |
|
(write #xFE) |
273 |
|
,utf-16le-encode-loop)) |
274 |
|
"Encode to UTF-16 (little endian with signature). |
275 |
|
Characters from the charsets ascii, eight-bit-control, |
276 |
|
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
277 |
|
after translation through the translation-table of name |
278 |
|
`utf-translation-table-for-encode'. |
279 |
|
Others are encoded as U+FFFD.") |
280 |
|
|
281 |
|
(define-ccl-program ccl-encode-mule-utf-16be-with-signature |
282 |
|
`(1 |
283 |
|
((write #xFE) |
284 |
|
(write #xFF) |
285 |
|
,utf-16be-encode-loop)) |
286 |
|
"Encode to UTF-16 (big endian with signature). |
287 |
|
Characters from the charsets ascii, eight-bit-control, |
288 |
|
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded |
289 |
|
after translation through the translation-table named |
290 |
|
`utf-translation-table-for-encode'. |
291 |
|
Others are encoded as U+FFFD.") |
292 |
|
|
293 |
(makunbound 'utf-16-decode-to-ucs) |
(makunbound 'utf-16-decode-to-ucs) |
294 |
|
(makunbound 'utf-16le-encode-loop) |
295 |
|
(makunbound 'utf-16be-encode-loop) |
296 |
|
|
297 |
|
(defun mule-utf-16-post-read-conversion (length) |
298 |
|
(when (> length 0) |
299 |
|
(let ((char (following-char))) |
300 |
|
(cond ((= char (decode-char 'ucs #xFFFE)) |
301 |
|
(delete-char 1) |
302 |
|
(setq last-coding-system-used |
303 |
|
(coding-system-change-text-conversion |
304 |
|
last-coding-system-used |
305 |
|
'mule-utf-16le-with-signature)) |
306 |
|
(setq length (1- length))) |
307 |
|
((= char (decode-char 'ucs #xFFFF)) |
308 |
|
(delete-char 1) |
309 |
|
(setq last-coding-system-used |
310 |
|
(coding-system-change-text-conversion |
311 |
|
last-coding-system-used |
312 |
|
'mule-utf-16be-with-signature)) |
313 |
|
(setq length (1- length))) |
314 |
|
(t |
315 |
|
(setq last-coding-system-used 'mule-utf-16be))))) |
316 |
|
length) |
317 |
|
|
318 |
(let ((doc " |
(let ((doc " |
319 |
|
|
331 |
any of the character sets listed above are encoded into the byte |
any of the character sets listed above are encoded into the byte |
332 |
sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
333 |
(make-coding-system |
(make-coding-system |
334 |
'mule-utf-16-le 4 |
'mule-utf-16le 4 |
335 |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
336 |
(concat |
(concat |
337 |
"Little endian UTF-16 encoding for Emacs-supported Unicode characters." |
"UTF-16LE encoding for Emacs-supported Unicode characters." |
338 |
doc) |
doc) |
339 |
|
|
340 |
'(ccl-decode-mule-utf-16-le . ccl-encode-mule-utf-16-le) |
'(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le) |
341 |
'((safe-charsets |
'((safe-charsets |
342 |
ascii |
ascii |
343 |
eight-bit-control |
eight-bit-control |
346 |
mule-unicode-2500-33ff |
mule-unicode-2500-33ff |
347 |
mule-unicode-e000-ffff) |
mule-unicode-e000-ffff) |
348 |
(mime-charset . utf-16le) |
(mime-charset . utf-16le) |
349 |
(coding-category . coding-category-utf-16-le) |
(mime-text-unsuitable . t) |
350 |
(valid-codes (0 . 255)) |
(valid-codes (0 . 255)) |
351 |
(dependency unify-8859-on-encoding-mode |
(dependency unify-8859-on-encoding-mode |
352 |
unify-8859-on-decoding-mode |
unify-8859-on-decoding-mode |
354 |
utf-translate-cjk))) |
utf-translate-cjk))) |
355 |
|
|
356 |
(make-coding-system |
(make-coding-system |
357 |
'mule-utf-16-be 4 ?u |
'mule-utf-16be 4 ?u |
358 |
(concat |
(concat |
359 |
"Big endian UTF-16 encoding for Emacs-supported Unicode characters." |
"UTF-16BE encoding for Emacs-supported Unicode characters." |
360 |
doc) |
doc) |
361 |
|
|
362 |
'(ccl-decode-mule-utf-16-be . ccl-encode-mule-utf-16-be) |
'(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be) |
363 |
'((safe-charsets |
'((safe-charsets |
364 |
ascii |
ascii |
365 |
eight-bit-control |
eight-bit-control |
368 |
mule-unicode-2500-33ff |
mule-unicode-2500-33ff |
369 |
mule-unicode-e000-ffff) |
mule-unicode-e000-ffff) |
370 |
(mime-charset . utf-16be) |
(mime-charset . utf-16be) |
371 |
|
(valid-codes (0 . 255)) |
372 |
|
(dependency unify-8859-on-encoding-mode |
373 |
|
unify-8859-on-decoding-mode |
374 |
|
utf-fragment-on-decoding |
375 |
|
utf-translate-cjk))) |
376 |
|
|
377 |
|
(make-coding-system |
378 |
|
'mule-utf-16le-with-signature 4 ?u |
379 |
|
(concat |
380 |
|
"Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
381 |
|
doc) |
382 |
|
|
383 |
|
'(ccl-decode-mule-utf-16le-with-signature |
384 |
|
. ccl-encode-mule-utf-16le-with-signature) |
385 |
|
'((safe-charsets |
386 |
|
ascii |
387 |
|
eight-bit-control |
388 |
|
latin-iso8859-1 |
389 |
|
mule-unicode-0100-24ff |
390 |
|
mule-unicode-2500-33ff |
391 |
|
mule-unicode-e000-ffff) |
392 |
|
(coding-category . coding-category-utf-16-le) |
393 |
|
(mime-charset . utf-16) |
394 |
|
(mime-text-unsuitable . t) |
395 |
|
(valid-codes (0 . 255)) |
396 |
|
(dependency unify-8859-on-encoding-mode |
397 |
|
unify-8859-on-decoding-mode |
398 |
|
utf-fragment-on-decoding |
399 |
|
utf-translate-cjk))) |
400 |
|
|
401 |
|
(make-coding-system |
402 |
|
'mule-utf-16be-with-signature 4 ?u |
403 |
|
(concat |
404 |
|
"Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters." |
405 |
|
doc) |
406 |
|
|
407 |
|
'(ccl-decode-mule-utf-16be-with-signature |
408 |
|
. ccl-encode-mule-utf-16be-with-signature) |
409 |
|
'((safe-charsets |
410 |
|
ascii |
411 |
|
eight-bit-control |
412 |
|
latin-iso8859-1 |
413 |
|
mule-unicode-0100-24ff |
414 |
|
mule-unicode-2500-33ff |
415 |
|
mule-unicode-e000-ffff) |
416 |
(coding-category . coding-category-utf-16-be) |
(coding-category . coding-category-utf-16-be) |
417 |
|
(mime-charset . utf-16) |
418 |
(valid-codes (0 . 255)) |
(valid-codes (0 . 255)) |
419 |
(dependency unify-8859-on-encoding-mode |
(dependency unify-8859-on-encoding-mode |
420 |
unify-8859-on-decoding-mode |
unify-8859-on-decoding-mode |
421 |
utf-fragment-on-decoding |
utf-fragment-on-decoding |
422 |
utf-translate-cjk)))) |
utf-translate-cjk))) |
423 |
|
|
424 |
(define-coding-system-alias 'utf-16-le 'mule-utf-16-le) |
(make-coding-system |
425 |
(define-coding-system-alias 'utf-16-be 'mule-utf-16-be) |
'mule-utf-16 4 ?u |
426 |
|
(concat |
427 |
|
"UTF-16 (with or without BOM) for Emacs-supported Unicode characters." |
428 |
|
doc) |
429 |
|
|
430 |
|
'(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature) |
431 |
|
'((safe-charsets |
432 |
|
ascii |
433 |
|
eight-bit-control |
434 |
|
latin-iso8859-1 |
435 |
|
mule-unicode-0100-24ff |
436 |
|
mule-unicode-2500-33ff |
437 |
|
mule-unicode-e000-ffff) |
438 |
|
(coding-category . coding-category-utf-16-be) |
439 |
|
(mime-charset . utf-16) |
440 |
|
(mime-text-unsuitable . t) |
441 |
|
(valid-codes (0 . 255)) |
442 |
|
(dependency unify-8859-on-encoding-mode |
443 |
|
unify-8859-on-decoding-mode |
444 |
|
utf-fragment-on-decoding |
445 |
|
utf-translate-cjk) |
446 |
|
(post-read-conversion . mule-utf-16-post-read-conversion))) |
447 |
|
) |
448 |
|
|
449 |
|
(define-coding-system-alias 'utf-16le 'mule-utf-16le) |
450 |
|
(define-coding-system-alias 'utf-16be 'mule-utf-16be) |
451 |
|
(define-coding-system-alias 'utf-16le-with-signature |
452 |
|
'mule-utf-16le-with-signature) |
453 |
|
(define-coding-system-alias 'utf-16be-with-signature |
454 |
|
'mule-utf-16be-with-signature) |
455 |
|
(define-coding-system-alias 'utf-16 'mule-utf-16) |
456 |
|
|
457 |
|
;; For backward compatibility. |
458 |
|
(define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature) |
459 |
|
(define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature) |
460 |
|
(define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature) |
461 |
|
(define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature) |
462 |
|
|
463 |
|
;;; arch-tag: 85455d46-d9c9-466d-a6f3-c3582a7367c4 |
464 |
;;; utf-16.el ends here |
;;; utf-16.el ends here |