/[emacs]/emacs/lisp/international/utf-16.el
ViewVC logotype

Diff of /emacs/lisp/international/utf-16.el

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.9 by handa, Tue Apr 8 07:23:44 2003 UTC revision 1.10 by handa, Sat Jun 21 02:26:13 2003 UTC
# Line 26  Line 26 
26    
27  ;; Support for UTF-16, which is a two-byte encoding (modulo  ;; Support for UTF-16, which is a two-byte encoding (modulo
28  ;; surrogates) of Unicode, written either in little or big endian  ;; surrogates) of Unicode, written either in little or big endian
29  ;; order: coding-systems `mule-utf-16-le' and `mule-utf-16-be'.  ;; order and either with or without the leading BOM (a two-byte
30  ;; (utf-16-le is used by the DozeN'T clipboard, for instance.)  The  ;; signature which identifies their byte sex)a.
31  ;; data are preceeded by a two-byte signature which identifies their  ;;
32  ;; byte sex.  These are used by the coding-category-utf-16-{b,l}e code  ;; We provides these base coding systems.
33  ;; to identify the coding, but ignored on decoding.  ;;      name                                    endian  BOM
34    ;;      ----                                    ------  ---
35    ;;      mule-utf-16le                           little  no
36    ;;      mule-utf-16be                           big     no
37    ;;      mule-utf-16le-with-signature            little  yes
38    ;;      mule-utf-16be-with-signature            big     yes
39    ;;      mule-utf-16                             both    yes
40    ;;
41  ;; Note that un-decodable sequences aren't (yet?) preserved as raw  ;; Note that un-decodable sequences aren't (yet?) preserved as raw
42  ;; bytes, as they are with utf-8, so reading and writing as utf-16 can  ;; bytes, as they are with utf-8, so reading and writing as utf-16 can
43  ;; corrupt data.  ;; corrupt data.
# Line 112  Line 118 
118                       (r1 %= 96)                       (r1 %= 96)
119                       (r1 += (r2 + 32)))))))))))))                       (r1 += (r2 + 32)))))))))))))
120    
121  (defconst utf-16-le-decode-loop  (defconst utf-16le-decode-loop
122    `(loop    `(loop
123      (read r3 r4)      (read r3 r4)
124      (r1 = (r4 <8 r3))      (r1 = (r4 <8 r3))
# Line 121  Line 127 
127      (write-multibyte-character r0 r1)      (write-multibyte-character r0 r1)
128      (repeat)))      (repeat)))
129    
130  (defconst utf-16-be-decode-loop  (defconst utf-16be-decode-loop
131    `(loop    `(loop
132      (read r3 r4)      (read r3 r4)
133      (r1 = (r3 <8 r4))      (r1 = (r3 <8 r4))
# Line 132  Line 138 
138    
139  )  )
140    
141  (define-ccl-program ccl-decode-mule-utf-16-le  (define-ccl-program ccl-decode-mule-utf-16le
142    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
143      ,utf-16-le-decode-loop)      ,utf-16le-decode-loop)
144    "Decode UTF-16LE (little endian without signature bytes).    "Decode UTF-16LE (little endian without signature bytes).
145  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
146  mule-unicode-*.  Un-representable Unicode characters are decoded as  mule-unicode-*.  Un-representable Unicode characters are decoded as
147  U+fffd.  The result is run through the translation-table named  U+fffd.  The result is run through the translation-table named
148  `utf-translation-table-for-decode'.")  `utf-translation-table-for-decode'.")
149    
150  (define-ccl-program ccl-decode-mule-utf-16-be  (define-ccl-program ccl-decode-mule-utf-16be
151    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
152      ,utf-16-be-decode-loop)      ,utf-16be-decode-loop)
153    "Decode UTF-16BE (big endian without signature bytes).    "Decode UTF-16BE (big endian without signature bytes).
154  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
155  mule-unicode-*.  Un-representable Unicode characters are  mule-unicode-*.  Un-representable Unicode characters are
156  decoded as U+fffd.  The result is run through the translation-table of  decoded as U+fffd.  The result is run through the translation-table of
157  name `utf-translation-table-for-decode'.")  name `utf-translation-table-for-decode'.")
158    
159  (define-ccl-program ccl-decode-mule-utf-16-le-with-signature  (define-ccl-program ccl-decode-mule-utf-16le-with-signature
160    `(2    `(2
161      ((read r3 r4)      ((read r3 r4)
162       ,utf-16-le-decode-loop))       ,utf-16le-decode-loop))
163    "Like ccl-decode-utf-16-le but skip the first 2-byte BOM.")    "Like ccl-decode-utf-16le but skip the first 2-byte BOM.")
164    
165  (define-ccl-program ccl-decode-mule-utf-16-be-with-signature  (define-ccl-program ccl-decode-mule-utf-16be-with-signature
166    `(2    `(2
167      ((read r3 r4)      ((read r3 r4)
168       ,utf-16-be-decode-loop))       ,utf-16be-decode-loop))
169    "Like ccl-decode-utf-16-be but skip the first 2-byte BOM.")    "Like ccl-decode-utf-16be but skip the first 2-byte BOM.")
170    
171  (define-ccl-program ccl-decode-mule-utf-16  (define-ccl-program ccl-decode-mule-utf-16
172    `(2    `(2
# Line 172  name `utf-translation-table-for-decode'. Line 178  name `utf-translation-table-for-decode'.
178           ;; function.           ;; function.
179           (,@utf-16-decode-ucs           (,@utf-16-decode-ucs
180            (write-multibyte-character r0 r1)            (write-multibyte-character r0 r1)
181            ,utf-16-le-decode-loop)            ,utf-16le-decode-loop)
182         ((if (r1 == #xFEFF)         ((if (r1 == #xFEFF)
183              ;; R1 is a BOM for big endian, but we can't keep that              ;; R1 is a BOM for big endian, but we can't keep that
184              ;; character in the output because it can't be              ;; character in the output because it can't be
# Line 184  name `utf-translation-table-for-decode'. Line 190  name `utf-translation-table-for-decode'.
190            (,@utf-16-decode-ucs            (,@utf-16-decode-ucs
191             (translate-character utf-translation-table-for-decode r0 r1)))             (translate-character utf-translation-table-for-decode r0 r1)))
192          (write-multibyte-character r0 r1)          (write-multibyte-character r0 r1)
193          ,utf-16-be-decode-loop))))          ,utf-16be-decode-loop))))
194    "Like ccl-decode-utf-16-be/le but check the first BOM.")    "Like ccl-decode-utf-16be/le but check the first BOM.")
195    
196  (makunbound 'utf-16-decode-ucs)         ; done with it  (makunbound 'utf-16-decode-ucs)         ; done with it
197  (makunbound 'utf-16-le-decode-loop)  (makunbound 'utf-16le-decode-loop)
198  (makunbound 'utf-16-be-decode-loop)  (makunbound 'utf-16be-decode-loop)
199    
200  (eval-and-compile  (eval-and-compile
201  (defconst utf-16-decode-to-ucs  (defconst utf-16-decode-to-ucs
# Line 216  name `utf-translation-table-for-decode'. Line 222  name `utf-translation-table-for-decode'.
222                      (r0 = (r3 + #xe000))                      (r0 = (r3 + #xe000))
223                    (r0 = #xfffd))))))))))                    (r0 = #xfffd))))))))))
224    
225  (defconst utf-16-le-encode-loop  (defconst utf-16le-encode-loop
226    `(loop    `(loop
227      (read-multibyte-character r0 r1)      (read-multibyte-character r0 r1)
228      (lookup-character utf-subst-table-for-encode r0 r1)      (lookup-character utf-subst-table-for-encode r0 r1)
# Line 227  name `utf-translation-table-for-decode'. Line 233  name `utf-translation-table-for-decode'.
233      (write (r0 >> 8))      (write (r0 >> 8))
234      (repeat)))      (repeat)))
235    
236  (defconst utf-16-be-encode-loop  (defconst utf-16be-encode-loop
237    `(loop    `(loop
238      (read-multibyte-character r0 r1)      (read-multibyte-character r0 r1)
239      (lookup-character utf-subst-table-for-encode r0 r1)      (lookup-character utf-subst-table-for-encode r0 r1)
# Line 239  name `utf-translation-table-for-decode'. Line 245  name `utf-translation-table-for-decode'.
245      (repeat)))      (repeat)))
246  )  )
247    
248  (define-ccl-program ccl-encode-mule-utf-16-le  
249    (define-ccl-program ccl-encode-mule-utf-16le
250    `(1    `(1
251      ,utf-16-le-encode-loop)      ,utf-16le-encode-loop)
252    "Encode to UTF-16LE (little endian without signature).    "Encode to UTF-16LE (little endian without signature).
253  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
254  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 249  after translation through the translatio Line 256  after translation through the translatio
256  `utf-translation-table-for-encode'.  `utf-translation-table-for-encode'.
257  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
258    
259  (define-ccl-program ccl-encode-mule-utf-16-be  (define-ccl-program ccl-encode-mule-utf-16be
260    `(1    `(1
261      ,utf-16-be-encode-loop)      ,utf-16be-encode-loop)
262    "Encode to UTF-16BE (big endian without signature).    "Encode to UTF-16BE (big endian without signature).
263  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
264  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 259  after translation through the translatio Line 266  after translation through the translatio
266  `utf-translation-table-for-encode'.  `utf-translation-table-for-encode'.
267  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
268    
269  (define-ccl-program ccl-encode-mule-utf-16-le-with-signature  (define-ccl-program ccl-encode-mule-utf-16le-with-signature
270    `(1    `(1
271      ((write #xFF)      ((write #xFF)
272       (write #xFE)       (write #xFE)
273       ,utf-16-le-encode-loop))       ,utf-16le-encode-loop))
274    "Encode to UTF-16 (little endian with signature).    "Encode to UTF-16 (little endian with signature).
275  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
276  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 271  after translation through the translatio Line 278  after translation through the translatio
278  `utf-translation-table-for-encode'.  `utf-translation-table-for-encode'.
279  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
280    
281  (define-ccl-program ccl-encode-mule-utf-16-be-with-signature  (define-ccl-program ccl-encode-mule-utf-16be-with-signature
282    `(1    `(1
283      ((write #xFE)      ((write #xFE)
284       (write #xFF)       (write #xFF)
285       ,utf-16-be-encode-loop))       ,utf-16be-encode-loop))
286    "Encode to UTF-16 (big endian with signature).    "Encode to UTF-16 (big endian with signature).
287  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
288  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 284  after translation through the translatio Line 291  after translation through the translatio
291  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
292    
293  (makunbound 'utf-16-decode-to-ucs)  (makunbound 'utf-16-decode-to-ucs)
294  (makunbound 'utf-16-le-encode-loop)  (makunbound 'utf-16le-encode-loop)
295  (makunbound 'utf-16-be-encode-loop)  (makunbound 'utf-16be-encode-loop)
296    
297  (defun mule-utf-16-post-read-conversion (length)  (defun mule-utf-16-post-read-conversion (length)
298    (when (> length 0)    (when (> length 0)
# Line 295  Others are encoded as U+FFFD.") Line 302  Others are encoded as U+FFFD.")
302               (setq last-coding-system-used               (setq last-coding-system-used
303                     (coding-system-change-text-conversion                     (coding-system-change-text-conversion
304                      last-coding-system-used                      last-coding-system-used
305                      'mule-utf-16-le-with-signature))                      'mule-utf-16le-with-signature))
306               (setq length (1- length)))               (setq length (1- length)))
307              ((= char (decode-char 'ucs #xFFFF))              ((= char (decode-char 'ucs #xFFFF))
308               (delete-char 1)               (delete-char 1)
309               (setq last-coding-system-used               (setq last-coding-system-used
310                     (coding-system-change-text-conversion                     (coding-system-change-text-conversion
311                      last-coding-system-used                      last-coding-system-used
312                      'mule-utf-16-be-with-signature))                      'mule-utf-16be-with-signature))
313               (setq length (1- length)))               (setq length (1- length)))
314              (t              (t
315               (setq last-coding-system-used 'mule-utf-16-be)))))               (setq last-coding-system-used 'mule-utf-16be)))))
316    length)    length)
317    
318  (let ((doc "  (let ((doc "
# Line 324  On encoding (e.g. writing a file), Emacs Line 331  On encoding (e.g. writing a file), Emacs
331  any of the character sets listed above are encoded into the byte  any of the character sets listed above are encoded into the byte
332  sequence representing U+FFFD (REPLACEMENT CHARACTER)."))  sequence representing U+FFFD (REPLACEMENT CHARACTER)."))
333    (make-coding-system    (make-coding-system
334     'mule-utf-16-le 4     'mule-utf-16le 4
335     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
336     (concat     (concat
337      "Little endian UTF-16 encoding for Emacs-supported Unicode characters."      "UTF-16LE encoding for Emacs-supported Unicode characters."
338      doc)      doc)
339    
340     '(ccl-decode-mule-utf-16-le . ccl-encode-mule-utf-16-le)     '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le)
341     '((safe-charsets     '((safe-charsets
342        ascii        ascii
343        eight-bit-control        eight-bit-control
# Line 346  sequence representing U+FFFD (REPLACEMEN Line 353  sequence representing U+FFFD (REPLACEMEN
353                   utf-translate-cjk)))                   utf-translate-cjk)))
354    
355    (make-coding-system    (make-coding-system
356     'mule-utf-16-be 4 ?u     'mule-utf-16be 4 ?u
357     (concat     (concat
358      "Big endian UTF-16 encoding for Emacs-supported Unicode characters."      "UTF-16BE encoding for Emacs-supported Unicode characters."
359      doc)      doc)
360    
361     '(ccl-decode-mule-utf-16-be . ccl-encode-mule-utf-16-be)     '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be)
362     '((safe-charsets     '((safe-charsets
363        ascii        ascii
364        eight-bit-control        eight-bit-control
# Line 367  sequence representing U+FFFD (REPLACEMEN Line 374  sequence representing U+FFFD (REPLACEMEN
374                   utf-translate-cjk)))                   utf-translate-cjk)))
375    
376    (make-coding-system    (make-coding-system
377     'mule-utf-16-le-with-signature 4 ?u     'mule-utf-16le-with-signature 4 ?u
378     (concat     (concat
379      "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters."      "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
380      doc)      doc)
381    
382     '(ccl-decode-mule-utf-16-le-with-signature     '(ccl-decode-mule-utf-16le-with-signature
383       . ccl-encode-mule-utf-16-le-with-signature)       . ccl-encode-mule-utf-16le-with-signature)
384     '((safe-charsets     '((safe-charsets
385        ascii        ascii
386        eight-bit-control        eight-bit-control
# Line 390  sequence representing U+FFFD (REPLACEMEN Line 397  sequence representing U+FFFD (REPLACEMEN
397                   utf-translate-cjk)))                   utf-translate-cjk)))
398    
399    (make-coding-system    (make-coding-system
400     'mule-utf-16-be-with-signature 4 ?u     'mule-utf-16be-with-signature 4 ?u
401     (concat     (concat
402      "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters."      "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
403      doc)      doc)
404    
405     '(ccl-decode-mule-utf-16-be-with-signature     '(ccl-decode-mule-utf-16be-with-signature
406       . ccl-encode-mule-utf-16-be-with-signature)       . ccl-encode-mule-utf-16be-with-signature)
407     '((safe-charsets     '((safe-charsets
408        ascii        ascii
409        eight-bit-control        eight-bit-control
# Line 418  sequence representing U+FFFD (REPLACEMEN Line 425  sequence representing U+FFFD (REPLACEMEN
425      "UTF-16 (with or without BOM) for Emacs-supported Unicode characters."      "UTF-16 (with or without BOM) for Emacs-supported Unicode characters."
426      doc)      doc)
427    
428     '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16-be-with-signature)     '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature)
429     '((safe-charsets     '((safe-charsets
430        ascii        ascii
431        eight-bit-control        eight-bit-control
# Line 436  sequence representing U+FFFD (REPLACEMEN Line 443  sequence representing U+FFFD (REPLACEMEN
443       (post-read-conversion . mule-utf-16-post-read-conversion)))       (post-read-conversion . mule-utf-16-post-read-conversion)))
444  )  )
445    
446  (define-coding-system-alias 'utf-16-le 'mule-utf-16-le)  (define-coding-system-alias 'utf-16le 'mule-utf-16le)
447  (define-coding-system-alias 'utf-16-be 'mule-utf-16-be)  (define-coding-system-alias 'utf-16be 'mule-utf-16be)
448  (define-coding-system-alias 'utf-16-le-with-signature  (define-coding-system-alias 'utf-16le-with-signature
449    'mule-utf-16-le-with-signature)    'mule-utf-16le-with-signature)
450  (define-coding-system-alias 'utf-16-be-with-signature  (define-coding-system-alias 'utf-16be-with-signature
451    'mule-utf-16-be-with-signature)    'mule-utf-16be-with-signature)
452  (define-coding-system-alias 'utf-16 'mule-utf-16)  (define-coding-system-alias 'utf-16 'mule-utf-16)
453    
454    ;; For backward compatibility.
455    (define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature)
456    (define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature)
457    (define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature)
458    (define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature)
459    
460  ;;; utf-16.el ends here  ;;; utf-16.el ends here

Legend:
Removed from v.1.9  
changed lines
  Added in v.1.10

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26