/[emacs]/emacs/lisp/international/utf-16.el
ViewVC logotype

Diff of /emacs/lisp/international/utf-16.el

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.8.2.1 by miles, Fri Apr 4 06:20:26 2003 UTC revision 1.8.2.2 by miles, Tue Oct 14 23:39:23 2003 UTC
# Line 1  Line 1 
1  ;;; utf-16.el --- UTF-16 encoding/decoding  ;;; utf-16.el --- UTF-16 encoding/decoding
2    
3  ;; Copyright (C) 2001, 2002 Free Software Foundation, Inc.  ;; Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc.
4    
5  ;; Author: Dave Love <fx@gnu.org>  ;; Author: Dave Love <fx@gnu.org>
6  ;; Keywords: Unicode, UTF-16, i18n  ;; Keywords: Unicode, UTF-16, i18n
# Line 25  Line 25 
25  ;;; Commentary:  ;;; Commentary:
26    
27  ;; Support for UTF-16, which is a two-byte encoding (modulo  ;; Support for UTF-16, which is a two-byte encoding (modulo
28  ;; surrogates) of Unicode, written either in little or big endian  ;; surrogates) of Unicode, defined in RFC 2781.  It is written either
29  ;; order: coding-systems `mule-utf-16-le' and `mule-utf-16-be'.  ;; in little or big endian order and either with or without the
30  ;; (utf-16-le is used by the DozeN'T clipboard, for instance.)  The  ;; leading BOM (a two-byte signature which identifies their byte sex).
31  ;; data are preceeded by a two-byte signature which identifies their  ;;
32  ;; byte sex.  These are used by the coding-category-utf-16-{b,l}e code  ;; We provide these base coding systems.
33  ;; to identify the coding, but ignored on decoding.  ;;      name                                    endian  BOM
34    ;;      ----                                    ------  ---
35    ;;      mule-utf-16le                           little  no
36    ;;      mule-utf-16be                           big     no
37    ;;      mule-utf-16le-with-signature            little  yes
38    ;;      mule-utf-16be-with-signature            big     yes
39    ;;      mule-utf-16                             both    yes
40    ;;
41  ;; Note that un-decodable sequences aren't (yet?) preserved as raw  ;; Note that un-decodable sequences aren't (yet?) preserved as raw
42  ;; bytes, as they are with utf-8, so reading and writing as utf-16 can  ;; bytes, as they are with utf-8, so reading and writing as utf-16 can
43  ;; corrupt data.  ;; corrupt data.
# Line 69  Line 75 
75  (eval-and-compile  (eval-and-compile
76  (defconst utf-16-decode-ucs  (defconst utf-16-decode-ucs
77    ;; We have the unicode in r1.  Output is charset ID in r0, code    ;; We have the unicode in r1.  Output is charset ID in r0, code
78    ;; point in r1.  As r6 keeps endian information, the value should    ;; point in r1.
   ;; not be changed.  
79    `((lookup-integer utf-subst-table-for-decode r1 r3)    `((lookup-integer utf-subst-table-for-decode r1 r3)
80      (if r7                              ; got a translation      (if r7                              ; got a translation
81          ((r0 = r1) (r1 = r3))          ((r0 = r1) (r1 = r3))
# Line 111  Line 116 
116                       (r1 -= #xe000)                       (r1 -= #xe000)
117                       (r2 = (((r1 / 96) + 32) << 7))                       (r2 = (((r1 / 96) + 32) << 7))
118                       (r1 %= 96)                       (r1 %= 96)
119                       (r1 += (r2 + 32))))))))))))))                       (r1 += (r2 + 32)))))))))))))
120    
121    (defconst utf-16le-decode-loop
122      `(loop
123        (read r3 r4)
124        (r1 = (r4 <8 r3))
125        ,utf-16-decode-ucs
126        (translate-character utf-translation-table-for-decode r0 r1)
127        (write-multibyte-character r0 r1)
128        (repeat)))
129    
130    (defconst utf-16be-decode-loop
131      `(loop
132        (read r3 r4)
133        (r1 = (r3 <8 r4))
134        ,@utf-16-decode-ucs
135        (translate-character utf-translation-table-for-decode r0 r1)
136        (write-multibyte-character r0 r1)
137        (repeat)))
138    
139    )
140    
141  (define-ccl-program ccl-decode-mule-utf-16-le  (define-ccl-program ccl-decode-mule-utf-16le
142    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
143      ((loop      ,utf-16le-decode-loop)
       (read r3 r4)  
       (r1 = (r4 <8 r3))  
       ,utf-16-decode-ucs  
       (translate-character utf-translation-table-for-decode r0 r1)  
       (write-multibyte-character r0 r1)  
       (repeat))))  
144    "Decode UTF-16LE (little endian without signature bytes).    "Decode UTF-16LE (little endian without signature bytes).
145  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
146  mule-unicode-*.  Un-representable Unicode characters are decoded as  mule-unicode-*.  Un-representable Unicode characters are decoded as
147  U+fffd.  The result is run through the translation-table named  U+fffd.  The result is run through the translation-table named
148  `utf-translation-table-for-decode'.")  `utf-translation-table-for-decode'.")
149    
150  (define-ccl-program ccl-decode-mule-utf-16-be  (define-ccl-program ccl-decode-mule-utf-16be
151    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
152      ((loop      ,utf-16be-decode-loop)
       (read r3 r4)  
       (r1 = (r3 <8 r4))  
       ,utf-16-decode-ucs  
       (translate-character utf-translation-table-for-decode r0 r1)  
       (write-multibyte-character r0 r1)  
       (repeat))))  
153    "Decode UTF-16BE (big endian without signature bytes).    "Decode UTF-16BE (big endian without signature bytes).
154  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
155  mule-unicode-*.  Un-representable Unicode characters are  mule-unicode-*.  Un-representable Unicode characters are
156  decoded as U+fffd.  The result is run through the translation-table of  decoded as U+fffd.  The result is run through the translation-table of
157  name `utf-translation-table-for-decode'.")  name `utf-translation-table-for-decode'.")
158    
159    (define-ccl-program ccl-decode-mule-utf-16le-with-signature
160      `(2
161        ((read r3 r4)
162         ,utf-16le-decode-loop))
163      "Like ccl-decode-utf-16le but skip the first 2-byte BOM.")
164    
165    (define-ccl-program ccl-decode-mule-utf-16be-with-signature
166      `(2
167        ((read r3 r4)
168         ,utf-16be-decode-loop))
169      "Like ccl-decode-utf-16be but skip the first 2-byte BOM.")
170    
171    (define-ccl-program ccl-decode-mule-utf-16
172      `(2
173        ((read r3 r4)
174         (r1 = (r3 <8 r4))
175         (if (r1 == #xFFFE)
176             ;; R1 is a BOM for little endian.  We keep this character as
177             ;; is temporarily.  It is removed by post-read-conversion
178             ;; function.
179             (,@utf-16-decode-ucs
180              (write-multibyte-character r0 r1)
181              ,utf-16le-decode-loop)
182           ((if (r1 == #xFEFF)
183                ;; R1 is a BOM for big endian, but we can't keep that
184                ;; character in the output because it can't be
185                ;; distinguished with the normal U+FEFF.  So, we keep
186                ;; #xFFFF instead.
187                ((r1 = #xFFFF)
188                 ,@utf-16-decode-ucs)
189              ;; R1 a normal Unicode character.
190              (,@utf-16-decode-ucs
191               (translate-character utf-translation-table-for-decode r0 r1)))
192            (write-multibyte-character r0 r1)
193            ,utf-16be-decode-loop))))
194      "Like ccl-decode-utf-16be/le but check the first BOM.")
195    
196  (makunbound 'utf-16-decode-ucs)         ; done with it  (makunbound 'utf-16-decode-ucs)         ; done with it
197    (makunbound 'utf-16le-decode-loop)
198    (makunbound 'utf-16be-decode-loop)
199    
200  (eval-and-compile  (eval-and-compile
201  (defconst utf-16-decode-to-ucs  (defconst utf-16-decode-to-ucs
# Line 168  name `utf-translation-table-for-decode'. Line 220  name `utf-translation-table-for-decode'.
220                    (r0 = (r3 + #x2500))                    (r0 = (r3 + #x2500))
221                  (if (r0 == ,(charset-id 'mule-unicode-e000-ffff))                  (if (r0 == ,(charset-id 'mule-unicode-e000-ffff))
222                      (r0 = (r3 + #xe000))                      (r0 = (r3 + #xe000))
223                    (r0 = #xfffd)))))))))))                    (r0 = #xfffd))))))))))
224    
225    (defconst utf-16le-encode-loop
226      `(loop
227        (read-multibyte-character r0 r1)
228        (lookup-character utf-subst-table-for-encode r0 r1)
229        (if (r7 == 0)
230            ((translate-character utf-translation-table-for-encode r0 r1)
231             ,utf-16-decode-to-ucs))
232        (write (r0 & 255))
233        (write (r0 >> 8))
234        (repeat)))
235    
236    (defconst utf-16be-encode-loop
237      `(loop
238        (read-multibyte-character r0 r1)
239        (lookup-character utf-subst-table-for-encode r0 r1)
240        (if (r7 == 0)
241            ((translate-character utf-translation-table-for-encode r0 r1)
242             ,utf-16-decode-to-ucs))
243        (write (r0 >> 8))
244        (write (r0 & 255))
245        (repeat)))
246    )
247    
248  (define-ccl-program ccl-encode-mule-utf-16-le  
249    (define-ccl-program ccl-encode-mule-utf-16le
250    `(1    `(1
251      ((loop      ,utf-16le-encode-loop)
       (read-multibyte-character r0 r1)  
       (lookup-character utf-subst-table-for-encode r0 r1)  
       (if (r7 == 0)  
           ((translate-character utf-translation-table-for-encode r0 r1)  
            ,utf-16-decode-to-ucs))  
       (write (r0 & 255))  
       (write (r0 >> 8))  
       (repeat))))  
252    "Encode to UTF-16LE (little endian without signature).    "Encode to UTF-16LE (little endian without signature).
253  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
254  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 188  after translation through the translatio Line 256  after translation through the translatio
256  `utf-translation-table-for-encode'.  `utf-translation-table-for-encode'.
257  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
258    
259  (define-ccl-program ccl-encode-mule-utf-16-be  (define-ccl-program ccl-encode-mule-utf-16be
260    `(1    `(1
261      ((loop      ,utf-16be-encode-loop)
       (read-multibyte-character r0 r1)  
       (lookup-character utf-subst-table-for-encode r0 r1)  
       (if (r7 == 0)  
           ((translate-character utf-translation-table-for-encode r0 r1)  
            ,utf-16-decode-to-ucs))  
       (write (r0 >> 8))  
       (write (r0 & 255))  
       (repeat))))  
262    "Encode to UTF-16BE (big endian without signature).    "Encode to UTF-16BE (big endian without signature).
263  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
264  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
# Line 206  after translation through the translatio Line 266  after translation through the translatio
266  `utf-translation-table-for-encode'.  `utf-translation-table-for-encode'.
267  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
268    
269    (define-ccl-program ccl-encode-mule-utf-16le-with-signature
270      `(1
271        ((write #xFF)
272         (write #xFE)
273         ,utf-16le-encode-loop))
274      "Encode to UTF-16 (little endian with signature).
275    Characters from the charsets ascii, eight-bit-control,
276    eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
277    after translation through the translation-table of name
278    `utf-translation-table-for-encode'.
279    Others are encoded as U+FFFD.")
280    
281    (define-ccl-program ccl-encode-mule-utf-16be-with-signature
282      `(1
283        ((write #xFE)
284         (write #xFF)
285         ,utf-16be-encode-loop))
286      "Encode to UTF-16 (big endian with signature).
287    Characters from the charsets ascii, eight-bit-control,
288    eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
289    after translation through the translation-table named
290    `utf-translation-table-for-encode'.
291    Others are encoded as U+FFFD.")
292    
293  (makunbound 'utf-16-decode-to-ucs)  (makunbound 'utf-16-decode-to-ucs)
294    (makunbound 'utf-16le-encode-loop)
295    (makunbound 'utf-16be-encode-loop)
296    
297    (defun mule-utf-16-post-read-conversion (length)
298      (when (> length 0)
299        (let ((char (following-char)))
300          (cond ((= char (decode-char 'ucs #xFFFE))
301                 (delete-char 1)
302                 (setq last-coding-system-used
303                       (coding-system-change-text-conversion
304                        last-coding-system-used
305                        'mule-utf-16le-with-signature))
306                 (setq length (1- length)))
307                ((= char (decode-char 'ucs #xFFFF))
308                 (delete-char 1)
309                 (setq last-coding-system-used
310                       (coding-system-change-text-conversion
311                        last-coding-system-used
312                        'mule-utf-16be-with-signature))
313                 (setq length (1- length)))
314                (t
315                 (setq last-coding-system-used 'mule-utf-16be)))))
316      length)
317    
318  (let ((doc "  (let ((doc "
319    
# Line 224  On encoding (e.g. writing a file), Emacs Line 331  On encoding (e.g. writing a file), Emacs
331  any of the character sets listed above are encoded into the byte  any of the character sets listed above are encoded into the byte
332  sequence representing U+FFFD (REPLACEMENT CHARACTER)."))  sequence representing U+FFFD (REPLACEMENT CHARACTER)."))
333    (make-coding-system    (make-coding-system
334     'mule-utf-16-le 4     'mule-utf-16le 4
335     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
336     (concat     (concat
337      "Little endian UTF-16 encoding for Emacs-supported Unicode characters."      "UTF-16LE encoding for Emacs-supported Unicode characters."
338      doc)      doc)
339    
340     '(ccl-decode-mule-utf-16-le . ccl-encode-mule-utf-16-le)     '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le)
341     '((safe-charsets     '((safe-charsets
342        ascii        ascii
343        eight-bit-control        eight-bit-control
# Line 239  sequence representing U+FFFD (REPLACEMEN Line 346  sequence representing U+FFFD (REPLACEMEN
346        mule-unicode-2500-33ff        mule-unicode-2500-33ff
347        mule-unicode-e000-ffff)        mule-unicode-e000-ffff)
348       (mime-charset . utf-16le)       (mime-charset . utf-16le)
349       (coding-category . coding-category-utf-16-le)       (mime-text-unsuitable . t)
350       (valid-codes (0 . 255))       (valid-codes (0 . 255))
351       (dependency unify-8859-on-encoding-mode       (dependency unify-8859-on-encoding-mode
352                   unify-8859-on-decoding-mode                   unify-8859-on-decoding-mode
# Line 247  sequence representing U+FFFD (REPLACEMEN Line 354  sequence representing U+FFFD (REPLACEMEN
354                   utf-translate-cjk)))                   utf-translate-cjk)))
355    
356    (make-coding-system    (make-coding-system
357     'mule-utf-16-be 4 ?u     'mule-utf-16be 4 ?u
358     (concat     (concat
359      "Big endian UTF-16 encoding for Emacs-supported Unicode characters."      "UTF-16BE encoding for Emacs-supported Unicode characters."
360      doc)      doc)
361    
362     '(ccl-decode-mule-utf-16-be . ccl-encode-mule-utf-16-be)     '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be)
363     '((safe-charsets     '((safe-charsets
364        ascii        ascii
365        eight-bit-control        eight-bit-control
# Line 261  sequence representing U+FFFD (REPLACEMEN Line 368  sequence representing U+FFFD (REPLACEMEN
368        mule-unicode-2500-33ff        mule-unicode-2500-33ff
369        mule-unicode-e000-ffff)        mule-unicode-e000-ffff)
370       (mime-charset . utf-16be)       (mime-charset . utf-16be)
371         (valid-codes (0 . 255))
372         (dependency unify-8859-on-encoding-mode
373                     unify-8859-on-decoding-mode
374                     utf-fragment-on-decoding
375                     utf-translate-cjk)))
376    
377      (make-coding-system
378       'mule-utf-16le-with-signature 4 ?u
379       (concat
380        "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
381        doc)
382    
383       '(ccl-decode-mule-utf-16le-with-signature
384         . ccl-encode-mule-utf-16le-with-signature)
385       '((safe-charsets
386          ascii
387          eight-bit-control
388          latin-iso8859-1
389          mule-unicode-0100-24ff
390          mule-unicode-2500-33ff
391          mule-unicode-e000-ffff)
392         (coding-category . coding-category-utf-16-le)
393         (mime-charset . utf-16)
394         (mime-text-unsuitable . t)
395         (valid-codes (0 . 255))
396         (dependency unify-8859-on-encoding-mode
397                     unify-8859-on-decoding-mode
398                     utf-fragment-on-decoding
399                     utf-translate-cjk)))
400    
401      (make-coding-system
402       'mule-utf-16be-with-signature 4 ?u
403       (concat
404        "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
405        doc)
406    
407       '(ccl-decode-mule-utf-16be-with-signature
408         . ccl-encode-mule-utf-16be-with-signature)
409       '((safe-charsets
410          ascii
411          eight-bit-control
412          latin-iso8859-1
413          mule-unicode-0100-24ff
414          mule-unicode-2500-33ff
415          mule-unicode-e000-ffff)
416       (coding-category . coding-category-utf-16-be)       (coding-category . coding-category-utf-16-be)
417         (mime-charset . utf-16)
418       (valid-codes (0 . 255))       (valid-codes (0 . 255))
419       (dependency unify-8859-on-encoding-mode       (dependency unify-8859-on-encoding-mode
420                   unify-8859-on-decoding-mode                   unify-8859-on-decoding-mode
421                   utf-fragment-on-decoding                   utf-fragment-on-decoding
422                   utf-translate-cjk))))                   utf-translate-cjk)))
423    
424  (define-coding-system-alias 'utf-16-le 'mule-utf-16-le)    (make-coding-system
425  (define-coding-system-alias 'utf-16-be 'mule-utf-16-be)     'mule-utf-16 4 ?u
426       (concat
427        "UTF-16 (with or without BOM) for Emacs-supported Unicode characters."
428        doc)
429    
430       '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature)
431       '((safe-charsets
432          ascii
433          eight-bit-control
434          latin-iso8859-1
435          mule-unicode-0100-24ff
436          mule-unicode-2500-33ff
437          mule-unicode-e000-ffff)
438         (coding-category . coding-category-utf-16-be)
439         (mime-charset . utf-16)
440         (mime-text-unsuitable . t)
441         (valid-codes (0 . 255))
442         (dependency unify-8859-on-encoding-mode
443                     unify-8859-on-decoding-mode
444                     utf-fragment-on-decoding
445                     utf-translate-cjk)
446         (post-read-conversion . mule-utf-16-post-read-conversion)))
447    )
448    
449    (define-coding-system-alias 'utf-16le 'mule-utf-16le)
450    (define-coding-system-alias 'utf-16be 'mule-utf-16be)
451    (define-coding-system-alias 'utf-16le-with-signature
452      'mule-utf-16le-with-signature)
453    (define-coding-system-alias 'utf-16be-with-signature
454      'mule-utf-16be-with-signature)
455    (define-coding-system-alias 'utf-16 'mule-utf-16)
456    
457    ;; For backward compatibility.
458    (define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature)
459    (define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature)
460    (define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature)
461    (define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature)
462    
463    ;;; arch-tag: 85455d46-d9c9-466d-a6f3-c3582a7367c4
464  ;;; utf-16.el ends here  ;;; utf-16.el ends here

Legend:
Removed from v.1.8.2.1  
changed lines
  Added in v.1.8.2.2

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26