/[emacs]/emacs/lisp/international/utf-16.el
ViewVC logotype

Diff of /emacs/lisp/international/utf-16.el

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.3 by fx, Tue Sep 10 17:37:38 2002 UTC revision 1.4 by handa, Mon Sep 30 06:37:00 2002 UTC
# Line 68  Line 68 
68  ;; Needed in macro expansion, so can't be let-bound.  Zapped after use.  ;; Needed in macro expansion, so can't be let-bound.  Zapped after use.
69  (eval-and-compile  (eval-and-compile
70  (defconst utf-16-decode-ucs  (defconst utf-16-decode-ucs
71    ;; We have the unicode in r1.  Output is character codes in r0, r1,    ;; We have the unicode in r1.  Output is charset ID in r0, code point
72    ;; and r2 if appropriate.    ;; in r1.
73    `((lookup-integer utf-8-subst-table r0 r3)    `((lookup-integer utf-subst-table-for-decode r1 r3)
74      (if r7 (r1 = r3))                   ; got a translation      (if r7                              ; got a translation
75      (if (r1 < 128)          ((r0 = r1) (r1 = r3))
76         (r0 = ,(charset-id 'ascii))        (if (r1 < 128)
77       (if (r1 < 160)            (r0 = ,(charset-id 'ascii))
78           (r0 = ,(charset-id 'eight-bit-control))          (if (r1 < 160)
79         (if (r1 < 256)              (r0 = ,(charset-id 'eight-bit-control))
80             ((r0 = ,(charset-id 'latin-iso8859-1))            (if (r1 < 256)
81              (r1 -= 128))                ((r0 = ,(charset-id 'latin-iso8859-1))
82           (if (r1 < #x2500)                 (r1 -= 128))
83               ((r0 = ,(charset-id 'mule-unicode-0100-24ff))              (if (r1 < #x2500)
84                (r1 -= #x100)                  ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
85                (r2 = (((r1 / 96) + 32) << 7))                   (r1 -= #x100)
86                (r1 %= 96)                   (r2 = (((r1 / 96) + 32) << 7))
87                (r1 += (r2 + 32)))                   (r1 %= 96)
88             (if (r1 < #x3400)                   (r1 += (r2 + 32)))
89                 ((r0 = ,(charset-id 'mule-unicode-2500-33ff))                (if (r1 < #x3400)
90                  (r1 -= #x2500)                    ((r0 = ,(charset-id 'mule-unicode-2500-33ff))
91                  (r2 = (((r1 / 96) + 32) << 7))                     (r1 -= #x2500)
92                  (r1 %= 96)                     (r2 = (((r1 / 96) + 32) << 7))
93                  (r1 += (r2 + 32)))                     (r1 %= 96)
94               (if (r1 < #xd800)          ; 2 untranslated bytes                     (r1 += (r2 + 32)))
95  ;;               ;; Assume this is rare, so don't worry about the                  (if (r1 < #xd800)       ; 2 untranslated bytes
96  ;;               ;; overhead of the call.                      ;;           ;; Assume this is rare, so don't worry about the
97  ;;               (call mule-utf-16-untrans)                      ;;           ;; overhead of the call.
98                   ((r0 = ,(charset-id 'mule-unicode-e000-ffff))                      ;;           (call mule-utf-16-untrans)
99                    (r1 = 15037))         ; U+fffd                      ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
100                 (if (r1 < #xe000)        ; surrogate                       (r1 = 15037))      ; U+fffd
101  ;;                      ((call mule-utf-16-untrans)                    (if (r1 < #xe000)     ; surrogate
102  ;;                       (write-multibyte-character r0 r1)                        ;;                        ((call mule-utf-16-untrans)
103  ;;                       (read r3 r4)                        ;;                         (write-multibyte-character r0 r1)
104  ;;                       (call mule-utf-16-untrans))                        ;;                         (read r3 r4)
105                     ((read r3 r4)                        ;;                         (call mule-utf-16-untrans))
106                      (r0 = ,(charset-id 'mule-unicode-e000-ffff))                        ((read r3 r4)
107                      (r1 = 15037))                         (r0 = ,(charset-id 'mule-unicode-e000-ffff))
108                   ((r0 = ,(charset-id 'mule-unicode-e000-ffff))                         (r1 = 15037))
109                    (r1 -= #xe000)                      ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
110                    (r2 = (((r1 / 96) + 32) << 7))                       (r1 -= #xe000)
111                    (r1 %= 96)                       (r2 = (((r1 / 96) + 32) << 7))
112                    (r1 += (r2 + 32)))))))))))))                       (r1 %= 96)
113                         (r1 += (r2 + 32))))))))))))))
114    
115  (define-ccl-program ccl-decode-mule-utf-16-le  (define-ccl-program ccl-decode-mule-utf-16-le
116    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
# Line 118  Line 119 
119        (read r3 r4)        (read r3 r4)
120        (r1 = (r4 <8 r3))        (r1 = (r4 <8 r3))
121        ,utf-16-decode-ucs        ,utf-16-decode-ucs
122        (translate-character utf-8-translation-table-for-decode r0 r1)        (translate-character utf-translation-table-for-decode r0 r1)
123        (write-multibyte-character r0 r1)        (write-multibyte-character r0 r1)
124        (repeat))))        (repeat))))
125    "Decode little endian UTF-16 (ignoring signature bytes).    "Decode little endian UTF-16 (ignoring signature bytes).
126  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
127  mule-unicode-*.  Un-representable Unicode characters are  mule-unicode-*.  Un-representable Unicode characters are decoded as
128  decoded as U+fffd.  The result is run through translation table  U+fffd.  The result is run through the translation-table named
129  `utf-8-translation-table-for-decode' if that is defined.")  `utf-translation-table-for-decode'.")
130    
131  (define-ccl-program ccl-decode-mule-utf-16-be  (define-ccl-program ccl-decode-mule-utf-16-be
132    `(2                                   ; 2 bytes -> 1 to 4 bytes    `(2                                   ; 2 bytes -> 1 to 4 bytes
# Line 134  decoded as U+fffd.  The result is run th Line 135  decoded as U+fffd.  The result is run th
135        (read r3 r4)        (read r3 r4)
136        (r1 = (r3 <8 r4))        (r1 = (r3 <8 r4))
137        ,utf-16-decode-ucs        ,utf-16-decode-ucs
138        (translate-character utf-8-translation-table-for-decode r0 r1)        (translate-character utf-translation-table-for-decode r0 r1)
139        (write-multibyte-character r0 r1)        (write-multibyte-character r0 r1)
140        (repeat))))        (repeat))))
141    "Decode big endian UTF-16 (ignoring signature bytes).    "Decode big endian UTF-16 (ignoring signature bytes).
142  Basic decoding is done into the charsets ascii, latin-iso8859-1 and  Basic decoding is done into the charsets ascii, latin-iso8859-1 and
143  mule-unicode-*.  Un-representable Unicode characters are  mule-unicode-*.  Un-representable Unicode characters are
144  decoded as U+fffd.  The result is run through translation table  decoded as U+fffd.  The result is run through the translation-table of
145  `utf-8-non-latin-8859-table'.")  name `utf-translation-table-for-decode'.")
146    
147  (makunbound 'utf-16-decode-ucs)         ; done with it  (makunbound 'utf-16-decode-ucs)         ; done with it
148    
# Line 176  decoded as U+fffd.  The result is run th Line 177  decoded as U+fffd.  The result is run th
177       (write #xfe)       (write #xfe)
178       (loop       (loop
179        (read-multibyte-character r0 r1)        (read-multibyte-character r0 r1)
180        (translate-character ucs-mule-to-mule-unicode r0 r1)        (lookup-character utf-subst-table-for-encode r0 r1)
181        ,utf-16-decode-to-ucs        (if (r7 == 0)
182              ((translate-character utf-translation-table-for-encode r0 r1)
183               ,utf-16-decode-to-ucs))
184        (write (r0 & 255))        (write (r0 & 255))
185        (write (r0 >> 8))        (write (r0 >> 8))
186        (repeat))))        (repeat))))
187    "Encode to little endian UTF-16 with signature.    "Encode to little endian UTF-16 with signature.
188  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
189  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
190  after translation through the table `ucs-mule-to-mule-unicode'.  after translation through the translation-table of name
191    `utf-translation-table-for-encode'.
192  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
193    
194  (define-ccl-program ccl-encode-mule-utf-16-be  (define-ccl-program ccl-encode-mule-utf-16-be
# Line 193  Others are encoded as U+FFFD.") Line 197  Others are encoded as U+FFFD.")
197       (write #xff)       (write #xff)
198       (loop       (loop
199        (read-multibyte-character r0 r1)        (read-multibyte-character r0 r1)
200        (translate-character ucs-mule-to-mule-unicode r0 r1)        (lookup-character utf-subst-table-for-encode r0 r1)
201        ,utf-16-decode-to-ucs        (if (r7 == 0)
202              ((translate-character utf-translation-table-for-encode r0 r1)
203               ,utf-16-decode-to-ucs))
204        (write (r0 >> 8))        (write (r0 >> 8))
205        (write (r0 & 255))        (write (r0 & 255))
206        (repeat))))        (repeat))))
207    "Encode to big endian UTF-16 with signature.    "Encode to big endian UTF-16 with signature.
208  Characters from the charsets ascii, eight-bit-control,  Characters from the charsets ascii, eight-bit-control,
209  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded  eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
210  after translation through the table `ucs-mule-to-mule-unicode'.  after translation through the translation-table named
211    `utf-translation-table-for-encode'.
212  Others are encoded as U+FFFD.")  Others are encoded as U+FFFD.")
213    
214  (makunbound 'utf-16-decode-to-ucs)  (makunbound 'utf-16-decode-to-ucs)
# Line 210  Others are encoded as U+FFFD.") Line 217  Others are encoded as U+FFFD.")
217    
218  Assumes and ignores the leading two-byte signature.  Assumes and ignores the leading two-byte signature.
219    
220  The supported Emacs character sets are the following, plus others  It supports Unicode characters of these ranges:
221  which may be included in the translation table      U+0000..U+33FF, U+E000..U+FFFF.
222  `ucs-mule-to-mule-unicode':  They correspond to these Emacs character sets:
223   ascii      ascii, latin-iso8859-1, mule-unicode-0100-24ff,
224   eight-bit-control      mule-unicode-2500-33ff, mule-unicode-e000-ffff
225   latin-iso8859-1  
226   mule-unicode-0100-24ff  On decoding (e.g. reading a file), Unicode characters not in the above
227   mule-unicode-2500-33ff  ranges are decoded as U+FFFD, effectively corrupting the data
228   mule-unicode-e000-ffff  if they are re-encoded.  
229    
230  Note that Unicode characters out of the ranges U+0000-U+33FF and  On encoding (e.g. writing a file), Emacs characters not belonging to
231  U+E200-U+FFFF are decoded as U+FFFD, effectively corrupting the data  any of the character sets listed above are encoded into the byte
232  if they are re-encoded.  Emacs characters without Unicode conversions  sequence representing U+FFFD (REPLACEMENT CHARACTER)."))
 are encoded as U+FFFD."))  
233    (make-coding-system    (make-coding-system
234     'mule-utf-16-le 4     'mule-utf-16-le 4
235     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.     ?u         ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
# Line 242  are encoded as U+FFFD.")) Line 248  are encoded as U+FFFD."))
248       (mime-charset . utf-16le)       (mime-charset . utf-16le)
249       (coding-category . coding-category-utf-16-le)       (coding-category . coding-category-utf-16-le)
250       (valid-codes (0 . 255))       (valid-codes (0 . 255))
251       (pre-write-conversion . utf-16-le-pre-write-conversion)))       (pre-write-conversion . utf-16-le-pre-write-conversion)
252         (dependency unify-8859-on-encoding-mode
253                     unify-8859-on-decoding-mode
254                     utf-fragment-on-decoding
255                     utf-translate-cjk)))
256    
257    (make-coding-system    (make-coding-system
258     'mule-utf-16-be 4 ?u     'mule-utf-16-be 4 ?u
# Line 261  are encoded as U+FFFD.")) Line 271  are encoded as U+FFFD."))
271       (mime-charset . utf-16be)       (mime-charset . utf-16be)
272       (coding-category . coding-category-utf-16-be)       (coding-category . coding-category-utf-16-be)
273       (valid-codes (0 . 255))       (valid-codes (0 . 255))
274       (pre-write-conversion . utf-16-be-pre-write-conversion)))       (pre-write-conversion . utf-16-be-pre-write-conversion)
275         (dependency unify-8859-on-encoding-mode
276    (register-char-codings 'mule-utf-16-le ucs-mule-to-mule-unicode)                   unify-8859-on-decoding-mode
277    (register-char-codings 'mule-utf-16-be ucs-mule-to-mule-unicode))                   utf-fragment-on-decoding
278                     utf-translate-cjk))))
279    
280  (define-coding-system-alias 'utf-16-le 'mule-utf-16-le)  (define-coding-system-alias 'utf-16-le 'mule-utf-16-le)
281  (define-coding-system-alias 'utf-16-be 'mule-utf-16-be)  (define-coding-system-alias 'utf-16-be 'mule-utf-16-be)

Legend:
Removed from v.1.3  
changed lines
  Added in v.1.4

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26