43 |
# Inspired by code from Jochen Hoenicke. |
# Inspired by code from Jochen Hoenicke. |
44 |
# author Eric Blake <ebb9@email.byu.edu> |
# author Eric Blake <ebb9@email.byu.edu> |
45 |
# |
# |
46 |
# Usage: ./unicode-muncher <UnicodeData.txt> <CharData.java> |
# Usage: ./unicode-muncher <UnicodeData> <SpecialCasing> <CharData.java> |
47 |
# where <UnicodeData.txt> is obtained from www.unicode.org (named |
# where <UnicodeData> and <SpecialCasing> are .txt files obtained from |
48 |
# UnicodeData-3.0.0.txt for Unicode version 3.0.0), and <CharData.java> |
# www.unicode.org (named UnicodeData-3.0.0.txt and SpecialCasing-2.txt for |
49 |
# is the final location for the Java interface gnu.java.lang.CharData. |
# Unicode version 3.0.0), and <CharData.java> is the final location for the |
50 |
|
# Java interface gnu.java.lang.CharData. |
51 |
# As of JDK 1.4, use Unicode version 3.0.0 for best results. |
# As of JDK 1.4, use Unicode version 3.0.0 for best results. |
52 |
|
|
53 |
## |
## |
76 |
my $NOBREAK_FLAG = 32; |
my $NOBREAK_FLAG = 32; |
77 |
my $MIRRORED_FLAG = 64; |
my $MIRRORED_FLAG = 64; |
78 |
|
|
79 |
|
my %special = (); |
80 |
my @info = (); |
my @info = (); |
81 |
my $titlecase = ""; |
my $titlecase = ""; |
82 |
my $count = 0; |
my $count = 0; |
83 |
my $range = 0; |
my $range = 0; |
84 |
|
|
85 |
die "Usage: $0 <UnicodeData.txt> <CharData.java>" unless @ARGV == 2; |
die "Usage: $0 <UnicodeData.txt> <SpecialCasing.txt> <CharData.java>" |
86 |
open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n"; |
unless @ARGV == 3; |
|
|
|
|
# Stage 1: Parse the attribute file |
|
87 |
$| = 1; |
$| = 1; |
88 |
print "GNU Classpath Unicode Attribute Database Generator 2.0\n"; |
print "GNU Classpath Unicode Attribute Database Generator 2.0\n"; |
89 |
print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n"; |
print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n"; |
90 |
|
|
91 |
|
# Stage 0: Parse the special casing file |
92 |
|
print "Parsing special casing file\n"; |
93 |
|
open (SPECIAL, "< $ARGV[1]") || die "Can't open special casing file: $!\n"; |
94 |
|
while (<SPECIAL>) { |
95 |
|
next if /^\#/; |
96 |
|
my ($ch, undef, undef, $upper) = split / *; */; |
97 |
|
|
98 |
|
# This grabs only the special casing for multi-char uppercase. Note that |
99 |
|
# there are no multi-char lowercase, and that Sun ignores multi-char |
100 |
|
# titlecase rules. This script omits 3 special cases in Unicode 3.0.0, |
101 |
|
# which must be hardcoded in java.lang.String: |
102 |
|
# \u03a3 (Sun ignores this special case) |
103 |
|
# \u0049 - lowercases to \u0131, but only in Turkish locale |
104 |
|
# \u0069 - uppercases to \u0130, but only in Turkish locale |
105 |
|
next unless defined $upper and $upper =~ / /; |
106 |
|
$special{hex $ch} = [map {hex} split ' ', $upper]; |
107 |
|
} |
108 |
|
|
109 |
|
close SPECIAL; |
110 |
|
|
111 |
|
# Stage 1: Parse the attribute file |
112 |
print "Parsing attributes file"; |
print "Parsing attributes file"; |
113 |
while(<UNICODE>) { |
open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n"; |
114 |
|
while (<UNICODE>) { |
115 |
print "." unless $count++ % 1000; |
print "." unless $count++ % 1000; |
116 |
chomp; |
chomp; |
117 |
s/\r//g; |
s/\r//g; |
165 |
last; |
last; |
166 |
} |
} |
167 |
} |
} |
168 |
|
$direction <<= 2; |
169 |
|
$direction += $#{$special{$ch}} if defined $special{$ch}; |
170 |
|
|
171 |
if ($range) { |
if ($range) { |
172 |
die "Expecting end of range at $ch\n" unless $name =~ /Last>$/; |
die "Expecting end of range at $ch\n" unless $name =~ /Last>$/; |
192 |
|
|
193 |
for my $ch (0 .. 0xffff) { |
for my $ch (0 .. 0xffff) { |
194 |
print "." unless $count++ % 0x1000; |
print "." unless $count++ % 0x1000; |
195 |
if (! defined $info[$ch]) { |
$info[$ch] = pack("n5", 0, -1, 0, 0, -4) unless defined $info[$ch]; |
|
$info[$ch] = pack("n5", 0, -1, 0, 0, -1); |
|
|
} |
|
196 |
|
|
197 |
my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]); |
my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]); |
198 |
if (! exists $charhash{$info[$ch]}) { |
if (! exists $charhash{$info[$ch]}) { |
232 |
for ($j = $blksize - 1; $j > 0; $j--) { |
for ($j = $blksize - 1; $j > 0; $j--) { |
233 |
my %tails = (); |
my %tails = (); |
234 |
for $k (0 .. $#blkarray) { |
for $k (0 .. $#blkarray) { |
235 |
next if ! defined $blkarray[$k]; |
next unless defined $blkarray[$k]; |
236 |
my $len = length $blkarray[$k]; |
my $len = length $blkarray[$k]; |
237 |
my $tail = substr $blkarray[$k], $len - $j * 2; |
my $tail = substr $blkarray[$k], $len - $j * 2; |
238 |
if (exists $tails{$tail}) { |
if (exists $tails{$tail}) { |
245 |
# tails are calculated, now calculate the heads and merge. |
# tails are calculated, now calculate the heads and merge. |
246 |
BLOCK: |
BLOCK: |
247 |
for $k (0 .. $#blkarray) { |
for $k (0 .. $#blkarray) { |
248 |
next if ! defined $blkarray[$k]; |
next unless defined $blkarray[$k]; |
249 |
my $tomerge = $k; |
my $tomerge = $k; |
250 |
while (1) { |
while (1) { |
251 |
my $head = substr($blkarray[$tomerge], 0, $j * 2); |
my $head = substr($blkarray[$tomerge], 0, $j * 2); |
252 |
my $entry = $tails{$head}; |
my $entry = $tails{$head}; |
253 |
next BLOCK if ! defined $entry; |
next BLOCK unless defined $entry; |
254 |
|
|
255 |
my $other = shift @{$entry}; |
my $other = shift @{$entry}; |
256 |
if ($other == $tomerge) { |
if ($other == $tomerge) { |
320 |
die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n" |
die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n" |
321 |
if length($bestblkstr) > 0xffff / 3; |
if length($bestblkstr) > 0xffff / 3; |
322 |
{ |
{ |
323 |
print "Generating $ARGV[1] with shift of $bestshift"; |
print "Generating $ARGV[2] with shift of $bestshift"; |
324 |
my ($i, $j); |
my ($i, $j); |
325 |
|
|
326 |
open OUTPUT, "> $ARGV[1]" or die "Failed creating output file: $!\n"; |
open OUTPUT, "> $ARGV[2]" or die "Failed creating output file: $!\n"; |
327 |
print OUTPUT <<EOF; |
print OUTPUT <<EOF; |
328 |
/* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info |
/* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info |
329 |
Copyright (C) 2002 Free Software Foundation, Inc. |
Copyright (C) 2002 Free Software Foundation, Inc. |
368 |
/** |
/** |
369 |
* This contains the info about the unicode characters, that |
* This contains the info about the unicode characters, that |
370 |
* java.lang.Character needs. It is generated automatically from |
* java.lang.Character needs. It is generated automatically from |
371 |
* <code>$ARGV[0]</code>, by some perl scripts. |
* <code>$ARGV[0]</code> and |
372 |
* This Unicode definition file can be found on the |
* <code>$ARGV[1]</code>, by some |
373 |
|
* perl scripts. These Unicode definition file can be found on the |
374 |
* <a href="http://www.unicode.org">http://www.unicode.org</a> website. |
* <a href="http://www.unicode.org">http://www.unicode.org</a> website. |
375 |
* JDK 1.4 uses Unicode version 3.0.0. |
* JDK 1.4 uses Unicode version 3.0.0. |
376 |
* |
* |
382 |
* into the attribute tables <code>UPPER</code>, <code>LOWER</code>, |
* into the attribute tables <code>UPPER</code>, <code>LOWER</code>, |
383 |
* <code>NUM_VALUE</code>, and <code>DIRECTION</code>. Notice that the |
* <code>NUM_VALUE</code>, and <code>DIRECTION</code>. Notice that the |
384 |
* attribute tables are much smaller than 0xffff entries; as many characters |
* attribute tables are much smaller than 0xffff entries; as many characters |
385 |
* in Unicode share common attributes. Finally, there is a listing for |
* in Unicode share common attributes. The DIRECTION table also contains |
386 |
* <code>TITLE</code> exceptions (most characters just have the same |
* a field for detecting characters with multi-character uppercase expansions. |
387 |
* title case as upper case). |
* Next, there is a listing for <code>TITLE</code> exceptions (most characters |
388 |
|
* just have the same title case as upper case). Finally, there are two |
389 |
|
* tables for multi-character capitalization, <code>UPPER_SPECIAL</code> |
390 |
|
* which lists the characters which are special cased, and |
391 |
|
* <code>UPPER_EXPAND</code>, which lists their expansion. |
392 |
* |
* |
393 |
* \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke, |
* \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke, |
394 |
* Eric Blake) |
* Eric Blake) |
395 |
* \@see Character |
* \@see Character |
396 |
|
* \@see String |
397 |
*/ |
*/ |
398 |
public interface CharData |
public interface CharData |
399 |
{ |
{ |
441 |
print OUTPUT $i ? "\n + \"" : " = \""; |
print OUTPUT $i ? "\n + \"" : " = \""; |
442 |
for $j (0 .. 10) { |
for $j (0 .. 10) { |
443 |
last if $len <= $i * 11 + $j; |
last if $len <= $i * 11 + $j; |
444 |
my $val = unpack "n", substr($bestblkstr, 2 * ($i*11 + $j), 2); |
my $val = unpack "n", substr($bestblkstr, 2 * ($i * 11 + $j), 2); |
445 |
print OUTPUT javaChar($val); |
print OUTPUT javaChar($val); |
446 |
} |
} |
447 |
print OUTPUT "\""; |
print OUTPUT "\""; |
475 |
; |
; |
476 |
|
|
477 |
/** |
/** |
478 |
* This is the attribute table for computing the uppercase representation |
* This is the attribute table for computing the single-character uppercase |
479 |
* of a character. The value is the signed difference between the |
* representation of a character. The value is the signed difference |
480 |
* character and its uppercase version. Note that this is stored as an |
* between the character and its uppercase version. Note that this is |
481 |
* unsigned char since this is a String literal. |
* stored as an unsigned char since this is a String literal. When |
482 |
|
* capitalizing a String, you must first check if a multi-character uppercase |
483 |
|
* sequence exists before using this character. |
484 |
*/ |
*/ |
485 |
String UPPER |
String UPPER |
486 |
EOF |
EOF |
509 |
EOF |
EOF |
510 |
|
|
511 |
$len = @charinfo; |
$len = @charinfo; |
512 |
for ($i = 0; $i < $len / 11; $i++) { |
for ($i = 0; $i < $len / 13; $i++) { |
513 |
print OUTPUT $i ? "\n + \"" : " = \""; |
print OUTPUT $i ? "\n + \"" : " = \""; |
514 |
for $j (0 .. 10) { |
for $j (0 .. 12) { |
515 |
last if $len <= $i * 11 + $j; |
last if $len <= $i * 13 + $j; |
516 |
my $val = $charinfo[$i * 11 + $j][2]; |
my $val = $charinfo[$i * 13 + $j][2]; |
517 |
print OUTPUT javaChar($val); |
print OUTPUT javaChar($val); |
518 |
} |
} |
519 |
print OUTPUT "\""; |
print OUTPUT "\""; |
524 |
|
|
525 |
/** |
/** |
526 |
* This is the attribute table for computing the directionality class |
* This is the attribute table for computing the directionality class |
527 |
* of a character. At present, the value is in the range 0 - 18 if the |
* of a character, as well as a marker of characters with a multi-character |
528 |
* character has a direction, otherwise it is -1. Note that this is |
* capitalization. The direction is taken by performing a signed shift |
529 |
* stored as an unsigned char since this is a String literal. |
* right by 2 (where a result of -1 means an unknown direction, such as |
530 |
|
* for undefined characters). The lower 2 bits form a count of the |
531 |
|
* additional characters that will be added to a String when performing |
532 |
|
* multi-character uppercase expansion. This count is also used, along with |
533 |
|
* the offset in UPPER_SPECIAL, to determine how much of UPPER_EXPAND to use |
534 |
|
* when performing the case conversion. Note that this information is stored |
535 |
|
* as an unsigned char since this is a String literal. |
536 |
*/ |
*/ |
537 |
String DIRECTION |
String DIRECTION |
538 |
EOF |
EOF |
539 |
|
|
540 |
$len = @charinfo; |
$len = @charinfo; |
541 |
for ($i = 0; $i < $len / 11; $i++) { |
for ($i = 0; $i < $len / 17; $i++) { |
542 |
print OUTPUT $i ? "\n + \"" : " = \""; |
print OUTPUT $i ? "\n + \"" : " = \""; |
543 |
for $j (0 .. 10) { |
for $j (0 .. 16) { |
544 |
last if $len <= $i * 11 + $j; |
last if $len <= $i * 17 + $j; |
545 |
my $val = $charinfo[$i * 11 + $j][3]; |
my $val = $charinfo[$i * 17 + $j][3]; |
546 |
print OUTPUT javaChar($val); |
print OUTPUT javaChar($val); |
547 |
} |
} |
548 |
print OUTPUT "\""; |
print OUTPUT "\""; |
554 |
/** |
/** |
555 |
* This is the listing of titlecase special cases (all other character |
* This is the listing of titlecase special cases (all other character |
556 |
* can use <code>UPPER</code> to determine their titlecase). The listing |
* can use <code>UPPER</code> to determine their titlecase). The listing |
557 |
* is a sequence of character pairs; converting the first character of the |
* is a sorted sequence of character pairs; converting the first character |
558 |
* pair to titlecase produces the second character. |
* of the pair to titlecase produces the second character. |
559 |
*/ |
*/ |
560 |
String TITLE |
String TITLE |
561 |
EOF |
EOF |
565 |
print OUTPUT $i ? "\n + \"" : " = \""; |
print OUTPUT $i ? "\n + \"" : " = \""; |
566 |
for $j (0 .. 10) { |
for $j (0 .. 10) { |
567 |
last if $len <= $i * 11 + $j; |
last if $len <= $i * 11 + $j; |
568 |
my $val = unpack "n", substr($titlecase, 2 * ($i*11 + $j), 2); |
my $val = unpack "n", substr($titlecase, 2 * ($i * 11 + $j), 2); |
569 |
|
print OUTPUT javaChar($val); |
570 |
|
} |
571 |
|
print OUTPUT "\""; |
572 |
|
} |
573 |
|
|
574 |
|
print OUTPUT <<EOF; |
575 |
|
; |
576 |
|
|
577 |
|
/** |
578 |
|
* This is a listing of characters with multi-character uppercase sequences. |
579 |
|
* A character appears in this list exactly when it has a non-zero entry |
580 |
|
* in the low-order 2-bit field of DIRECTION. The listing is a sorted |
581 |
|
* sequence of pairs (hence a binary search on the even elements is an |
582 |
|
* efficient way to lookup a character). The first element of a pair is the |
583 |
|
* character with the expansion, and the second is the index into |
584 |
|
* UPPER_EXPAND where the expansion begins. Use the 2-bit field of |
585 |
|
* DIRECTION to determine where the expansion ends. |
586 |
|
*/ |
587 |
|
String UPPER_SPECIAL |
588 |
|
EOF |
589 |
|
|
590 |
|
my @list = sort {$a <=> $b} keys %special; |
591 |
|
my $expansion = ""; |
592 |
|
my $offset = 0; |
593 |
|
$len = @list; |
594 |
|
for ($i = 0; $i < $len / 5; $i++) { |
595 |
|
print OUTPUT $i ? "\n + \"" : " = \""; |
596 |
|
for $j (0 .. 4) { |
597 |
|
last if $len <= $i * 5 + $j; |
598 |
|
my $ch = $list[$i * 5 + $j]; |
599 |
|
print OUTPUT javaChar($ch); |
600 |
|
print OUTPUT javaChar($offset); |
601 |
|
$offset += @{$special{$ch}}; |
602 |
|
$expansion .= pack "n*", @{$special{$ch}}; |
603 |
|
} |
604 |
|
print OUTPUT "\""; |
605 |
|
} |
606 |
|
|
607 |
|
print OUTPUT <<EOF; |
608 |
|
; |
609 |
|
|
610 |
|
/** |
611 |
|
* This is the listing of special case multi-character uppercase sequences. |
612 |
|
* Characters listed in UPPER_SPECIAL index into this table to find their |
613 |
|
* uppercase expansion. Remember that you must also perform special-casing |
614 |
|
* on two single-character sequences in the Turkish locale, which are not |
615 |
|
* covered here in CharData. |
616 |
|
*/ |
617 |
|
String UPPER_EXPAND |
618 |
|
EOF |
619 |
|
|
620 |
|
$len = length($expansion) / 2; |
621 |
|
for ($i = 0; $i < $len / 11; $i++) { |
622 |
|
print OUTPUT $i ? "\n + \"" : " = \""; |
623 |
|
for $j (0 .. 10) { |
624 |
|
last if $len <= $i * 11 + $j; |
625 |
|
my $val = unpack "n", substr($expansion, 2 * ($i * 11 + $j), 2); |
626 |
print OUTPUT javaChar($val); |
print OUTPUT javaChar($val); |
627 |
} |
} |
628 |
print OUTPUT "\""; |
print OUTPUT "\""; |