/[classpath]/classpath/scripts/unicode-muncher.pl
ViewVC logotype

Diff of /classpath/scripts/unicode-muncher.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1 by ericb, Wed Feb 20 03:28:17 2002 UTC revision 1.2 by ericb, Sat Mar 9 04:01:12 2002 UTC
# Line 43  Line 43 
43  # Inspired by code from Jochen Hoenicke.  # Inspired by code from Jochen Hoenicke.
44  # author Eric Blake <ebb9@email.byu.edu>  # author Eric Blake <ebb9@email.byu.edu>
45  #  #
46  # Usage: ./unicode-muncher <UnicodeData.txt> <CharData.java>  # Usage: ./unicode-muncher <UnicodeData> <SpecialCasing> <CharData.java>
47  #   where <UnicodeData.txt> is obtained from www.unicode.org (named  #   where <UnicodeData> and <SpecialCasing> are .txt files obtained from
48  #   UnicodeData-3.0.0.txt for Unicode version 3.0.0), and <CharData.java>  #   www.unicode.org (named UnicodeData-3.0.0.txt and SpecialCasing-2.txt for
49  #   is the final location for the Java interface gnu.java.lang.CharData.  #   Unicode version 3.0.0), and <CharData.java> is the final location for the
50    #   Java interface gnu.java.lang.CharData.
51  #   As of JDK 1.4, use Unicode version 3.0.0 for best results.  #   As of JDK 1.4, use Unicode version 3.0.0 for best results.
52    
53  ##  ##
# Line 75  my @DIRCODES = qw(L R AL EN ES ET AN CS Line 76  my @DIRCODES = qw(L R AL EN ES ET AN CS
76  my $NOBREAK_FLAG  = 32;  my $NOBREAK_FLAG  = 32;
77  my $MIRRORED_FLAG = 64;  my $MIRRORED_FLAG = 64;
78    
79    my %special = ();
80  my @info = ();  my @info = ();
81  my $titlecase = "";  my $titlecase = "";
82  my $count = 0;  my $count = 0;
83  my $range = 0;  my $range = 0;
84    
85  die "Usage: $0 <UnicodeData.txt> <CharData.java>" unless @ARGV == 2;  die "Usage: $0 <UnicodeData.txt> <SpecialCasing.txt> <CharData.java>"
86  open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n";      unless @ARGV == 3;
   
 # Stage 1: Parse the attribute file  
87  $| = 1;  $| = 1;
88  print "GNU Classpath Unicode Attribute Database Generator 2.0\n";  print "GNU Classpath Unicode Attribute Database Generator 2.0\n";
89  print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n";  print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n";
90    
91    # Stage 0: Parse the special casing file
92    print "Parsing special casing file\n";
93    open (SPECIAL, "< $ARGV[1]") || die "Can't open special casing file: $!\n";
94    while (<SPECIAL>) {
95        next if /^\#/;
96        my ($ch, undef, undef, $upper) = split / *; */;
97    
98        # This grabs only the special casing for multi-char uppercase. Note that
99        # there are no multi-char lowercase, and that Sun ignores multi-char
100        # titlecase rules. This script omits 3 special cases in Unicode 3.0.0,
101        # which must be hardcoded in java.lang.String:
102        #  \u03a3 (Sun ignores this special case)
103        #  \u0049 - lowercases to \u0131, but only in Turkish locale
104        #  \u0069 - uppercases to \u0130, but only in Turkish locale
105        next unless defined $upper and $upper =~ / /;
106        $special{hex $ch} = [map {hex} split ' ', $upper];
107    }
108    
109    close SPECIAL;
110    
111    # Stage 1: Parse the attribute file
112  print "Parsing attributes file";  print "Parsing attributes file";
113  while(<UNICODE>) {  open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n";
114    while (<UNICODE>) {
115      print "." unless $count++ % 1000;      print "." unless $count++ % 1000;
116      chomp;      chomp;
117      s/\r//g;      s/\r//g;
# Line 142  while(<UNICODE>) { Line 165  while(<UNICODE>) {
165              last;              last;
166          }          }
167      }      }
168        $direction <<= 2;
169        $direction += $#{$special{$ch}} if defined $special{$ch};
170    
171      if ($range) {      if ($range) {
172          die "Expecting end of range at $ch\n" unless $name =~ /Last>$/;          die "Expecting end of range at $ch\n" unless $name =~ /Last>$/;
# Line 167  my @charinfo = (); Line 192  my @charinfo = ();
192    
193  for my $ch (0 .. 0xffff) {  for my $ch (0 .. 0xffff) {
194      print "." unless $count++ % 0x1000;      print "." unless $count++ % 0x1000;
195      if (! defined $info[$ch]) {      $info[$ch] = pack("n5", 0, -1, 0, 0, -4) unless defined $info[$ch];
         $info[$ch] = pack("n5", 0, -1, 0, 0, -1);  
     }  
196    
197      my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]);      my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]);
198      if (! exists $charhash{$info[$ch]}) {      if (! exists $charhash{$info[$ch]}) {
# Line 209  for my $i (3 .. 8) { Line 232  for my $i (3 .. 8) {
232      for ($j = $blksize - 1; $j > 0; $j--) {      for ($j = $blksize - 1; $j > 0; $j--) {
233          my %tails = ();          my %tails = ();
234          for $k (0 .. $#blkarray) {          for $k (0 .. $#blkarray) {
235              next if ! defined $blkarray[$k];              next unless defined $blkarray[$k];
236              my $len = length $blkarray[$k];              my $len = length $blkarray[$k];
237              my $tail = substr $blkarray[$k], $len - $j * 2;              my $tail = substr $blkarray[$k], $len - $j * 2;
238              if (exists $tails{$tail}) {              if (exists $tails{$tail}) {
# Line 222  for my $i (3 .. 8) { Line 245  for my $i (3 .. 8) {
245          # tails are calculated, now calculate the heads and merge.          # tails are calculated, now calculate the heads and merge.
246        BLOCK:        BLOCK:
247          for $k (0 .. $#blkarray) {          for $k (0 .. $#blkarray) {
248              next if ! defined $blkarray[$k];              next unless defined $blkarray[$k];
249              my $tomerge = $k;              my $tomerge = $k;
250              while (1) {              while (1) {
251                  my $head = substr($blkarray[$tomerge], 0, $j * 2);                  my $head = substr($blkarray[$tomerge], 0, $j * 2);
252                  my $entry = $tails{$head};                  my $entry = $tails{$head};
253                  next BLOCK if ! defined $entry;                  next BLOCK unless defined $entry;
254    
255                  my $other = shift @{$entry};                  my $other = shift @{$entry};
256                  if ($other == $tomerge) {                  if ($other == $tomerge) {
# Line 297  die "UTF-8 limit of blocks may be exceed Line 320  die "UTF-8 limit of blocks may be exceed
320  die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n"  die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n"
321      if length($bestblkstr) > 0xffff / 3;      if length($bestblkstr) > 0xffff / 3;
322  {  {
323      print "Generating $ARGV[1] with shift of $bestshift";      print "Generating $ARGV[2] with shift of $bestshift";
324      my ($i, $j);      my ($i, $j);
325    
326      open OUTPUT, "> $ARGV[1]" or die "Failed creating output file: $!\n";      open OUTPUT, "> $ARGV[2]" or die "Failed creating output file: $!\n";
327      print OUTPUT <<EOF;      print OUTPUT <<EOF;
328  /* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info  /* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info
329     Copyright (C) 2002 Free Software Foundation, Inc.     Copyright (C) 2002 Free Software Foundation, Inc.
# Line 345  package gnu.java.lang; Line 368  package gnu.java.lang;
368  /**  /**
369   * This contains the info about the unicode characters, that   * This contains the info about the unicode characters, that
370   * java.lang.Character needs.  It is generated automatically from   * java.lang.Character needs.  It is generated automatically from
371   * <code>$ARGV[0]</code>, by some perl scripts.   * <code>$ARGV[0]</code> and
372   * This Unicode definition file can be found on the   * <code>$ARGV[1]</code>, by some
373     * perl scripts. These Unicode definition file can be found on the
374   * <a href="http://www.unicode.org">http://www.unicode.org</a> website.   * <a href="http://www.unicode.org">http://www.unicode.org</a> website.
375   * JDK 1.4 uses Unicode version 3.0.0.   * JDK 1.4 uses Unicode version 3.0.0.
376   *   *
# Line 358  package gnu.java.lang; Line 382  package gnu.java.lang;
382   * into the attribute tables <code>UPPER</code>, <code>LOWER</code>,   * into the attribute tables <code>UPPER</code>, <code>LOWER</code>,
383   * <code>NUM_VALUE</code>, and <code>DIRECTION</code>.  Notice that the   * <code>NUM_VALUE</code>, and <code>DIRECTION</code>.  Notice that the
384   * attribute tables are much smaller than 0xffff entries; as many characters   * attribute tables are much smaller than 0xffff entries; as many characters
385   * in Unicode share common attributes.  Finally, there is a listing for   * in Unicode share common attributes.  The DIRECTION table also contains
386   * <code>TITLE</code> exceptions (most characters just have the same   * a field for detecting characters with multi-character uppercase expansions.
387   * title case as upper case).   * Next, there is a listing for <code>TITLE</code> exceptions (most characters
388     * just have the same title case as upper case).  Finally, there are two
389     * tables for multi-character capitalization, <code>UPPER_SPECIAL</code>
390     * which lists the characters which are special cased, and
391     * <code>UPPER_EXPAND</code>, which lists their expansion.
392   *   *
393   * \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke,   * \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke,
394   *         Eric Blake)   *         Eric Blake)
395   * \@see Character   * \@see Character
396     * \@see String
397   */   */
398  public interface CharData  public interface CharData
399  {  {
# Line 412  EOF Line 441  EOF
441          print OUTPUT $i ? "\n    + \"" : "    = \"";          print OUTPUT $i ? "\n    + \"" : "    = \"";
442          for $j (0 .. 10) {          for $j (0 .. 10) {
443              last if $len <= $i * 11 + $j;              last if $len <= $i * 11 + $j;
444              my $val = unpack "n", substr($bestblkstr, 2 * ($i*11 + $j), 2);              my $val = unpack "n", substr($bestblkstr, 2 * ($i * 11 + $j), 2);
445              print OUTPUT javaChar($val);              print OUTPUT javaChar($val);
446          }          }
447          print OUTPUT "\"";          print OUTPUT "\"";
# Line 446  EOF Line 475  EOF
475  ;  ;
476    
477    /**    /**
478     * This is the attribute table for computing the uppercase representation     * This is the attribute table for computing the single-character uppercase
479     * of a character.  The value is the signed difference between the     * representation of a character.  The value is the signed difference
480     * character and its uppercase version.  Note that this is stored as an     * between the character and its uppercase version.  Note that this is
481     * unsigned char since this is a String literal.     * stored as an unsigned char since this is a String literal.  When
482       * capitalizing a String, you must first check if a multi-character uppercase
483       * sequence exists before using this character.
484     */     */
485    String UPPER    String UPPER
486  EOF  EOF
# Line 478  EOF Line 509  EOF
509  EOF  EOF
510    
511      $len = @charinfo;      $len = @charinfo;
512      for ($i = 0; $i < $len / 11; $i++) {      for ($i = 0; $i < $len / 13; $i++) {
513          print OUTPUT $i ? "\n    + \"" : "    = \"";          print OUTPUT $i ? "\n    + \"" : "    = \"";
514          for $j (0 .. 10) {          for $j (0 .. 12) {
515              last if $len <= $i * 11 + $j;              last if $len <= $i * 13 + $j;
516              my $val = $charinfo[$i * 11 + $j][2];              my $val = $charinfo[$i * 13 + $j][2];
517              print OUTPUT javaChar($val);              print OUTPUT javaChar($val);
518          }          }
519          print OUTPUT "\"";          print OUTPUT "\"";
# Line 493  EOF Line 524  EOF
524    
525    /**    /**
526     * This is the attribute table for computing the directionality class     * This is the attribute table for computing the directionality class
527     * of a character.  At present, the value is in the range 0 - 18 if the     * of a character, as well as a marker of characters with a multi-character
528     * character has a direction, otherwise it is -1.  Note that this is     * capitalization.  The direction is taken by performing a signed shift
529     * stored as an unsigned char since this is a String literal.     * right by 2 (where a result of -1 means an unknown direction, such as
530       * for undefined characters). The lower 2 bits form a count of the
531       * additional characters that will be added to a String when performing
532       * multi-character uppercase expansion. This count is also used, along with
533       * the offset in UPPER_SPECIAL, to determine how much of UPPER_EXPAND to use
534       * when performing the case conversion. Note that this information is stored
535       * as an unsigned char since this is a String literal.
536     */     */
537    String DIRECTION    String DIRECTION
538  EOF  EOF
539    
540      $len = @charinfo;      $len = @charinfo;
541      for ($i = 0; $i < $len / 11; $i++) {      for ($i = 0; $i < $len / 17; $i++) {
542          print OUTPUT $i ? "\n    + \"" : "    = \"";          print OUTPUT $i ? "\n    + \"" : "    = \"";
543          for $j (0 .. 10) {          for $j (0 .. 16) {
544              last if $len <= $i * 11 + $j;              last if $len <= $i * 17 + $j;
545              my $val = $charinfo[$i * 11 + $j][3];              my $val = $charinfo[$i * 17 + $j][3];
546              print OUTPUT javaChar($val);              print OUTPUT javaChar($val);
547          }          }
548          print OUTPUT "\"";          print OUTPUT "\"";
# Line 517  EOF Line 554  EOF
554    /**    /**
555     * This is the listing of titlecase special cases (all other character     * This is the listing of titlecase special cases (all other character
556     * can use <code>UPPER</code> to determine their titlecase).  The listing     * can use <code>UPPER</code> to determine their titlecase).  The listing
557     * is a sequence of character pairs; converting the first character of the     * is a sorted sequence of character pairs; converting the first character
558     * pair to titlecase produces the second character.     * of the pair to titlecase produces the second character.
559     */     */
560    String TITLE    String TITLE
561  EOF  EOF
# Line 528  EOF Line 565  EOF
565          print OUTPUT $i ? "\n    + \"" : "    = \"";          print OUTPUT $i ? "\n    + \"" : "    = \"";
566          for $j (0 .. 10) {          for $j (0 .. 10) {
567              last if $len <= $i * 11 + $j;              last if $len <= $i * 11 + $j;
568              my $val = unpack "n", substr($titlecase, 2 * ($i*11 + $j), 2);              my $val = unpack "n", substr($titlecase, 2 * ($i * 11 + $j), 2);
569                print OUTPUT javaChar($val);
570            }
571            print OUTPUT "\"";
572        }
573    
574        print OUTPUT <<EOF;
575    ;
576    
577      /**
578       * This is a listing of characters with multi-character uppercase sequences.
579       * A character appears in this list exactly when it has a non-zero entry
580       * in the low-order 2-bit field of DIRECTION.  The listing is a sorted
581       * sequence of pairs (hence a binary search on the even elements is an
582       * efficient way to lookup a character). The first element of a pair is the
583       * character with the expansion, and the second is the index into
584       * UPPER_EXPAND where the expansion begins. Use the 2-bit field of
585       * DIRECTION to determine where the expansion ends.
586       */
587      String UPPER_SPECIAL
588    EOF
589    
590        my @list = sort {$a <=> $b} keys %special;
591        my $expansion = "";
592        my $offset = 0;
593        $len = @list;
594        for ($i = 0; $i < $len / 5; $i++) {
595            print OUTPUT $i ? "\n    + \"" : "    = \"";
596            for $j (0 .. 4) {
597                last if $len <= $i * 5 + $j;
598                my $ch = $list[$i * 5 + $j];
599                print OUTPUT javaChar($ch);
600                print OUTPUT javaChar($offset);
601                $offset += @{$special{$ch}};
602                $expansion .= pack "n*", @{$special{$ch}};
603            }
604            print OUTPUT "\"";
605        }
606    
607        print OUTPUT <<EOF;
608    ;
609    
610      /**
611       * This is the listing of special case multi-character uppercase sequences.
612       * Characters listed in UPPER_SPECIAL index into this table to find their
613       * uppercase expansion. Remember that you must also perform special-casing
614       * on two single-character sequences in the Turkish locale, which are not
615       * covered here in CharData.
616       */
617      String UPPER_EXPAND
618    EOF
619    
620        $len = length($expansion) / 2;
621        for ($i = 0; $i < $len / 11; $i++) {
622            print OUTPUT $i ? "\n    + \"" : "    = \"";
623            for $j (0 .. 10) {
624                last if $len <= $i * 11 + $j;
625                my $val = unpack "n", substr($expansion, 2 * ($i * 11 + $j), 2);
626              print OUTPUT javaChar($val);              print OUTPUT javaChar($val);
627          }          }
628          print OUTPUT "\"";          print OUTPUT "\"";

Legend:
Removed from v.1.1  
changed lines
  Added in v.1.2

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26