bugGNU gettext - Bugs: bug #66378, cldr-plurals fails to parse recent...

 
 

You are not allowed to post comments on this tracker with your current authentication level.

bug #66378: cldr-plurals fails to parse recent CLDR data: 1c6, ...

Submitter:  Michele Locati <mlocati>
Submitted:  Fri 25 Oct 2024 12:40:04 PM UTC
   
 
Category:  Plural forms Severity:  3 - Normal
Item Group:  None Status:  None
Privacy:  Public Assigned to:  None
Open/Closed:  Open

Wed 06 Nov 2024 08:40:25 AM UTC, comment #3: 

I've updated the changes: see attached file 0001-Accept-CLDR-rules-with-XcY-samples-and-c-and-e-vars.patch

(file #56602)

Michele Locati <mlocati>
Tue 05 Nov 2024 07:36:50 AM UTC, comment #2: 

@Bruno if the approach I described in comment #1 is ok for you, I can submit a patch (with possibly some tests)

Michele Locati <mlocati>
Mon 28 Oct 2024 11:27:37 AM UTC, comment #1: 

I managed to get cldr-plurals work with the following patch:

From fadb2220b6cd04c50d6d2bb5c6c6488c119c5484 Mon Sep 17 00:00:00 2001
From: Michele Locati <michele@locati.it>
Date: Mon, 28 Oct 2024 12:15:15 +0100
Subject: [PATCH] Accept CLDR rules with XcY samples and c and e vars

---
 gettext-tools/src/cldr-plural-exp.c | 128 ++++++++++++++++++++++++++--
 gettext-tools/src/cldr-plural.y     |   4 +-
 2 files changed, 126 insertions(+), 6 deletions(-)

diff --git a/gettext-tools/src/cldr-plural-exp.c b/gettext-tools/src/cldr-plural-exp.c
index 60d3d0baf..dbce9627b 100644
--- a/gettext-tools/src/cldr-plural-exp.c
+++ b/gettext-tools/src/cldr-plural-exp.c
@@ -24,6 +24,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include "string-buffer.h"
 #include "unistr.h"
 #include "xalloc.h"

@@ -101,20 +102,136 @@ cldr_plural_rule_list_free (struct cldr_plural_rule_list_ty *rules)
   free (rules);
 }

+static
+const char * get_XcY_end(const char *str)
+{
+  bool found_c = false;
+  if (str[0] < '0' || str[0] > '9')
+    return NULL;
+  str++;
+  while (str[0] != '\0')
+    {
+      if (str[0] == 'c')
+        {
+          if (found_c || str[1] < '0' || str[1] > '9')
+            return NULL;
+          found_c = true;
+        }
+      else if ((str[0] < '0' || str[0] > '9') && str[0] != '.')
+        break;
+      str++;
+    }
+  if (!found_c)
+    return NULL;
+  while(str[0] == ' ')
+    str++;
+  if (str[0] == ',')
+    {
+      str++;
+      while(str[0] == ' ')
+        str++;
+    }
+  return str;
+}
+
+static
+char * remove_XcY(const char *input)
+{
+  struct string_buffer buffer;
+  const char *p;
+  const char *p_next;
+  const char *p_next1;
+  const char *p_next2;
+
+  sb_init (&buffer);
+  p = (char*) input;
+  for (;;) {
+    int comma_and_spaces;
+    p_next1 = strstr(p, "@integer ");
+    p_next2 = strstr(p, "@decimal ");
+    if (p_next1 == NULL && p_next2 == NULL)
+      {
+        sb_append_c(&buffer, p);
+        break;
+      }
+    if (p_next1 != NULL && (p_next2 == NULL || p_next1 < p_next2))
+      p_next = p_next1 + /* strlen("@integer ")*/ 9;
+    else
+      p_next = p_next2 + /* strlen("@decimal ")*/ 9;
+    while (p < p_next)
+      sb_append1(&buffer, *p++);
+    while (p[0] == ' ')
+      sb_append1(&buffer, *p++);
+    comma_and_spaces = -1;
+    for (;;)
+      {
+        const char * XcY_end;
+        if (p[0] < '0' || p[0] > '9')
+          break;
+        XcY_end = get_XcY_end(p);
+        if (XcY_end != NULL)
+          {
+            p = XcY_end;
+            continue;
+          }
+        if (comma_and_spaces >= 0)
+          {
+            sb_append1(&buffer, ',');
+            while (comma_and_spaces > 0)
+              {
+                sb_append1(&buffer, ' ');
+                comma_and_spaces--;
+              }
+          }
+        while ((p[0] >= '0' && p[0] <= '9') || p[0] == '.' || p[0] == '~')
+          {
+            sb_append1(&buffer, p[0]);
+            p++;
+          }
+        if (p[0] != ',')
+          break;
+        comma_and_spaces = 0;
+        p++;
+        while (p[0] == ' ')
+          {
+            comma_and_spaces++;
+            p++;
+          }
+      }
+      if (comma_and_spaces > 0 && p[0] == '\xE2' && p[1] == '\x80' && p[2] == '\xA6')
+        {
+          sb_append1(&buffer, ',');
+          while (comma_and_spaces > 0)
+            {
+              sb_append1(&buffer, ' ');
+              comma_and_spaces--;
+            }
+        }
+  }
+  return sb_dupfree_c(&buffer);
+}
+
 struct cldr_plural_rule_list_ty *
 cldr_plural_parse (const char *input)
 {
   struct cldr_plural_parse_args arg;
+  char *input_without_XcY;

   memset (&arg, 0, sizeof (struct cldr_plural_parse_args));
-  arg.cp = input;
-  arg.cp_end = input + strlen (input);
+  input_without_XcY = remove_XcY(input);
+  if (input_without_XcY == NULL)
+    return NULL;
+  arg.cp = input_without_XcY;
+  arg.cp_end = input_without_XcY + strlen(input_without_XcY);;
   arg.result = XMALLOC (struct cldr_plural_rule_list_ty);
   memset (arg.result, 0, sizeof (struct cldr_plural_rule_list_ty));

   if (yyparse (&arg) != 0)
-    return NULL;
-
+    {
+      free(input_without_XcY);
+      return NULL;
+    }
+  free(input_without_XcY);
   return arg.result;
 }

@@ -156,10 +273,11 @@ eval_relation (struct cldr_plural_relation_ty *relation)
       break;
     case 'f': case 't':
     case 'v': case 'w':
+    case 'c': case 'e':
       {
         /* Since plural expression in gettext only supports unsigned
            integer, turn relations whose operand is either 'f', 't',
-           'v', or 'w' into a constant truth value.  */
+           'v', 'w', 'c', or 'e' into a constant truth value.  */
         /* FIXME: check mod?  */
         size_t i;
         for (i = 0; i < relation->ranges->nitems; i++)
diff --git a/gettext-tools/src/cldr-plural.y b/gettext-tools/src/cldr-plural.y
index 05c1b56ec..3e28d224b 100644
--- a/gettext-tools/src/cldr-plural.y
+++ b/gettext-tools/src/cldr-plural.y
@@ -263,6 +263,7 @@ at_decimal: %empty
         ;

 sample_list: sample_list1 sample_ellipsis
+        | ELLIPSIS
         ;
 sample_list1: sample_range
         | sample_list1 ',' sample_range
@@ -413,7 +414,8 @@ yylex (YYSTYPE *lval, struct cldr_plural_parse_args *arg)
           {
             switch (ident[0])
               {
-              case 'n': case 'i': case 'f': case 't': case 'v': case 'w':
+              // See https://unicode.org/reports/tr35/tr35-numbers.html#table-plural-operand-meanings
+              case 'n': case 'i': case 'f': case 't': case 'v': case 'w': case 'c': case 'e':
                 arg->cp = exp;
                 lval->ival = ident[0];
                 sb_free (&buffer);


This patch basically:

1. accept the "e" and "c" variables described in
https://unicode.org/reports/tr35/tr35-numbers.html#table-plural-operand-meanings
(assuming they have a value of zero)

2. strips out the XcY samples (eg 1c6, 1.0000001c6), thus working for example on

one: i = 0,1 @integer 0, 1 @decimal 0.0~1.5; many: e = 0 and i != 0 and i % 1000000 = 0 and v = 0 or e != 0..5 @integer 1000000, … @decimal …; other:  @integer 2~17, 100, 1000, 10000, 100000, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, …

instead of

one: i = 0,1 @integer 0, 1 @decimal 0.0~1.5; many: e = 0 and i != 0 and i % 1000000 = 0 and v = 0 or e != 0..5 @integer 1000000, 1c6, 2c6, 3c6, 4c6, 5c6, 6c6, … @decimal 1.0000001c6, 1.1c6, 2.0000001c6, 2.1c6, 3.0000001c6, 3.1c6, …; other:  @integer 2~17, 100, 1000, 10000, 100000, 1c3, 2c3, 3c3, 4c3, 5c3, 6c3, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 1.0001c3, 1.1c3, 2.0001c3, 2.1c3, 3.0001c3, 3.1c3, …


Generating this output:

nplurals=3; plural=(n==0 || n==1 ? 0 : n!=0 && n%1000000==0 ? 1 : 2);


Michele Locati <mlocati>
Fri 25 Oct 2024 12:40:04 PM UTC, original submission:  

msginit may use cldr-plurals to generate plural rules starting from the common/supplemental/plurals.xml file provided by CLDR.

For example, from this chunk of CLDR v37 data

<pluralRules locales="ff fr hy kab">
  <pluralRule count="one">i = 0,1 @integer 0, 1 @decimal 0.0~1.5</pluralRule>
  <pluralRule count="other"> @integer 2~17, 100, 1000, 10000, 100000, 1000000, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, …</pluralRule>
</pluralRules>


cldr-plurals generates this rule for French:

nplurals=2; plural=(n > 1);


This allows 2 different plural translations like:
- 1 page
- 2 pages

Since CLDR v38 they extended the format of the plurals.xml file [1]:

<pluralRules locales="fr">
  <pluralRule count="one">i = 0,1 @integer 0, 1 @decimal 0.0~1.5</pluralRule>
  <pluralRule count="many">e = 0 and i != 0 and i % 1000000 = 0 and v = 0 or e != 0..5 @integer 1000000, 1e6, 2e6, 3e6, 4e6, 5e6, 6e6, … @decimal 1.0000001e6, 1.1e6, 2.0000001e6, 2.1e6, 3.0000001e6, 3.1e6, …</pluralRule>
  <pluralRule count="other"> @integer 2~17, 100, 1000, 10000, 100000, 1e3, 2e3, 3e3, 4e3, 5e3, 6e3, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 1.0001e3, 1.1e3, 2.0001e3, 2.1e3, 3.0001e3, 3.1e3, …</pluralRule>
</pluralRules>


This is used to add a third translation for 1000000, 1e6 (1 million expressed for example like 1M), and so on.
For example, we can now have 3 translations like:
- 1 page
- 1M de pages
- 2 pages

PS: More and more languages are using this new syntax in later CLDR versions (as of v46 we have: ca, es, fr, it, lld, pt, pt_PT, scn, vec).

The problem is that cldr-plurals can't parse this new format:

$ cldr-plurals -c fr /cldr-46/common/supplemental/plurals.xml
one: i = 0,1 @integer 0, 1 @decimal 0.0~1.5; many: e = 0 and i != 0 and i % 1000000 = 0 and v = 0 or e != 0..5 @integer 1000000, 1c6, 2c6, 3c6, 4c6, 5c6, 6c6, … @decimal 1.0000001c6, 1.1c6, 2.0000001c6, 2.1c6, 3.0000001c6, 3.1c6, …; other:  @integer 2~17, 100, 1000, 10000, 100000, 1c3, 2c3, 3c3, 4c3, 5c3, 6c3, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 1.0001c3, 1.1c3, 2.0001c3, 2.1c3, 3.0001c3, 3.1c3, …

$ cldr-plurals fr /cldr-46/common/supplemental/plurals.xml
syntax error
cldr-plurals.exe: cannot parse CLDR rule



[1] https://cldr.unicode.org/downloads/cldr-38#h.7ahoxbsngtxp_l

Michele Locati <mlocati>

 

(Note: upload size limit is set to 16384 kB, after insertion of the required escape characters.)

Attached Files

 

Depends on the following items: None found

Items that depend on this one: None found

 

Carbon-Copy List
  • -email is unavailable- added by mlocati (Submitted the item)
  •  

    There are 0 votes so far. Votes easily highlight which items people would like to see resolved in priority, independently of the priority of the item set by tracker managers.

     

    Follows 1 latest change.

    Date Changed by Updated Field Previous Value => Replaced by
    2024-11-06 mlocati Attached File- Added 0001-Accept-CLDR-rules-with-XcY-samples-and-c-and-e-vars.patch, #56602

    Back to the top

    Powered by Savane 3.15-4cd8.
    Corresponding source code