# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1589576604 -7200
#      Fri May 15 23:03:24 2020 +0200
# Node ID d42025a277ea1abe1a1bb5bbc0ad367ceb2fd5af
# Parent  a01ad9893641d0c94c8a8e0df31923c70f82ddd0
Convert character vectors in .mat files (-v5) to UTF-8 (bug #58368).

* ls-mat5.cc (read_mat5_binary_element): Convert UTF-16 and UTF-32 vectors to
UTF-8. Take UTF-8 as is. Fall back to replacing non-ASCII characters with "?"
for non-convertable strings or ND arrays.

* unistr-wrappers.h, unistr-wrappers.cc (octave_u16_to_u8_wrapper,
octave_u32_to_u8_wrapper): New wrappers for gnulib functions.

* bootstrap.conf: Add gnulib modules to list.

diff -r a01ad9893641 -r d42025a277ea bootstrap.conf
--- a/bootstrap.conf	Thu May 14 11:32:06 2020 +0200
+++ b/bootstrap.conf	Fri May 15 23:03:24 2020 +0200
@@ -117,6 +117,8 @@
   unictype/ctype-upper
   unictype/ctype-xdigit
   unistd
+  unistr/u16-to-u8
+  unistr/u32-to-u8
   unistr/u8-check
   unistr/u8-strmblen
   unistr/u8-strmbtouc
diff -r a01ad9893641 -r d42025a277ea libinterp/corefcn/ls-mat5.cc
--- a/libinterp/corefcn/ls-mat5.cc	Thu May 14 11:32:06 2020 +0200
+++ b/libinterp/corefcn/ls-mat5.cc	Fri May 15 23:03:24 2020 +0200
@@ -50,6 +50,7 @@
 #include "oct-time.h"
 #include "quit.h"
 #include "str-vec.h"
+#include "unistr-wrappers.h"
 
 #include "Cell.h"
 #include "defaults.h"
@@ -1404,58 +1405,80 @@
 
             tc = ctmp;
           }
-        else
+        else if (arrayclass == MAT_FILE_CHAR_CLASS)
           {
-            if (arrayclass == MAT_FILE_CHAR_CLASS)
+            bool converted = false;
+            if (type == miUTF16 && re.isvector ())
               {
-                if (type == miUTF16 || type == miUTF32)
-                  {
-                    bool found_big_char = false;
-                    for (octave_idx_type i = 0; i < n; i++)
-                      {
-                        if (re(i) > 127)
-                          {
-                            re(i) = '?';
-                            found_big_char = true;
-                          }
-                      }
-
-                    if (found_big_char)
-                      warning_with_id ("Octave:load:unsupported-utf-char",
-                               "load: can not read non-ASCII portions of UTF characters; replacing unreadable characters with '?'");
-                  }
-                else if (type == miUTF8)
+                uint16NDArray u16 = re;
+                const uint16_t *u16_str
+                  = reinterpret_cast<const uint16_t *> (u16.data ());
+
+                // convert to UTF-8
+                size_t n8;
+                uint8_t *u8_str = octave_u16_to_u8_wrapper (u16_str,
+                                                            u16.numel (),
+                                                            nullptr, &n8);
+                if (u8_str)
                   {
-                    // Search for multi-byte encoded UTF8 characters and
-                    // replace with 0x3F for '?'...  Give the user a warning
-
-                    bool utf8_multi_byte = false;
-                    for (octave_idx_type i = 0; i < n; i++)
+                    tc = charMatrix (std::string (reinterpret_cast<char *> (u8_str), n8));
+                    free (u8_str);
+                    converted = true;
+                  }
+              }
+            else if (type == miUTF32 && re.isvector ())
+              {
+                uint32NDArray u32 = re;
+                const uint32_t *u32_str
+                  = reinterpret_cast<const uint32_t *> (u32.data ());
+
+                // convert to UTF-8
+                size_t n8;
+                uint8_t *u8_str = octave_u32_to_u8_wrapper (u32_str,
+                                                            u32.numel (),
+                                                            nullptr, &n8);
+                if (u8_str)
+                  {
+                    tc = charMatrix (std::string (reinterpret_cast<char *> (u8_str), n8));
+                    free (u8_str);
+                    converted = true;
+                  }
+              }
+            else if (type == miUTF8)
+              {
+                // Octave's internal encoding is UTF-8.  So we should be
+                // able to use this natively.
+                tc = re;
+                tc = tc.convert_to_str (false, true, '\'');
+                converted = true;
+              }
+
+            if (! converted)
+              {
+                // fall back to manually replacing non-ASCII
+                // characters by "?"
+                bool found_big_char = false;
+                for (octave_idx_type i = 0; i < n; i++)
+                  {
+                    if (re(i) > 127)
                       {
-                        unsigned char a = static_cast<unsigned char> (re(i));
-                        if (a > 0x7f)
-                          utf8_multi_byte = true;
-                      }
-
-                    if (utf8_multi_byte)
-                      {
-                        warning_with_id ("Octave:load:unsupported-utf-char",
-                                         "load: can not read multi-byte encoded UTF8 characters; replacing unreadable characters with '?'");
-                        for (octave_idx_type i = 0; i < n; i++)
-                          {
-                            unsigned char a
-                              = static_cast<unsigned char> (re(i));
-                            if (a > 0x7f)
-                              re(i) = '?';
-                          }
+                        re(i) = '?';
+                        found_big_char = true;
                       }
                   }
+
+                if (found_big_char)
+                  warning_with_id ("Octave:load:unsupported-utf-char",
+                    "load: failed to convert from input to UTF-8; "
+                    "replacing non-ASCII characters with '?'");
+
                 tc = re;
                 tc = tc.convert_to_str (false, true, '\'');
               }
-            else
-              tc = re;
+
           }
+        else
+          tc = re;
       }
     }
 
diff -r a01ad9893641 -r d42025a277ea liboctave/wrappers/unistr-wrappers.c
--- a/liboctave/wrappers/unistr-wrappers.c	Thu May 14 11:32:06 2020 +0200
+++ b/liboctave/wrappers/unistr-wrappers.c	Fri May 15 23:03:24 2020 +0200
@@ -49,6 +49,20 @@
   return u8_strmbtouc (puc, src);
 }
 
+uint8_t *
+octave_u16_to_u8_wrapper (const uint16_t *src, size_t src_len,
+                          uint8_t *result_buf, size_t *lengthp)
+{
+  return u16_to_u8 (src, src_len, result_buf, lengthp);
+}
+
+uint8_t *
+octave_u32_to_u8_wrapper (const uint32_t *src, size_t src_len,
+                          uint8_t *result_buf, size_t *lengthp)
+{
+  return u32_to_u8 (src, src_len, result_buf, lengthp);
+}
+
 uint32_t *
 octave_u8_to_u32_wrapper (const uint8_t *src, size_t src_len,
                           uint32_t *result_buf, size_t *lengthp)
diff -r a01ad9893641 -r d42025a277ea liboctave/wrappers/unistr-wrappers.h
--- a/liboctave/wrappers/unistr-wrappers.h	Thu May 14 11:32:06 2020 +0200
+++ b/liboctave/wrappers/unistr-wrappers.h	Fri May 15 23:03:24 2020 +0200
@@ -39,6 +39,14 @@
 extern int
 octave_u8_strmbtouc_wrapper (uint32_t *puc, const uint8_t *src);
 
+extern uint8_t *
+octave_u16_to_u8_wrapper (const uint16_t *src, size_t src_len,
+                          uint8_t *result_buf, size_t *lengthp);
+
+extern uint8_t *
+octave_u32_to_u8_wrapper (const uint32_t *src, size_t src_len,
+                          uint8_t *result_buf, size_t *lengthp);
+
 extern uint32_t *
 octave_u8_to_u32_wrapper (const uint8_t *src, size_t src_len,
                           uint32_t *result_buf, size_t *lengthp);
