diff -r 68e3a747ca02 libinterp/corefcn/c-file-ptr-stream.cc --- a/libinterp/corefcn/c-file-ptr-stream.cc Fri Nov 13 14:10:26 2015 -0500 +++ b/libinterp/corefcn/c-file-ptr-stream.cc Sat Nov 14 11:26:39 2015 +1100 @@ -127,12 +127,10 @@ seekdir_to_whence (std::ios::seekdir dir } std::streampos -c_file_ptr_buf::seekoff (std::streamoff /* offset */, - std::ios::seekdir /* dir */, +c_file_ptr_buf::seekoff (std::streamoff offset, + std::ios::seekdir dir, std::ios::openmode) { - // FIXME -#if 0 if (f) { fseek (f, offset, seekdir_to_whence (dir)); @@ -141,15 +139,11 @@ c_file_ptr_buf::seekoff (std::streamoff } else return 0; -#endif - return -1; } std::streampos -c_file_ptr_buf::seekpos (std::streampos /* offset */, std::ios::openmode) +c_file_ptr_buf::seekpos (std::streampos offset, std::ios::openmode) { - // FIXME -#if 0 if (f) { fseek (f, offset, SEEK_SET); @@ -158,8 +152,6 @@ c_file_ptr_buf::seekpos (std::streampos } else return 0; -#endif - return -1; } int diff -r 68e3a747ca02 libinterp/corefcn/oct-stream.cc --- a/libinterp/corefcn/oct-stream.cc Fri Nov 13 14:10:26 2015 -0500 +++ b/libinterp/corefcn/oct-stream.cc Sat Nov 14 11:26:39 2015 +1100 @@ -267,7 +267,7 @@ scanf_format_list::add_elt_to_list (int } } -static std::string +std::string expand_char_class (const std::string& s) { std::string retval; diff -r 68e3a747ca02 libinterp/corefcn/oct-stream.h --- a/libinterp/corefcn/oct-stream.h Fri Nov 13 14:10:26 2015 -0500 +++ b/libinterp/corefcn/oct-stream.h Sat Nov 14 11:26:39 2015 +1100 @@ -747,4 +747,6 @@ private: int do_get_file_number (const octave_value& fid) const; }; +std::string expand_char_class (const std::string& s); + #endif diff -r 68e3a747ca02 libinterp/dldfcn/module-files --- a/libinterp/dldfcn/module-files Fri Nov 13 14:10:26 2015 -0500 +++ b/libinterp/dldfcn/module-files Sat Nov 14 11:26:39 2015 +1100 @@ -1,4 +1,5 @@ # FILE|CPPFLAGS|LDFLAGS|LIBRARIES +textscan.cc|-I../../liboctave/array|| __delaunayn__.cc|$(QHULL_CPPFLAGS)|$(QHULL_LDFLAGS)|$(QHULL_LIBS) __eigs__.cc|$(ARPACK_CPPFLAGS) $(SPARSE_XCPPFLAGS)|$(ARPACK_LDFLAGS) $(SPARSE_XLDFLAGS)|$(ARPACK_LIBS) $(SPARSE_XLIBS) $(LAPACK_LIBS) $(BLAS_LIBS) __fltk_uigetfile__.cc|$(FLTK_CPPFLAGS) $(FT2_CPPFLAGS)|$(FLTK_LDFLAGS) $(FT2_LDFLAGS)|$(FLTK_LIBS) $(FT2_LIBS) diff -r 68e3a747ca02 libinterp/dldfcn/textscan.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/dldfcn/textscan.cc Sat Nov 14 11:26:39 2015 +1100 @@ -0,0 +1,2189 @@ +/* +Copyright (C) 2015 Lachlan Andrew + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either version 3 of the License, or (at your +option) any later version. + +Octave is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +. + +*/ + +/* Implement the 'textscan' octave command */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "list" + +#include "Array.cc" + +#include "Cell.h" +#include "textscan.h" +#include "oct-stream.h" +#include "oct-obj.h" +#include "utils.h" +#include "defun-dld.h" +#include "ov-re-mat.h" + +DEFUN_DLD (textscan, args, , + "-*- texinfo -*-\n\ +@deftypefn {Function File} {@var{C} =} textscan (@var{fid}, @var{format})\n\ +@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{repeat})\n\ +@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{param}, @var{value}, @dots{})\n\ +@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{repeat}, @var{param}, @var{value}, @dots{})\n\ +@deftypefnx {Function File} {@var{C} =} textscan (@var{str}, @dots{})\n\ +@deftypefnx {Function File} {[@var{C}, @var{position}] =} textscan (@dots{})\n\ +Read data from a text file or string.\n\ +\n\ +The string @var{str} or file associated with @var{fid} is read from and\n\ +parsed according to @var{format}. The function is an extension\n\ +of @code{strread} and @code{textread}. Differences include: the ability to\n\ +read from either a file or a string, additional options, and additional\n\ +format specifiers.\n\ +\n\ +The input is interpreted as a sequence of \"words\", delimiters (such as\n\ +whitespace) and literals. The characters that form delimiters and whitespace\n\ +are determined by the options. The format consists of format specifiers\n\ +interspersed between literals. In the format, whitespace forms a delimiter\n\ +between consecutive literals, but is otherwise ignored.\n\ +\n\ +The output @var{C} is a cell array whose second dimension is determined\n\ +by the number of format specifiers.\n\ +\n\ +The first word of the input is matched to the first specifier of the\n\ +format and placed in the first column of the output; the second is matched\n\ +to the second specifier and placed in the second column and so forth.\n\ +If there are more words than specifiers, the process is repeated until all\n\ +words have been processed or the limit imposed by @var{repeat} has been met\n\ +(see below).\n\ +\n\ +The string @var{format} describes how the words in @var{str} should be\n\ +parsed. As in @var{fscanf},\n\ +any (non-whitespace) text in the format that is not one of these specifiers\n\ +is considered a literal; if there is a literal between two format specifiers\n\ +then that same literal must appear in the input stream between the matching\n\ +words.\n\ +\n\ +The following specifiers are valid:\n\ +\n\ +@table @code\n\ +@item %f\n\ +@itemx %f64\n\ +@itemx %n\n\ +The word is parsed as a number and converted to double.\n\ +\n\ +@item %f32\n\ +The word is parsed as a number and converted to single (float).\n\ +\n\ +@item %d\n\ +@itemx %d8\n\ +@itemx %d16\n\ +@itemx %d32\n\ +@itemx %d64\n\ +The word is parsed as a number and converted to int8, int16, int32 or int64.\n\ +If not size is specified, int32 is used.\n\ +\n\ +@item %u\n\ +@itemx %u8\n\ +@itemx %u16\n\ +@itemx %u32\n\ +@itemx %u64\n\ +The word is parsed as a number and converted to uint8, uint16, uint32 or\n\ +uint64. If not size is specified, uint32 is used.\n\ +\n\ +@item %s\n\ +The word is parsed as a string, ending at the last character before\n\ +whitespace, an end-of-line or a delimiter specified in the options.\n\ +\n\ +@item %q\n\ +The word is parsed as a \"quoted string\". If the first character of the\n\ +string is a double quote (\") then the string includes everything until a\n\ +matching double quote, including whitespace, delimiters and end of line\n\ +characters. If a pair of consecutive double quotes appears in the input,\n\ +it is replaced in the output by a single double quote. That is, the input\n\ +\"He said \"\"Hello\"\"\" would return the value 'He said \"Hello\"'.\n\ +\n\ +@item %c\n\ +The next character of the input is read. This includes delimiters,\n\ +whitespace and end of line characters.\n\ +\n\ +@item %[...]\n\ +@itemx %[^...]\n\ +In the first form, the word consists of the longest run consisting of only\n\ +characters between the brackets. Ranges of characters can be specified by\n\ +a hyphen; for example, %[0-9a-zA-Z] matches all alphanumeric characters\n\ +(if the underlying character set is ASCII). To include '-' in the set,\n\ +it should appear first or last in the brackets; to include ']', it should\n\ +be the first character. If the first character is '^' then the word\n\ +consists of characters NOT listed.\n\ +\n\ +For %s, %c %d, %f, %n, %u, an\n\ +optional width can be specified as %Ns, etc. where N is an integer > 1.\n\ +For %c, this causes exactly the next N characters to be read instead of\n\ +a single character. For the other specifiers, it is an upper bound on the\n\ +number of characters read; normal delimiters can cause fewer characters to\n\ +be read. For complex numbers, this limit applies to the real and imaginary\n\ +components individually\n\ +For %f and %n, format specifiers like %N.Mf are allowed, where M is an upper\n\ +bound on number of characters after the decimal point to be considered;\n\ +subsequent digits are skipped. For example, the specifier %8.2f would\n\ +read 12.345e6 as 1.234e7.\n\ +\n\ +@item %*...\n\ +The word specified by the remainder of the conversion specifier is skipped.\n\ +\n\ +@item literals\n\ +In addition the format may contain literal character strings; these will be\n\ +skipped during reading. If the input string does not match this literal,\n\ +the processing terminates.\n\ +@end table\n\ +\n\ +Parsed words corresponding to the first specifier are returned in the first\n\ +output argument and likewise for the rest of the specifiers.\n\ +\n\ +By default, @var{format} is @t{\"%f\"}, meaning that numbers are read from\n\ +@var{str}. This will do if @var{str} contains only numeric fields.\n\ +\n\ +For example, the string\n\ +\n\ +@example\n\ +@group\n\ +@var{str} = \"\\\n\ +Bunny Bugs 5.5\n\\\n\ +Duck Daffy -7.5e-5\n\\\n\ +Penguin Tux 6\"\n\ +@end group\n\ +@end example\n\ +\n\ +@noindent\n\ +can be read using\n\ +\n\ +@example\n\ +[@var{a}] = textscan (@var{str}, \"%s %s %f\");\n\ +@end example\n\ +\n\ +Optional numeric argument @var{repeat} can be used for limiting the\n\ +number of items read:\n\ +\n\ +@table @asis\n\ +@item -1\n\ +(default) read all of the string or file until the end.\n\ +\n\ +@item N\n\ +Read until the first of two conditions occurs: the format has been processed\n\ +N times, or N lines of the input have been processed. Zero (0) is an\n\ +acceptable value for @var{repeat}. Currently, end-of-line characters\n\ +inside %q, %c, and %[...]$ conversions do not contribute to the line count.\n\ +This is incompatible with Matlab and may change in future.\n\ +@end table\n\ +\n\ +The behavior of @code{textscan} can be changed via property-value pairs. The\n\ +following properties are recognized:\n\ +\n\ +@table @asis\n\ +@item @qcode{\"CollectOutput\"}\n\ +A value of 1 or true instructs textscan to concatenate consecutive columns\n\ +of the same class in the output cell array. A value of 0 or false (default)\n\ +leaves output in distinct columns.\n\ +\n\ +@item @qcode{\"CommentStyle\"}\n\ +Parts of @var{str} are considered comments and will be skipped.\n\ +@var{value} is the comment style and can be\n\ +either\n\ +@c any of the following.\n\ +@c \n\ +@c @itemize\n\ +@c @item @qcode{\"shell\"}\n\ +@c Everything from @code{#} characters to the nearest end-of-line is skipped.\n\ +@c \n\ +@c @item @qcode{\"c\"}\n\ +@c Everything between @code{/*} and @code{*/} is skipped.\n\ +@c \n\ +@c @item @qcode{\"c++\"}\n\ +@c Everything from @code{//} characters to the nearest end-of-line is\n\ +@c skipped.\n\ +@c \n\ +@c @item @qcode{\"matlab\"}\n\ +@c Everything from @code{%} characters to the nearest end-of-line is skipped.\n\ +@c \n\ +@c @item user-supplied. Two options:\n\ +(1) One string, or 1x1 cell string, to skip everything to the right of it;\n\ +(2) A cell array of two strings, to skip everything between the first and\n\ +second strings.\n\ +@c @end itemize\n\ +Comments are only parsed where whitespace is accepted, and do not act as\n\ +delimiters.\n\ +\n\ +@item @qcode{\"Delimiter\"}\n\ +If @var{value} is a string, any character in @var{value} will be used to\n\ +split @var{str} into words. If @var{value} is a cell array of strings,\n\ +any string in the array will be used to split @var{str} into words.\n\ +(default value = any whitespace).\n\ +\n\ +@item @qcode{\"EmptyValue\"}\n\ +Value to return for empty numeric values in non-whitespace delimited data.\n\ +The default is NaN@. When the data type does not support NaN (int32 for\n\ +example), then default is zero.\n\ +\n\ +@item @qcode{\"EndOfLine\"}\n\ +@var{value} can be either a emtpy or one character specifying the end of line\n\ +character, or the pair\n\ +@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} (CRLF).\n\ +In the latter case, any of\n\ +@qcode{\"@xbackslashchar{}r\"}, @qcode{\"@xbackslashchar{}n\"} or\n\ +@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is counted as a (single)\n\ +newline.\n\ +If no value is given, @qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is\n\ +used.\n\ +@c If set to \"\" (empty string) EOLs are ignored as delimiters and added\n\ +@c to whitespace.\n\ +\n\ +@c When reading from a character string, optional input argument @var{n}\n\ +@c specifies the number of times @var{format} should be used (i.e., to limit\n\ +@c the amount of data read).\n\ +@c When reading from file, @var{n} specifies the number of data lines to read;\n\ +@c in this sense it differs slightly from the format repeat count in strread.\n\ +\n\ +@item @qcode{\"HeaderLines\"}\n\ +The first @var{value} number of lines of @var{fid} are skipped.\n\ +\n\ +@item @qcode{\"MultipleDelimsAsOne\"}\n\ +If @var{value} is non-zero,\n\ +treat a series of consecutive delimiters, without whitespace in between,\n\ +as a single delimiter. Consecutive delimiter series need not be vertically\n\ +@qcode{\"aligned\"}.\n\ +Without this option, a single delimiter before the end of the line does\n\ +not cause the line to be considered to start with an empty value, but a\n\ +single delimiter at the start of a line causes the line to be considered\n\ +to start with an empty value.\n\ +\n\ +@item @qcode{\"TreatAsEmpty\"}\n\ +Treat single occurrences (surrounded by delimiters or whitespace) of the\n\ +string(s) in @var{value} as missing values.\n\ +\n\ +@item @qcode{\"ReturnOnError\"}\n\ +If set to numerical 1 or true (default), return normally when read errors\n\ +have been encountered. If set to 0 or false, return an error and no data.\n\ +As the string or file is read by columns rather than by rows, and because\n\ +textscan is fairly forgiving as regards read errors, setting this option\n\ +may have little or no actual effect.\n\ +\n\ +@item @qcode{\"Whitespace\"}\n\ +Any character in @var{value} will be interpreted as whitespace and trimmed;\n\ +The default value for whitespace is\n\ +@c Note: the next line specifically has a newline which generates a space\n\ +@c in the output of qcode, but keeps the next line < 80 characters.\n\ +@qcode{\"\n\ +@xbackslashchar{}b@xbackslashchar{}r@xbackslashchar{}n@xbackslashchar{}t\"}\n\ +(note the space). Unless whitespace is set to @qcode{\"\"} (empty) AND at\n\ +least one @qcode{\"%s\"} format conversion specifier is supplied, a space is\n\ +always part of whitespace.\n\ +\n\ +@end table\n\ +\n\ +When the number of words in @var{str} or @var{fid} doesn't match an exact\n\ +multiple of\n\ +the number of format conversion specifiers,\n\ +textscan fills the last line with empty strings, NaNs or 0s (for integer\n\ +fields). In the case that the last line does not end with an end-of-line\n\ +character, this is incompatible with Matlab and subject to change.\n\ +@c textscan's behavior depends on\n\ +@c whether the last line of the string or file is an end-of-line as specified\n\ +@c by the EndOfLine option:\n\ +@c \n\ +@c @table @asis\n\ +@c @item last character = end-of-line\n\ +@c Data columns are padded with empty fields, NaN or 0 so that all columns have\n\ +@c equal length\n\ +@c \n\ +@c @item last character is not end-of-line\n\ +@c Data columns are not padded; textscan returns columns of unequal length\n\ +@c @end table\n\ +\n\ +If the format string is explicitly empty (not omitted) and the file contains\n\ +only\n\ +numeric data (excluding headerlines), textscan will return data in a number\n\ +of columns matching the number of fields on the first data line of\n\ +the input.\n\ +\n\ +The second output, @var{position}, provides the position, in characters\n\ +from the beginning of the file or string, at which the processing stopped.\n\ +\n\ +@seealso{dlmread, fscanf, load, strread, textread}\n\ +@end deftypefn") +{ + octave_value_list retval; + std::string format; + int params = 1; + + if (args.length () < 1) + { + print_usage (); + error_state = true; + } + else if (args.length () == 1) + format = "%f"; // ommited format = %f. explicit "" = width from file + else if (args(1).is_string ()) + { + format = args(1).string_value (); + params++; + } + else + error ("text:scan: FORMAT must be a string, not <%s>", + args(1).class_name ().c_str ()); + + if (! error_state) + { + octave_idx_type ntimes = -1; + textscan tscanner; + + if (args.length () >= 3) + { + if (args(2).is_numeric_type ()) + { + ntimes = args(2).idx_type_value (); + params = 3; + } + } + textscan_format_list fmt_list (format); + + tscanner.parse_options (args, params, fmt_list); + + fmt_list.consider_options (tscanner); + + if (! error_state) + { + if (args(0).is_string ()) + { + std::istringstream is (args(0).string_value ()); + octave_value tmp = tscanner.scan (&is, fmt_list, ntimes); + if (! error_state) + { + retval(0) = tmp; + + std::ios::iostate state = is.rdstate (); + is.clear (); + retval(1) = octave_value (static_cast(is.tellg ())); + is.setstate (state); + } + } + else + { + octave_stream os = octave_stream_list::lookup (args(0), + "textscan"); + octave_value tmp = tscanner.scan (os.input_stream (), + fmt_list, ntimes); + + if (! error_state) + { + retval(0) = tmp; + // FIXME -- warn if stream is not opened in binary mode? + std::ios::iostate state = os.input_stream ()->rdstate (); + os.input_stream ()->clear (); + retval(1) = os.tell (); + os.input_stream ()->setstate (state); + } + } + } + } + + return retval; +} + + +/********************/ +/* Read a double considering the "precision" field of fmt and the */ +/* exp_chars option of options. */ +double +textscan::formatted_double (std::istream& is, const textscan_format_elt& fmt) + const +{ + int ch = is.peek (); + int sign = 1; + unsigned int width_left = fmt.width; + double retval = 0; + bool valid = false; // syntactically correct double? + + if (ch == '+') + { + is.get (); + ch = is.peek (); + if (width_left) + width_left--; + } + else if (ch == '-') + { + sign = -1; + is.get (); + ch = is.peek (); + if (width_left) + width_left--; + } + + // Read integer part + if (ch != '.') + { + if (ch >= '0' && ch <= '9') // valid if at least one digit + valid = true; + while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9') + retval = retval * 10 + (ch - '0'); + width_left++; + } + // Read fractional part, up to specified precision + if (ch == '.' && width_left) + { + double multiplier = 1; + int precision = fmt.prec; + int i; + + if (width_left) + width_left--; // Consider width of '.' + + if (precision == -1) + precision = 1<<30; // FIXME Should be MAXINT + if (!valid) // if there was nothing before '.'... + is.get (); // ...ch was a "peek", not "get". + + for (i = 0; i < precision; i++) + { + if (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9') + retval += (ch - '0') * (multiplier *= 0.1); + else + { + width_left++; + break; + } + } + + // round up if we truncated and the next digit is >= 5 + if ((i == precision || !width_left) && (ch = is.peek ()) >= '5' + && ch <= '9') + retval += multiplier; + + if (i > 0) + valid = true; // valid if at least one digit after '.' + + // skip remainder after '.', to field width, to look for exponent + while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9') + ; /* discard */ + width_left++; + } + + // look for exponent part in, e.g., 6.023E+23 + const char *ec = exp_chars.c_str (); + bool used_exp = false; + if (valid && width_left > 1 && strchr (ec, ch)) + { + int ch1 = is.peek (); + if (ch1 == '-' || ch1 == '+' || (ch1 >= '0' && ch1 <= '9')) + { // if 1.0e+$ or some such, this will set failbit, as we want + width_left--; // count "E" + int exp = 0; + int exp_sign = 1; + if (ch1 == '+') + { + if (width_left) + width_left--; + is.get (); + } + else if (ch1 == '-') + { + exp_sign = -1; + is.get (); + if (width_left) + width_left--; + } + valid = false; + while (width_left-- && is && (ch = is.get ()) >= '0' && ch1 <= '9') + { + exp = exp*10 + ch - '0'; + valid = true; + } + width_left++; + if (ch != EOF && width_left) + is.putback (ch); + + if (exp > 5) + retval *= pow10 (exp * exp_sign); + else // repeated mult/div is more accurate + { + int multiplier = 1; + for (int i = 0; i < exp; i++) + multiplier *= 10; + if (exp_sign > 0) + retval *= multiplier; + else + retval /= multiplier; + } + used_exp = true; + } + } + is.clear (); + if (!used_exp && ch != EOF && width_left) + is.putback (ch); + + if (!valid) + is.setstate (std::ios_base::failbit); + else + is.setstate (is.rdstate () & ~std::ios_base::failbit); + + return retval * sign; +} + +/********************/ +std::istream& +textscan::scan_complex (std::istream& is, const textscan_format_elt& fmt, + Complex& val) const +{ + double im = 0; + double re = 0; + bool as_empty = false; // did we fail but match a "treat_as_empty" string? + + int ch = is.peek (); + + if (ch == '+' || ch == '-') // check for [+-][ij] with no coefficients + { + ch = is.get (); + int ch2 = is.peek (); + if (ch2 == 'i' || ch2 == 'j') + { + is.get (); + im = (ch == '+') ? 1 : -1; + } + else + is.putback (ch); + } + + if (!im) // if not [+-][ij], read real normally + { + std::streampos pos = is.tellg (); + std::ios::iostate state = is.rdstate (); + //re = octave_read_value (is); + re = read_double (is, fmt); + + // check for "treat as empty" string + if (treat_as_empty.numel () + && (is.fail () || octave_is_NaN_or_NA (Complex (re)) + || re == octave_Inf)) + { + + for (int i = 0; i < treat_as_empty.numel (); i++) + { + if (ch == treat_as_empty (i).string_value ()[0]) + { + as_empty = true; // first char matches, so read the lot + break; + } + } + if (as_empty) // if first char matched... + { + as_empty = false; // ...look for the whole string + + is.clear (state); // treat_as_empty "-" causes partial read + is.seekg (pos); // reset to position before failed read, since + + // treat_as_empty strings may be different sizes. + // Read ahead longest, put it all back, then re-read the string + // that matches. + char look [treat_as_empty_len + 1]; + // prefill, in case EOF means part-filled. + memset (look, '\0', treat_as_empty_len); + is.read (look, treat_as_empty_len); + + is.clear (state); + is.seekg (pos); // reset to position before look-ahead + + for (int i = 0; i < treat_as_empty.numel (); i++) + { + std::string s = treat_as_empty (i).string_value (); + if (!strncmp (s.c_str (), look, s.size ())) + { + as_empty = true; + is.read (look, s.size ()); // read just the right amount + // FIXME Could this be combined with the above seek? + break; + } + } + } + } + + if (!is.eof () && !as_empty) + { + state = is.rdstate (); // before tellg, since that fails at EOF + pos = is.tellg (); + ch = is.peek (); // ch == EOF if read failed; no need to chk fail + if (ch == 'i' || ch == 'j') // pure imaginary + { + is.get (); + im = re; + re = 0; + } + else if (ch == '+' || ch == '-') // see if it is real+imag[ij] + { + // save stream state in case we have to restore it + pos = is.tellg (); + state = is.rdstate (); + + //im = octave_read_value (is); + im = read_double (is, fmt); + if (is.fail ()) + im = 1; + + if (is.peek () == 'i' || is.peek () == 'j') + is.get (); + else + { + im = 0; // no valid imaginary part. Restore state + is.clear (state); + is.seekg (pos); + } + } + else if (ch == EOF) // we've read enough to be a "valid" read + is.clear (state); // failed peek shouldn't cause fail + } + } + if (as_empty) + val = empty_value.scalar_value (); + else + val = Complex (re, im); + + return is; +} + +/********************/ +// Return in val the run of characters from is NOT contained in pattern. +int +textscan::scan_caret (std::istream& is, const char *pattern, std::string& val) + const +{ + int c1 = EOF; + std::ostringstream obuf; // Is this optimised for growing? + + while (is && (c1 = (is && !is.eof ()) ? is.get () : EOF) != EOF + && !strchr (pattern, c1)) + obuf << static_cast (c1); + + val = obuf.str (); + + // can only push back if not failed. + is.clear (is.rdstate () & ~std::ios_base::failbit); + if (c1 != EOF) + is.putback (c1); + + return c1; +} + +/********************/ +// Read until one of the strings in delimiters is found. +// For efficiency, ends is a list of the last character of each delimiter +std::string +textscan::read_until (std::istream& is, const Cell& delimiters, + const std::string& ends) const +{ + std::string retval (""); + bool done = false; + do + { // find sequence ending with an ending char + std::string next; + scan_caret (is, ends.c_str (), next); + retval = retval + next; + + int last = (!is.eof ()) ? is.get () : EOF; + if (last != EOF) + { + retval = retval + static_cast (last); + for (int i = 0; i < delimiters.numel (); i++) + { + std::string delim = delimiters(i).string_value (); + int start = retval.length () - delim.length (); + if (start < 0) + start = 0; + std::string may_match = retval.substr (start); + if (may_match == delim) + { + done = true; + retval = retval.substr (0, start); + break; + } + } + } + } + while (!done && is && !is.eof ()); + + return retval; +} + + +/********************/ +// Read stream until either fmt.width chars have been read, or options.delimiter +// has been found +// Does *not* rely on fmt being 's' -- used by formats like %6f to limit to 6 +std::istream& +textscan::scan_string (std::istream& is, const textscan_format_elt& fmt, std::string& val) const +{ + if (delim_list.numel () == 0) + { + unsigned int i = 0; + unsigned int width = fmt.width; + + for (i = 0; i < width; i++) + { + if (i+1 > val.length ()) + val = val + val + ' '; // grow even if empty + int ch = is.get (); + if (is_delim (ch) || ch == EOF) + { + is.putback (ch); + break; + } + else + val[i] = ch; + } + val = val.substr (0, i); // trim pre-allocation + } + else // Cell array of multi-character delimiters + { + std::string ends (""); + for (int i = 0; i < delim_list.numel (); i++) + { + std::string tmp = delim_list(i).string_value (); + ends += tmp.substr (tmp.length () - 1); + } + val = textscan::read_until (is, delim_list, ends); + } + + return is; +} + +/********************/ +// Return in val the run of characters from is contained in pattern. +int +textscan::scan_bracket (std::istream& is, const char *pattern, std::string& val) + const +{ + int c1 = EOF; + std::ostringstream obuf; // Is this optimised for growing? + + while (is && strchr (pattern, (c1 = is.get ()))) + obuf << static_cast (c1); + + val = obuf.str (); + if (c1 != EOF) + is.putback (c1); + return c1; +} + +/********************/ +// Return in val a string, either delimited by whitespace/delimiters, or +// enclosed in a pair of double quotes ("..."). Enclosing quotes are removed. +// A consecutive pair "" is inserted into val as a single ". +std::istream& +textscan::scan_qstring (std::istream& is, const textscan_format_elt& fmt, + std::string& val) +{ + skip_whitespace (is); + + if (is.peek () != '\"') + scan_string (is, fmt, val); + else + { + is.get (); + scan_caret (is, "\"", val); // read everything until " + is.get (); // swallow " + + while (is && is.peek () == '\"') // if double ", insert one in stream, + { // and keep looking for single " + is.get (); + std::string val1; + scan_caret (is, "\"", val1); + val = val + "\"" + val1; + is.get (); + } + } + return is; +} + +/********************/ +// Read from is into val a string of the next fmt.width characters, +// including any whitespace or delimiters. +std::istream& +textscan::scan_cstring (std::istream& is, const textscan_format_elt& fmt, + std::string& val) const +{ + int ch; + val.resize (fmt.width); + for (unsigned int i = 0; is && i < fmt.width; i++) + if ((ch = is.get ()) != EOF) + val[i] = ch; + else + { + val.resize (i); + break; + } + + return is; +} + + +/********************/ +std::istream& +textscan::scan_one (std::istream& is, const textscan_format_elt& fmt, + octave_value& ov, Array row) +{ + skip_whitespace (is); + + is.clear (); + + octave_value val; + if (fmt.numeric) + { + if (fmt.type == 'f' || fmt.type == 'n') + { + Complex v; + scan_complex (is, fmt, v); + + if (!fmt.discard) + { + if (fmt.bitwidth == 64) + { + if (ov.is_real_type () && v.imag () == 0) + ov.internal_rep ()->fast_elem_insert (row(0), v.real ()); + else + { + if (ov.is_real_type ()) // cat does type conversion + ov = do_cat_op (ov, octave_value (v), row); + else + ov.internal_rep ()->fast_elem_insert (row(0), v); + } + } + else + { + if (ov.is_real_type () && v.imag () == 0) + ov.internal_rep ()->fast_elem_insert (row(0), + float (v.real ())); + else + { + if (ov.is_real_type ()) // cat does type conversion + ov = do_cat_op (ov, octave_value (v), row); + else + ov.internal_rep ()->fast_elem_insert (row(0), + FloatComplex (v)); + } + } + } + } + else + { + double v; // Matlab docs say 1e30 etc should be valid for %d and + // 1000 as a %d8 should be 127, so read as double. + // Some loss of precision for d64 and u64. + v = read_double (is, fmt); + if (!fmt.discard) + switch (fmt.bitwidth) + { + case 64: + switch (fmt.type) + { + case 'd': + ov.internal_rep ()->fast_elem_insert (row(0), + octave_int64 (v)); + break; + case 'u': + ov.internal_rep ()->fast_elem_insert (row(0), + octave_uint64 (v)); + break; + } + break; + case 32: + switch (fmt.type) + { + case 'd': + { + octave_int32 vv = v; + if (vv < v) + vv = (1L<<31)-1; + else if (vv > v) + vv = (1<<31); + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + break; + case 'u': + { + octave_uint32 vv = v; + if (vv < v) + vv = -1; + else if (vv > v) + vv = 0; + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + break; + } + break; + case 16: + if (fmt.type == 'd') + { + octave_int16 vv = v; + if (vv < v) + vv = (1<<15)-1; + else if (vv > v) + vv = (1<<15); + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + else + { + octave_uint16 vv = v; + if (vv < v) + vv = -1; + else if (vv > v) + vv = 0; + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + break; + case 8: + if (fmt.type == 'd') + { + octave_int8 vv = v; + if (vv < v) + vv = (1<<7)-1; + else if (vv > v) + vv = (1<<7); + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + else + { + octave_uint8 vv = v; + if (vv < v) + vv = -1; + else if (vv > v) + vv = 0; + ov.internal_rep ()->fast_elem_insert (row(0), vv); + } + break; + } + } + } + else + { + std::string vv (" "); // initial buffer. Grows as needed + switch (fmt.type) + { + case 's': + scan_string (is, fmt, vv); + break; + case 'q': + scan_qstring (is, fmt, vv); + break; + case 'c': + scan_cstring (is, fmt, vv); + break; + case '[': + scan_bracket (is, fmt.char_class.c_str (), vv); + break; + case '^': + scan_caret (is, fmt.char_class.c_str (), vv); + break; + } + if (!fmt.discard) + ov.internal_rep ()->fast_elem_insert (row (0), + Cell (octave_value (vv))); + // fixme: why does failbit get set at EOF, instead of eofbit?? + if (vv.length () != 0) + is.clear (is.rdstate () & ~std::ios_base::failbit); + } + + if (is.fail () && fmt.numeric) + { + if (!fmt.discard) + ov = do_cat_op (ov, empty_value, row); + std::string dummy; // FIXME: should fail instead of skipping + scan_string (is, fmt, dummy); + } + + return is; +} + +/********************/ +int +textscan::read_format_once (std::istream* isp, + textscan_format_list& fmt_list, + std::list & retval, + Array row) +{ + const textscan_format_elt *elem = fmt_list.first (); + std::list::iterator out=retval.begin (); + bool no_conversions = true; + bool done = false; + int i; + + for (i = 0; i < fmt_list.numel (); i++) + { + isp->clear (); // clear fail of previous numeric conversions + switch (elem->type) + { + case 'C': + case 'D': + std::cerr << "textscan: Conversion %" << elem->type + << " not yet implemented\n"; + break; + + case 'u': + case 'd': + case 'f': + case 'n': + case 's': + case '[': + case '^': + case 'q': + case 'c': + scan_one (*isp, *elem, *out, row); + break; + + case textscan_format_elt::literal_conversion : + match_literal (*isp, *elem); + break; + } + + if (!isp->fail () && !elem->discard) + no_conversions = false; + if (isp->eof ()) + done = true; // note EOF, but process others to get empty_val + else + isp->clear (isp->rdstate () & ~std::ios::failbit); + if (!elem->discard) + out++; + + elem = fmt_list.next (); + // fixme: these conversions "ignore delimiters". Should they include + // delimiters at the start of the conversion, or can those be skipped? + if (// elem->type != '[' && elem->type != '^' && elem->type != 'c' + /*&&*/ elem->type != textscan_format_elt::literal_conversion) + skip_delim (*isp); + + if (error_state) + break; + } + if (done) + isp->setstate (std::ios::eofbit); + + // err = -2 means "error, and no columns read this row + if (error_state || isp->eof ()) + return (-1 - no_conversions); + + return 1; +} + +/********************/ +void +textscan::parse_options (const octave_value_list& args, int first_param, + textscan_format_list& fmt_list) +{ + int last = args.length (); + int n = last - first_param; + + if (n & 1) + error ("textscan: %d parameters given, but only %d values", n-n/2, n/2); + else + { + std::string delims; + bool have_delims = false; + for (int i = first_param; i < last; i += 2) + { + if (!args(i).is_string ()) + { + error ("textscan: Invalid paramter type <%s> for parameter %d", + args(i).type_name ().c_str (), (i-first_param)/2 + 1); + } + else + { + std::string param = args(i).string_value (); + std::transform (param.begin (), param.end (), + param.begin (), ::tolower); + if (param == "delimiter") + { + bool invalid = true; + if (args(i+1).is_string ()) + { + invalid = false; + have_delims = true; + delims = args(i+1).string_value (); + } + else if (args(i+1).is_cell ()) + { + invalid = false; + delim_list = args(i+1).cell_value (); + delim_table = " "; // non-empty, to flag non-default delim + + // Check that all elements are strings + for (int j = 0; j < delim_list.numel (); j++) + if (!delim_list (j).is_string ()) + invalid = true; + } + if (invalid) + error ("textscan: Delimimters must be either a string" + " or cell array of strings"); + } + else if (param == "commentstyle") + { + if (args(i+1).is_string ()) + { // check here for names like "C++", "C", "shell", ...? + comment_style = Cell (args(i+1)); + } + else if (args(i+1).is_cell ()) + { + comment_style = args(i+1).cell_value (); + int len = comment_style.numel (); + if ((len >= 1 && !comment_style (0).is_string ()) + || (len >= 2 && !comment_style (1).is_string ()) + || (len >= 3)) + error ("textscan: CommentStyle must be either a string" + " or cell array of one or two strings"); + } + else + error ("textscan: CommentStyle must be either a string" + " or cell array of one or two strings, not <%s>", + args(i+1).class_name ().c_str ()); + + // How far ahead do we need to look to detect an open comment + // and which character do we look for? + if (comment_style.numel () >= 1) + { + comment_len = comment_style (0).string_value ().size (); + comment_char = comment_style (0).string_value ()[0]; + } + } + else if (param == "treatasempty") + { + bool invalid = false; + if (args(i+1).is_string ()) + { + treat_as_empty = Cell (args(i+1)); + treat_as_empty_len = args(i+1).string_value ().size (); + } + else if (args(i+1).is_cell ()) + { + treat_as_empty = args(i+1).cell_value (); + for (int j = 0; j < treat_as_empty.numel (); j++) + if (!treat_as_empty (j).is_string ()) + invalid = true; + else + { + int k = treat_as_empty (j).string_value ().size (); + if (k > treat_as_empty_len) + treat_as_empty_len = k; + } + } + if (invalid) + error ("textscan: TreatAsEmpty must be either a string" + " or cell array of one or two strings"); + else + { + // FIXME Ensure none is a prefix of a later one. sort len? + } + } + else if (param == "collectoutput") + { + if (args(i+1).is_numeric_type ()) + collect_output = args(i+1).bool_value (); + else + error ("textscan: CollectOutput must be logical or numeric"); + } + else if (param == "emptyvalue") + { + if (args(i+1).is_numeric_type ()) + empty_value = args(i+1).scalar_value (); + else + error ("textscan: EmptyValue must be numeric, not <%s>", + args(i+1).class_name ().c_str ()); + } + else if (param == "headerlines") + { + if (args(i+1).is_numeric_type ()) + header_lines = args(i+1).scalar_value (); + else + error ("textscan: HeaderLines must be numeric"); + } + else if (param == "multipledelimsasone") + { + if (args(i+1).is_numeric_type ()) + { + if (args(i+1).bool_value ()) + multiple_delims_as_one = true; + } + else + error ("textscan: MultipleDimsAsOne must be logical or numeric"); + } + else if (param == "returnonerror") + { + if (args(i+1).is_numeric_type ()) + { + if (!args(i+1).bool_value ()) + return_on_error = false; + } + else + error ("textscan: ReturnOnError must be logical or numeric"); + } + else if (param == "whitespace") + { + if (args(i+1).is_string ()) + whitespace = args(i+1).string_value (); + else + error ("textscan: Whitespace must be a character string"); + } + else if (param == "expchars") + { + if (args(i+1).is_string ()) + { + exp_chars = args(i+1).string_value (); + default_exp = false; + } + else + error ("textscan: ExpChars must be a character string"); + } + else if (param == "endofline") + { + bool valid = true; + if (args(i+1).is_string ()) + { + std::string s = args(i+1).string_value (); + if (args(i+1).is_sq_string ()) + s = do_string_escapes (s); + int l = s.length (); + if (l == 0) + eol1 = eol2 = -2; + else if (l == 1) + eol1 = eol2 = s.c_str ()[0]; + else if (l == 2) + { + eol1 = s.c_str ()[0]; + eol2 = s.c_str ()[1]; + if (eol1 != '\r' || eol2 != '\n') // Why limit it? + valid = false; + } + else + valid = false; + } + else + valid = false; + if (!valid) + error ("textscan: EndOfLine must be at most one character or '\\r\\n'"); + } + else + { + error ("textscan: Unrecognised option '%s'", param.c_str ()); + } + } + } + + // Create look-up table of delimiters, based on 'delimiter' + if (have_delims) + { + delim_table = std::string (256, '\0'); + for (unsigned int i = 0; i < delims.length (); i++) + delim_table[delims[i]] = '1'; + delim_table[eol1] = '1'; // EOL is always a delimiter + delim_table[eol2] = '1'; // EOL is always a delimiter + } + + whitespace_table = std::string (256, '\0'); + for (unsigned int i = 0; i < whitespace.length (); i++) + whitespace_table[whitespace[i]] = '1'; + + // For Matlab compatibility, add 0x20 to whitespace, unless + // whitespace is explicitly ignored. + if (! (whitespace.length () == 0 && fmt_list.has_string)) + whitespace_table[' '] = '1'; + +// // Add CR and LF to whitespace if they are not end-of-line. +// if (!strchr (end_of_line.c_str (), '\r')) +// whitespace_table['\r'] = '1'; +// if (!strchr (end_of_line.c_str (), '\n')) +// whitespace_table['\n'] = '1'; + } +} + +/********************/ +// skip comments, and characters specified by the "Whitespace" option +// If EOLstop==true, doesn't skip end of line +int +textscan::skip_whitespace (std::istream& is, bool EOLstop) +{ + int c1 = EOF; + bool found_comment = false; + + do + { + found_comment = false; + int prev = -1; + while (is && (c1 = is.get ()) != EOF + && ( ( (c1 == eol1 || c1 == eol2) && ++lines && !EOLstop) + || isspace (c1))) + { + if (prev == eol1 && eol1 != eol2 && c1 == eol2) + lines--; + prev = c1; + } + + if (c1 == comment_char) // see if we match an open comment + { + // save stream state in case we have to restore it + std::streampos pos = is.tellg (); + std::ios::iostate state = is.rdstate (); + + char look [comment_len]; + is.read (look, comment_len-1); // already read first char + look[comment_len-1] = '\0'; + if (is && comment_style(0).string_value ().substr (1) == look) + { + found_comment = true; + + std::string dummy; + char eol [3] = {static_cast (eol1), + static_cast (eol2), + '\0'}; + if (comment_style.numel () == 1) // skip to end of line + { + scan_caret (is, eol, dummy); + c1 = is.get (); + if (c1 == eol1 && eol1 != eol2 && is.peek () == eol2) + is.get (); + lines++; + } + else // matching pair + { + std::string end_c = comment_style(1).string_value (); + // last char of end-comment sequence + char last = *(end_c.substr (end_c.length ()-1).c_str ()); + std::string may_match (""); + do + { // find sequence ending with last char + scan_caret (is, &last, dummy); + is.get (); // (read last itself) + + may_match = may_match + dummy + last; + int start = may_match.length () - end_c.length (); + if (start < 0) + start = 0; + may_match = may_match.substr (start); + } + while (may_match != end_c && is && !is.eof ()); + } + } + else // wasn't really a comment; restore state + { + is.clear (state); + is.seekg (pos); + } + } + } + while (found_comment); + + if (c1 != EOF) + is.putback (c1); + return c1; +} + +/********************/ +// See if the next few characters match one of the strings in target. +// For efficiency, max_len is the cached longest length of any target. +// Returns -1 if none is found, or the inde of the match. +int +textscan::lookahead (std::istream& is, const Cell& targets, int max_len) const +{ + // target strings may be different sizes. + // Read ahead longest, put it all back, then re-read the string + // that matches. + + std::streampos pos = is.tellg (); + std::ios::iostate state = is.rdstate (); + + char look [max_len + 1]; + + memset (look, '\0', max_len); // prefill, in case EOF means part-filled. + is.read (look, max_len); + + is.clear (state); + is.seekg (pos); // reset to position before look-ahead + + int i; + for (i = 0; i < targets.numel (); i++) + { + std::string s = targets (i).string_value (); + if (!strncmp (s.c_str (), look, s.size ())) + { + is.read (look, s.size ()); // read just the right amount + break; + } + } + + if (i == targets.numel ()) + i = -1; + + return i; +} + +/********************/ +// Skip delimiters -- multiple if MultipleDelimsAsOne specified. +int +textscan::skip_delim (std::istream& is) +{ + int c1 = skip_whitespace (is, true); // 'true': stop once EOL is read + if (delim_list.numel () == 0) // single character delimiter + { + if (is_delim (c1) || c1 == eol1 || c1 == eol2) + { + is.get (); + if (c1 == eol1 && is.peek () == eol2) + is.get (); // if \r\n, skip the \n too. + + if (multiple_delims_as_one) + { + int prev = -1; + // skip multiple delims. + // Increment lines for each end-of-line seen; for \r\n, decrement + while (is && (c1 = is.get ()) != EOF + && (((c1 == eol1 || c1 == eol2) && ++lines) + || isspace (c1) || is_delim (c1))) + { + if (prev == eol1 && eol1 != eol2 && c1 == eol2) + lines--; + prev = c1; + } + if (c1 != EOF) + is.putback (c1); + } + } + } + else // multi-character delimiter + { + int first_match; + int max_len = 0; + for (int i = 0; i < delim_list.numel (); i++) + { + int len = delim_list(i).numel (); + if (len > max_len) + max_len = len; + } + + if (c1 == eol1 || c1 == eol2 + || (-1 != (first_match = lookahead (is, delim_list, max_len)))) + { + if (c1 == eol1) + { + is.get (); + if (is.peek () == eol2) + is.get (); + } + else if (c1 == eol2) + { + is.get (); + } + + if (multiple_delims_as_one) + { + int prev = -1; + // skip multiple delims. + // Increment lines for each end-of-line seen; for \r\n, decrement + while (is && (c1 == skip_whitespace (is, true)) != EOF + && (((c1 == eol1 || c1 == eol2) && ++lines) + || -1 != lookahead (is, delim_list, max_len))) + { + if (prev == eol1 && eol1 != eol2 && c1 == eol2) + lines--; + prev = c1; + } + } + } + } + + return c1; +} + +/********************/ +// Read in as much of the input as coincides with the literal in +// the format string. +// Return "true" if the entire literal is matched, else false (and set failbit) +bool +textscan::match_literal (std::istream& is, const textscan_format_elt& fmt) +{ + // "false" -> treat EOL as normal space + // since a delimiter at the start of a line is a mismatch, not empty field + skip_whitespace (is, false); + + for (unsigned int i = 0; i < fmt.width; i++) + { + int ch = is.get (); + if (ch != fmt.text[i]) + { + if (ch != EOF) + is.putback (ch); + is.setstate (std::ios_base::failbit); + return false; + } + } + return true; +} + + +/********************/ +textscan_format_list::textscan_format_list (const std::string& s) + : set_from_first (false), has_string (false), nconv (0), curr_idx (0), + list (dim_vector (16, 1)), buf (0) +{ + octave_idx_type num_elts = 0; + + size_t n = s.length (); + + size_t i = 0; + + unsigned int width = -1; // Unspecified width = max (except %c) + int prec = -1; + int bitwidth = 0; + bool discard = false; + char type = '\0'; + + bool have_more = true; + + if (s.length () == 0) + { + buf = new std::ostringstream ("%f"); + bitwidth = 64; + type = 'f'; + add_elt_to_list (width, prec, bitwidth, octave_value (NDArray ()), + discard, type, num_elts); + have_more = false; + set_from_first = true; + nconv = 1; + } + else + { + set_from_first = false; + while (i < n) + { + have_more = true; + + if (! buf) + buf = new std::ostringstream (); + + if (s[i] == '%' && (i+1 == n || s[i+1] != '%')) + { + // Process percent-escape conversion type. + + process_conversion (s, i, n, num_elts); + + have_more = (buf != 0); + } + else if (isspace (s[i])) + { + while (++i < n && isspace (s[i])) + /* skip whitespace */; + + have_more = false; + } + else + { + type = textscan_format_elt::literal_conversion; + + width = 0; + prec = -1; + bitwidth = 0; + discard = true; + + while (i < n && ! isspace (s[i]) + && (s[i] != '%' || (i+1 < n && s[i+1] == '%'))) + { + if (s[i] == '%') // if double %, skip one + i++; + *buf << s[i++]; + width++; + } + + add_elt_to_list (width, prec, bitwidth, octave_value (), discard, + type, num_elts); + + have_more = false; + } + + if (nconv < 0) + { + have_more = false; + break; + } + } + } + + if (have_more) + add_elt_to_list (width, prec, bitwidth, octave_value (), discard, type, num_elts); + + list.resize (dim_vector (num_elts, 1)); + + delete buf; +} + +/********************/ +textscan_format_list::~textscan_format_list (void) +{ + octave_idx_type n = list.numel (); + + for (octave_idx_type i = 0; i < n; i++) + { + textscan_format_elt *elt = list(i); + delete elt; + } +} + +/********************/ +void +textscan_format_list::add_elt_to_list (unsigned int width, int prec, + int bitwidth, octave_value val_type, + bool discard, char type, + octave_idx_type& num_elts, + const std::string& char_class) +{ + if (buf) + { + std::string text = buf->str (); + + if (! text.empty ()) + { + textscan_format_elt *elt + = new textscan_format_elt (text.c_str (), width, prec, bitwidth, + discard, type, char_class); + + if (num_elts == list.numel ()) + { + list.resize (dim_vector (2 * num_elts, 1)); + } + + if (!discard) + { + output_container.push_back (val_type); + } + list(num_elts++) = elt; + } + + delete buf; + buf = 0; + } +} + +/********************/ +void +textscan_format_list::process_conversion (const std::string& s, size_t& i, + size_t n, octave_idx_type& num_elts) +{ + unsigned width = 0; + int prec = -1; + int bitwidth = 0; + bool discard = false; + octave_value val_type; + char type = '\0'; + + *buf << s[i++]; + + bool have_width = false; + + while (i < n) + { + switch (s[i]) + { + case '*': + if (discard) + nconv = -1; + else + { + discard = true; + *buf << s[i++]; + } + break; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + if (have_width) + nconv = -1; + else + { + char c = s[i++]; + width = width * 10 + c - '0'; + have_width = true; + *buf << c; + while (i < n && isdigit (s[i])) + { + c = s[i++]; + width = width * 10 + c - '0'; + *buf << c; + } + + if (i < n && s[i] == '.') + { + *buf << s[i++]; + prec = 0; + while (i < n && isdigit (s[i])) + { + c = s[i++]; + prec = prec * 10 + c - '0'; + *buf << c; + } + } + } + break; + + case 'd': case 'u': + { + bool done = true; + *buf << (type = s[i++]); + if (i < n) + { + if (s[i] == '8') + { + bitwidth = 8; + if (type == 'd') + val_type = octave_value (int8NDArray ()); + else + val_type = octave_value (uint8NDArray ()); + *buf << s[i++]; + } + else if (s[i] == '1' && i+1 < n && s[i+1] == '6') + { + bitwidth = 16; + if (type == 'd') + val_type = octave_value (int16NDArray ()); + else + val_type = octave_value (uint16NDArray ()); + *buf << s[i++]; + *buf << s[i++]; + } + else if (s[i] == '3' && i+1 < n && s[i+1] == '2') + { + done = false; // use default size below + *buf << s[i++]; + *buf << s[i++]; + } + else if (s[i] == '6' && i+1 < n && s[i+1] == '4') + { + bitwidth = 64; + if (type == 'd') + val_type = octave_value (int64NDArray ()); + else + val_type = octave_value (uint64NDArray ()); + *buf << s[i++]; + *buf << s[i++]; + } + else + done = false; + } + else + done = false; + + if (!done) + { + bitwidth = 32; + if (type == 'd') + val_type = octave_value (int32NDArray ()); + else + val_type = octave_value (uint32NDArray ()); + } + goto fini; + } + + case 'f': + *buf << (type = s[i++]); + bitwidth = 64; + if (i < n) + { + if (s[i] == '3' && i+1 < n && s[i+1] == '2') + { + bitwidth = 32; + val_type = octave_value (FloatNDArray ()); + *buf << s[i++]; + *buf << s[i++]; + } + else if (s[i] == '6' && i+1 < n && s[i+1] == '4') + { + val_type = octave_value (NDArray ()); + *buf << s[i++]; + *buf << s[i++]; + } + else + val_type = octave_value (NDArray ()); + } + else + val_type = octave_value (NDArray ()); + goto fini; + + case 'n': + *buf << (type = s[i++]); + bitwidth = 64; + val_type = octave_value (NDArray ()); + goto fini; + + case 's': case 'q': case '[': case 'c': + if (!discard) + val_type = octave_value (Cell ()); + *buf << (type = s[i++]); + has_string = true; + goto fini; + + fini: + { + if (!have_width) + { + if (type == 'c') // %c defaults to one character + width = 1; + else + width = static_cast (-1); // others: unlimited + } + + if (finish_conversion (s, i, n, width, prec, bitwidth, val_type, + discard, type, num_elts) == 0) + return; + } + break; + + default: + error ("textscan: '%%%c' is not a valid format specifier", s[i]); + nconv = -1; + break; + } + + if (nconv < 0) + break; + } + + nconv = -1; +} + +/********************/ +int +textscan_format_list::finish_conversion (const std::string& s, size_t& i, + size_t n, unsigned int& width, + int& prec, int& bitwidth, + octave_value& val_type, bool discard, + char& type, octave_idx_type& num_elts) +{ + int retval = 0; + + std::string char_class; + + size_t beg_idx = std::string::npos; + size_t end_idx = std::string::npos; + + if (type != '%') + { + nconv++; + if (type == '[') + { + if (i < n) + { + beg_idx = i; + + if (s[i] == '^') + { + type = '^'; + *buf << s[i++]; + + if (i < n) + { + beg_idx = i; + + if (s[i] == ']') + *buf << s[i++]; + } + } + else if (s[i] == ']') + *buf << s[i++]; + } + + while (i < n && s[i] != ']') + *buf << s[i++]; + + if (i < n && s[i] == ']') + { + end_idx = i-1; + *buf << s[i++]; + } + + if (s[i-1] != ']') + retval = nconv = -1; + } + } + + if (nconv >= 0) + { + if (beg_idx != std::string::npos && end_idx != std::string::npos) + char_class = expand_char_class (s.substr (beg_idx, + end_idx - beg_idx + 1)); + + add_elt_to_list (width, prec, bitwidth, val_type, discard, type, + num_elts, char_class); + } + + return retval; +} + +/********************/ +// Apply global options to individual conversion specifiers. +// Currently, records whether or not we need the custom parser for doubles, +// or if octave_read_value is sufficient. +void +textscan_format_list::consider_options (const textscan& ts) +{ + // We need to use custom number parser if the list of exponential signifiers + // (the "e" in 1e20) is non-default + bool all_need_custom = !ts.default_exp; + + // Check if any delimiter starts with an exponential signifier. + // If so, we need the custom number parser. + if (!all_need_custom) + { + for (int i = 0; i < ts.delim_list.numel (); i++) + { + std::string s = ts.delim_list(i).string_value (); + if (s.length () > 0) + { + char last = s[s.length () - 1]; + if (strchr (ts.exp_chars.c_str (), last)) + { + all_need_custom = true; + break; + } + } + } + } + + // Need custom parser if options mean that all do, or if this format + // is something like %10f or %10.4f. + for (int i = 0; i < numel (); i++) + { + list(i)->custom_number = (all_need_custom || list(i)->prec != -1 + || list(i)->width != static_cast (-1)); + } +} + +/******************** +void +textscan_format_list::printme (void) const +{ + octave_idx_type n = list.numel (); + + for (octave_idx_type i = 0; i < n; i++) + { + textscan_format_elt *elt = list(i); + + std::cerr + << "width: " << elt->width << "\n" + << "digits " << elt->prec << "\n" + << "bitwidth: " << elt->bitwidth << "\n" + << "discard: " << elt->discard << "\n" + << "type: "; + + if (elt->type == textscan_format_elt::literal_conversion) + std::cerr << "literal text\n"; + else if (elt->type == textscan_format_elt::whitespace_conversion) + std::cerr << "whitespace\n"; + else + std::cerr << elt->type << "\n"; + + std::cerr + << "char_class: `" << undo_string_escapes (elt->char_class) << "'\n" + << "text: `" << undo_string_escapes (elt->text) << "'\n\n"; + } +} +* */ + +/********************/ +// If FORMAT is explicitly "", it is assumed to be "%f" repeated enough times +// to read the first row of the file. Set it now. +void +textscan_format_list::read_first_row (std::istream& is, textscan& ts) +{ + std::string first_line; + getline (is, first_line, static_cast (ts.eol2)); + std::istringstream strstr (first_line); + + dim_vector dv (1,1); // initial size of each output_container + Complex val; + octave_value val_type; + nconv = 0; + int max_empty = 1000; // failsafe, if strstr fails but not with eof + long int pos; + + // read line, creating output_container as we go + while (!strstr.eof ()) + { + pos = strstr.tellg (); + ts.scan_complex (strstr, *list(0), val); + if (strstr.fail ()) // allow missing values in first row + { + strstr.clear (strstr.rdstate () & ~std::ios_base::failbit); + val = ts.empty_value.scalar_value (); + if (!--max_empty) + break; + } + if (val.imag () == 0) + val_type = octave_value (NDArray (dv, val.real ())); + else + val_type = octave_value (ComplexNDArray (dv, val)); + output_container.push_back (val_type); + ts.skip_delim (strstr); + if (strstr.tellg () == pos) // Stop if we're not making progress + break; + nconv++; + } + output_container.pop_front (); // discard empty element from constructor + + //Create fmt now that the size is known + list.resize (dim_vector (nconv, 1)); + for (octave_idx_type i = 1; i < nconv; i++) + list(i) = new textscan_format_elt (*list(0)); +} + +/********************/ +// Perform actual textscan: read data from stream, and create cell array +octave_value +textscan::scan (std::istream *isp, textscan_format_list& fmt_list, + octave_idx_type ntimes) +{ + octave_value retval = Cell (); + + if (!isp) + error ("internal error: textscan called with invalid istream"); + else if (fmt_list.num_conversions () == -1) + ::error ("textscan: invalid format specified"); + else if (! error_state) + { + if (fmt_list.num_conversions () == 0) + { + error ("textscan: no valid format conversion specifiers\n"); + return Cell (); + } + + // skip the first header_lines + std::string dummy; + for (int i = 0; i < header_lines && *isp; i++) + getline (*isp, dummy, static_cast (eol2)); + + // Grow retval dynamically. (If ntimes large, start smaller?) + // This is half the initial size + octave_idx_type size = ((ntimes < 8 && ntimes >= 0) ? ntimes : 1); + Array row_idx (dim_vector (1,2)); + row_idx(1) = 0; + + int err = 0; + octave_idx_type row = 0; + + if (multiple_delims_as_one) // bug #44750? + skip_delim (*isp); + + // If FORMAT explicitly "", read first line and see how many "%f" match + if (fmt_list.set_from_first) + { + row = 1; // the above puts the first line into fmt_list.out_buf () + fmt_list.read_first_row (*isp, *this); + lines = 1; + } + std::list out = fmt_list.out_buf (); + + // Check which columns to merge. Reals may become complex, and so can't + // trust types after reading in data. + bool merge_with_prev[fmt_list.numel ()]; + int conv = 0; + if (collect_output) + { + int prev_type = -1; + for (std::list::iterator col = out.begin (); + col != out.end (); col++) + { + if (col->type_id () == prev_type) + merge_with_prev [conv++] = true; + else + merge_with_prev [conv++] = false; + prev_type = col->type_id (); + } + } + + // Read the data + for (/* row set ~25 lines above */; row < ntimes || ntimes == -1; row++) + { + if (row == 0 || row >= size) + { + size += size+1; + for (std::list::iterator col = out.begin (); + col != out.end (); col++) + *col = (*col).resize (dim_vector (size, 1), 0); + } + row_idx(0) = row; + err = read_format_once (isp, fmt_list, out, row_idx); + if (err < 0 || !*isp || error_state + || (lines >= ntimes && ntimes > -1)) + break; + } + + if (return_on_error) + error_state = false; + + // convert return value to Cell array + int i = 0; + Array ra_idx (dim_vector (1,2)); + // err = -2 means "error, and no columns read this row + int valid_rows = (row == ntimes) ? ntimes : ((err == -2) ? row : row+1); + dim_vector dv (valid_rows, 1); + ra_idx(0) = 0; + if (!collect_output) + { + for (std::list::iterator col = out.begin (); + col != out.end (); col++, i++) + { + // resize col to size row + ra_idx(1) = i; + retval = do_cat_op (retval, + octave_value (Cell (col->resize (dv,0))), + ra_idx); + } + } + else // group adjacent cells of the same type into a single cell + { + octave_value cur; // current cell, accumulating columns + octave_idx_type group_size = 0; // columns in this cell + int prev_type = -1; + + conv = 0; + for (std::list::iterator col = out.begin (); + col != out.end (); col++) + { + if (!merge_with_prev [conv++]) // including first time + { + if (prev_type != -1) + { + ra_idx(1) = i++; + retval = do_cat_op (retval, octave_value (Cell(cur)), + ra_idx); + } + cur = octave_value (col->resize (dv,0)); + group_size = 1; + prev_type = col->type_id (); + } + else + { + ra_idx(1) = group_size++; + cur = do_cat_op (cur, octave_value (col->resize (dv,0)), + ra_idx); + } + } + ra_idx(1) = i; + retval = do_cat_op (retval, octave_value (Cell (cur)), ra_idx); + } + } + return retval; +} diff -r 68e3a747ca02 libinterp/dldfcn/textscan.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/dldfcn/textscan.h Sat Nov 14 11:26:39 2015 +1100 @@ -0,0 +1,283 @@ + +#if !defined (octave_textscan_h) +#define octave_textscan_h 1 + +// For Inf and NaN +#include "lo-ieee.h" + +class +OCTINTERP_API +textscan_format_elt +{ +public: + + enum special_conversion + { + whitespace_conversion = 1, + literal_conversion = 2 + }; + + textscan_format_elt (const char *txt = 0, int w = 0, int p = -1, + int bw = 0, bool dis = false, char typ = '\0', + const std::string& ch_class = std::string ()) + : text (strsave (txt)), width (w), prec (p), bitwidth (bw), + char_class (ch_class), type (typ), discard (dis), + numeric(typ == 'd' || typ == 'u' || type == 'f' || type == 'n') + { } + + textscan_format_elt (const textscan_format_elt& e) + : text (strsave (e.text)), width (e.width), prec (e.prec), + bitwidth (e.bitwidth), char_class (e.char_class), type (e.type), + discard (e.discard), numeric (e.numeric), custom_number (e.custom_number) + { } + + textscan_format_elt& operator = (const textscan_format_elt& e) + { + if (this != &e) + { + text = strsave (e.text); + width = e.width; + prec = e.prec; + bitwidth = e.bitwidth; + discard = e.discard; + type = e.type; + numeric = e.numeric; + custom_number = e.custom_number; + char_class = e.char_class; + } + + return *this; + } + + ~textscan_format_elt (void) { delete [] text; } + + // The C-style format string. + const char *text; + + // The maximum field width. + unsigned int width; + + // The maximum number of digits to read after the decimal in a + // floating point conversion. + int prec; + + // The size of the result. For integers, bitwidth may be 8, 16, 34, + // or 64. For floating point values, bitwidth may be 32 or 64. + int bitwidth; + + // The class of characters in a `[' or `^' format. + std::string char_class; + + // Type of conversion + // -- `d', `u', `f', `n', `s', `q', `c', `%', `C', `D', `[' or `^'. + char type; + + // TRUE if we are not storing the result of this conversion. + bool discard; + + // TRUE if the type is 'd', 'u', 'f', 'n' + bool numeric; + + // TRUE if width or precision is specified, or options.exp_chars is set + bool custom_number; +}; + +class textscan; + +class +OCTINTERP_API +textscan_format_list +{ +public: + + textscan_format_list (const std::string& fmt = std::string ()); + + ~textscan_format_list (void); + + void consider_options (const textscan& options); + + octave_idx_type num_conversions (void) const { return nconv; } + + // The length can be different than the number of conversions. + // For example, "x %d y %d z" has 2 conversions but the length of + // the list is 3 because of the characters that appear after the + // last conversion. + + octave_idx_type numel (void) const { return list.numel (); } + + const textscan_format_elt *first (void) + { + curr_idx = 0; + return current (); + } + + const textscan_format_elt *current (void) const + { return list.numel () > 0 ? list.elem (curr_idx) : 0; } + + const textscan_format_elt *next (bool cycle = true) + { + curr_idx++; + + if (curr_idx >= list.numel ()) + { + if (cycle) + curr_idx = 0; + else + return 0; + } + return current (); + } + + void printme (void) const; + + bool ok (void) const { return (nconv >= 0); } + + operator bool () const { return ok (); } + + bool set_from_first; // true if number of %f to be set from data file + bool has_string; // at least one conversion specifier is s,q,c, or [...] + void read_first_row (std::istream& is, textscan& ts); + + std::list out_buf () const { return (output_container); } + +private: + + // Number of conversions specified by this format string, or -1 if + // invalid conversions have been found. + octave_idx_type nconv; + + // Index to current element; + octave_idx_type curr_idx; + + // FIXME -- maybe LIST should be a std::list object? + // List of format elements. + Array list; + + // list holding column arrays of types specified by conversions + std::list output_container; + + // Temporary buffer. + std::ostringstream *buf; + + void add_elt_to_list (unsigned int width, int prec, int bitwidth, + octave_value val_type, bool discard, + char type, octave_idx_type& num_elts, + const std::string& char_class = std::string ()); + + void process_conversion (const std::string& s, size_t& i, size_t n, + octave_idx_type& num_elts); + + int finish_conversion (const std::string& s, size_t& i, size_t n, + unsigned int& width, int& prec, int& bitwidth, + octave_value& val_type, + bool discard, char& type, + octave_idx_type& num_elts); + // No copying! + + textscan_format_list (const textscan_format_list&); + + textscan_format_list& operator = (const textscan_format_list&); +}; + +class +textscan +{ + friend textscan_format_list; + + std::string buf; + // Three cases for delim_table and delim_list + // 1. delim_table empty, delim_list empty: whitespace delimiters + // 2. delim_table = look-up table of delim chars, delim_list empty. + // 3. delim_table non-empty, delim_list = Cell array of delim strings + std::string delim_table; // delim_table[i]=='\0' if i is not a delimiter, + std::string whitespace_table; + Cell comment_style; + int comment_len; // How far ahead to look to detect an open comment + int comment_char; // first character of open comment + + std::string date_locale; + + Cell delim_list; // Array of strings of delimiters + + octave_value empty_value; + std::string exp_chars; + int header_lines; + Cell treat_as_empty; + int treat_as_empty_len; // longest string to treat as "N/A" + std::string whitespace; + short eol1, eol2; + + bool collect_output; + bool multiple_delims_as_one; + bool return_on_error; + + bool default_exp; + bool numeric_delim; + + octave_idx_type lines; + + char *get_field (std::istream& isp, unsigned int width) const; + bool match_literal (std::istream& isp, const textscan_format_elt& elem); + int skip_whitespace (std::istream& is, bool EOLstop = false); + int skip_delim (std::istream& is); + bool is_delim (unsigned char ch) const + { + return (delim_table.length () == 0 + && (isspace (ch) || ch == eol1 || ch == eol2)) + || delim_table[ch] != '\0'; + } + // the only delimiter is whitespace + bool whitespace_delim () const { return delim_table.length () == 0; } + bool isspace (unsigned int ch) const { return whitespace_table[ch & 0xff]; } + + std::istream& scan_string (std::istream& is, const textscan_format_elt& fmt, + std::string& val) const; + std::string read_until (std::istream& is, const Cell& delimiters, + const std::string& ends) const; + std::istream& scan_cstring (std::istream& is, const textscan_format_elt& fmt, + std::string& val) const; + std::istream& scan_qstring (std::istream& is, const textscan_format_elt& fmt, + std::string& val); + int read_format_once (std::istream* isp, textscan_format_list& fmt_list, + std::list & retval, + Array row); + std::istream& scan_one (std::istream& is, const textscan_format_elt& fmt, + octave_value& ov, Array row); + int scan_bracket (std::istream& is, const char *pattern, std::string& val) + const; + int scan_caret (std::istream& is, const char *, std::string& val) const; + int lookahead (std::istream& is, const Cell& targets, int max_len) const; + int read_format_once(std::istream* isp, textscan_format_list& fmt_list, + Cell& retval, octave_idx_type row) const; + + double formatted_double (std::istream& is, const textscan_format_elt& fmt) + const; + double read_double (std::istream& is, const textscan_format_elt& fmt) const + { + if (fmt.custom_number) + return formatted_double (is, fmt); + else + return octave_read_value (is); + } + std::istream& scan_complex (std::istream& is, const textscan_format_elt& fmt, + Complex& val) const; + public: + + textscan () + : buf (""), delim_table (""), comment_char(-2), empty_value (octave_NaN), + exp_chars ("edED"), header_lines (0), treat_as_empty_len (0), + whitespace (" \b\t"), + eol1('\r'), eol2('\n'), + collect_output (false), multiple_delims_as_one (false), + return_on_error (true), default_exp (true), numeric_delim (false), + lines (0) + { }; + + octave_value scan (std::istream* isp, textscan_format_list& fmt_list, + octave_idx_type ntimes); + void parse_options (const octave_value_list& args, int first_param, + textscan_format_list& formats); +}; + + +#endif diff -r 68e3a747ca02 scripts/io/module.mk --- a/scripts/io/module.mk Fri Nov 13 14:10:26 2015 -0500 +++ b/scripts/io/module.mk Sat Nov 14 11:26:39 2015 +1100 @@ -9,7 +9,6 @@ scripts_io_FCN_FILES = \ scripts/io/importdata.m \ scripts/io/is_valid_file_id.m \ scripts/io/strread.m \ - scripts/io/textscan.m \ scripts/io/textread.m scripts_iodir = $(fcnfiledir)/io diff -r 68e3a747ca02 test/textscan.tst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/textscan.tst Sat Nov 14 11:26:39 2015 +1100 @@ -0,0 +1,570 @@ + + +%!test +%! str = "1, 2, 3, 4\n 5, , , 8\n 9, 10, 11, 12"; +%! fmtstr = "%f %d %f %s"; +%! c = textscan (str, fmtstr, 2, "delimiter", ",", "emptyvalue", -Inf); +%! assert (isequal (c{1}, [1;5])); +%! assert (length (c{1}), 2); +%! assert (iscellstr (c{4})); +%! assert (isequal (c{3}, [3; -Inf])); + +%!test +%! b = [10:10:100]; +%! b = [b; 8*b/5]; +%! str = sprintf ("%g miles/hr = %g kilometers/hr\n", b); +%! fmt = "%f miles/hr = %f kilometers/hr"; +%! c = textscan (str, fmt); +%! assert (c{1}, b(1,:)', 1e-5); +%! assert (c{2}, b(2,:)', 1e-5); + +%!test +%! str = "13, -, NA, str1, -25\r\n// Middle line\r\n36, na, 05, str3, 6"; +%! a = textscan (str, "%d %n %f %s %n", "delimiter", ",", +%! "treatAsEmpty", {"NA", "na", "-"},"commentStyle", "//"); +%! assert (a{1}, int32 ([13; 36])); +%! assert (a{2}, [NaN; NaN]); +%! assert (a{3}, [NaN; 5]); +%! assert (a{4}, {"str1"; "str3"}); +%! assert (a{5}, [-25; 6]); + +%!test +%! str = "Km:10 = hhhBjjj miles16hour\r\n"; +%! str = [str "Km:15 = hhhJjjj miles241hour\r\n"]; +%! str = [str "Km:2 = hhhRjjj miles3hour\r\n"]; +%! str = [str "Km:25 = hhhZ\r\n"]; +%! fmt = "Km:%d = hhh%1sjjj miles%dhour"; +%! a = textscan (str, fmt, "delimiter", " "); +%! assert (a{1}', int32 ([10 15 2 25])); +%! assert (a{2}', {'B' 'J' 'R' 'Z'}); +%! assert (a{3}', int32 ([16 241 3 0])); + +## Test with default endofline parameter +%!test +%! c = textscan ("L1\nL2", "%s"); +%! assert (c{:}, {"L1"; "L2"}); + +## Test with endofline parameter set to "" (empty) - newline should be in word +%!test +%! c = textscan ("L1\nL2", "%s", "endofline", ""); +%! assert (int8 (c{:}{:}), int8 ([ 76, 49, 10, 76, 50 ])); + +### Matlab fails this test. A literal after a conversion is not a delimiter +#%!test +#%! ## No delimiters at all besides EOL. Skip fields, even empty fields +#%! str = "Text1Text2Text\nTextText4Text\nText57Text"; +#%! c = textscan (str, "Text%*dText%dText"); +#%! assert (c{1}, int32 ([2; 4; 0])); + +## CollectOutput test +%!test +%! b = [10:10:100]; +%! b = [b; 8*b/5; 8*b*1000/5]; +%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b); +%! fmt = "%f miles%s %s %f (%f) kilometers %*s"; +%! c = textscan (str, fmt, "collectoutput", 1); +%! assert (size (c{3}), [10, 2]); +%! assert (size (c{2}), [10, 2]); + +## CollectOutput test with uneven column length files +%!test +%! b = [10:10:100]; +%! b = [b; 8*b/5; 8*b*1000/5]; +%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b); +%! str = [str "110 miles/hr"]; +%! fmt = "%f miles%s %s %f (%f) kilometers %*s"; +%! c = textscan (str, fmt, "collectoutput", 1); +%! assert (size (c{1}), [11, 1]); +%! assert (size (c{3}), [11, 2]); +%! assert (size (c{2}), [11, 2]); +%! assert (c{3}(end), NaN); +%! assert (c{2}{11, 1}, "/hr"); +%! assert (isempty (c{2}{11, 2}), true); + +## Double quoted string +%!test +%! str = 'First "the second called ""the middle""" third'; +%! fmt = "%q"; +%! c = textscan (str, fmt); +%! assert (c{1}, {"First"; 'the second called "the middle"'; "third"}); + +## Arbitrary character +%!test +%! c = textscan ("a first, \n second, third", "%s %c %11c", 'delimiter', ' ,'); +%! assert (c{1}, {"a"; "ond"}); +%! assert (c{2}, {"f"; "t"}); +%! assert (c{3}, {"irst, \n sec"; "hird"}); + +## Field width and non-standard delimiters +%!test +%! str = "12;34;123456789;7"; +%! c = textscan (str, "%4d %4d", "delimiter", ";", "collectOutput", 1); +%! assert (c, {[12 34; 1234 5678; 9 7]}); + +## Field width and non-standard delimiters +%!test +%! str = "12;34;123456789;7"; +%! c = textscan (str, "%4f %f", "delimiter", ";", "collectOutput", 1); +%! assert (c, {[12 34; 1234 56789; 7 NaN]}); + +## Ignore trailing delimiter, but use leading one +%!test +%! str = "12.234e+2,34, \n12345.789-9876j,78\n,10|3"; +%! c = textscan (str, "%10.2f %f", "delimiter", ",", "collectOutput", 1, +%! "expChars", "e|"); +%! assert (c, {[1223 34; 12345.79-9876j 78; NaN 10000]}, 1e-6); + +%!test +%! ## Multi-character delimiter +%! str = "99end2 space88gap 4564"; +%! c = textscan (str, "%d %s", "delimiter", {"end", "gap", "space"}); +%! assert (c{1}, int32([99; 88])); +%! assert (c{2}, {"2 "; "4564"}); + +### Delimiters as part of literals, and following literals +#%!test +#%! str = "12 R&D & 7"; +#%! c = textscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1, "EmptyValue", -99); +#%! assert (c, {[12 -99; 7 -99]}); +# +### Delimiters as part of literals, and before literals +#%!test +#%! str = "12 & R&D 7"; +#%! c = textscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1); +#%! assert (c, {[12 7]}); + +%!test +%! ## Check number of lines read, not number of passes through format string +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid, "1\n2\n3\n4\n5\n6"); +%! fseek (fid, 0, "bof"); +%! c = textscan (fid, "%f %f", 2); +%! E = feof (fid); +%! fclose (fid); +%! unlink (f); +%! assert (c, {1, 2}); +%! assert (!E); + +%!test +%! ## Check number of lines read, not number of passes through format string +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid, "1\r\n2\r3\n4\r\n5\n6"); +%! fseek (fid, 0, "bof"); +%! c = textscan (fid, "%f %f", 4); +%! fclose (fid); +%! unlink (f); +%! assert (c, {[1;3], [2;4]}) + +%!test +%! ## Check number of lines read, with multiple delimiters +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid, "1-\r\n-2\r3-\n-4\r\n5\n6"); +%! fseek (fid, 0, "bof"); +%! c = textscan (fid, "%f %f", 4, "delimiter", "-", "multipleDelimsAsOne", 1); +%! fclose (fid); +%! unlink (f); +%! assert (c, {[1;3], [2;4]}) + +## Test input validation +%!error textscan () +%!error textscan (single (40)) +%!error textscan ({40}) +%!error textscan ("Hello World", 2) +#%!error [C, pos] = textscan ("Hello World") +%!error textscan ("Hello World", '%s', 'EndOfLine', 3) +%!error <'%z' is not a valid format specifier> textscan ("1.0", "%z"); +%!error textscan ("1.0", "foo"); + +## Test incomplete first data line +%! R = textscan (['Empty1' char(10)], 'Empty%d %f'); +%! assert (R{1}, int32 (1)); +%! assert (isempty (R{2}), true); + +## bug #37023 +%!test +%! data = textscan(" 1. 1 \n 2 3\n", '%f %f'); +%! assert (data{1}, [1; 2], 1e-15); +%! assert (data{2}, [1; 3], 1e-15); + +## Whitespace test (bug #37333) using delimiter ";" +%!test +%! tc = []; +%! tc{1, 1} = "C:/code;"; +%! tc{1, end+1} = "C:/code/meas;"; +%! tc{1, end+1} = " C:/code/sim;"; +%! tc{1, end+1} = "C:/code/utils;"; +%! string = [tc{:}]; +%! c = textscan (string, "%s", "delimiter", ";"); +%! for k = 1:max (numel (c{1}), numel (tc)) +%! lh = c{1}{k}; +%! rh = tc{k}; +%! rh(rh == ";") = ""; +%! rh = strtrim (rh); +%! assert (strcmp (lh, rh)); +%! end + +## Whitespace test (bug #37333), adding multipleDelimsAsOne true arg +%!test +%! tc = []; +%! tc{1, 1} = "C:/code;"; +%! tc{1, end+1} = " C:/code/meas;"; +%! tc{1, end+1} = "C:/code/sim;;"; +%! tc{1, end+1} = "C:/code/utils;"; +%! string = [tc{:}]; +%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 1); +%! for k = 1:max (numel (c{1}), numel (tc)) +%! lh = c{1}{k}; +%! rh = tc{k}; +%! rh(rh == ";") = ""; +%! rh = strtrim (rh); +%! assert (strcmp (lh, rh)); +%! end + +## Whitespace test (bug #37333), adding multipleDelimsAsOne false arg +%!test +%! tc = []; +%! tc{1, 1} = "C:/code;"; +%! tc{1, end+1} = " C:/code/meas;"; +%! tc{1, end+1} = "C:/code/sim;;"; +%! tc{1, end+1} = ""; +%! tc{1, end+1} = "C:/code/utils;"; +%! string = [tc{:}]; +%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 0); +%! for k = 1:max (numel (c{1}), numel (tc)) +%! lh = c{1}{k}; +%! rh = tc{k}; +%! rh(rh == ";") = ""; +%! rh = strtrim (rh); +%! assert (strcmp (lh, rh)); +%! end + +## Whitespace test (bug #37333) whitespace "" arg +%!test +%! tc = []; +%! tc{1, 1} = "C:/code;"; +%! tc{1, end+1} = " C:/code/meas;"; +%! tc{1, end+1} = "C:/code/sim;"; +%! tc{1, end+1} = "C:/code/utils;"; +%! string = [tc{:}]; +%! c = textscan (string, "%s", "delimiter", ";", "whitespace", ""); +%! for k = 1:max (numel (c{1}), numel (tc)) +%! lh = c{1}{k}; +%! rh = tc{k}; +%! rh(rh == ";") = ""; +%! assert (strcmp (lh, rh)); +%! end + +## Whitespace test (bug #37333), whitespace " " arg +%!test +%! tc = []; +%! tc{1, 1} = "C:/code;"; +%! tc{1, end+1} = " C:/code/meas;"; +%! tc{1, end+1} = "C:/code/sim;"; +%! tc{1, end+1} = "C:/code/utils;"; +%! string = [tc{:}]; +%! c = textscan (string, "%s", "delimiter", ";", "whitespace", " "); +%! for k = 1:max (numel (c{1}), numel (tc)) +%! lh = c{1}{k}; +%! rh = tc{k}; +%! rh(rh == ";") = ""; +%! rh = strtrim (rh); +%! assert (strcmp (lh, rh)); +%! end + +%!test +%!## Test unfinished comment +%! c = textscan ("1 2 /* half comment", "%n %u8", "CommentStyle", {"/*", "*/}); +%! assert (c, {1, 2}); + +%!test +%!## Test start of comment as string +%! c = textscan ("1 / 2 // 3", "%n %s %u8", "CommentStyle", {"//"}); +%! assert (c, {1, "/", 2}); + +## Test reading from a real file +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! d = rand (1, 4); +%! fprintf (fid, " %f %f /* comment */ %f %f ", d); +%! fseek (fid, 0, "bof"); +%! A = textscan (fid, "%f %f", "CommentStyle", {"/*", "*/"}); +%! E = feof (fid); +%! fclose (fid); +%! unlink (f); +%! assert (A{1}, [d(1); d(3)], 1e-6); +%! assert (A{2}, [d(2); d(4)], 1e-6); +%! assert (E); + +## Tests reading with empty format, should return proper nr of columns +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid, " 1 2 3 4\n5 6 7 8"); +%! fseek (fid, 0, "bof"); +%! A = textscan (fid, ""); +%! E = feof (fid); +%! fclose (fid); +%! unlink (f); +%! assert (A{1}, [1 ; 5], 1e-6); +%! assert (A{2}, [2 ; 6], 1e-6); +%! assert (A{3}, [3 ; 7], 1e-6); +%! assert (A{4}, [4 ; 8], 1e-6); +%! assert (E); + +## Tests reading with empty format; empty fields & incomplete lower row +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid, " ,2,,4\n5,6"); +%! fseek (fid, 0, "bof"); +%! A = textscan (fid, "", "delimiter", ",", "EmptyValue", 999, "CollectOutput" , 1); +%! fclose (fid); +%! unlink (f); +%! assert (A{1}, [999, 2, 999, 4; 5, 6, 999, 999], 1e-6); + +## Error message tests + +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! msg1 = "textscan: 1 parameters given, but only 0 values"; +%! try +%! A = textscan (fid, "", "headerlines"); +%! end_try_catch; +%! assert (!feof (fid)); +%! fclose (fid); +%! unlink (f); +%! assert (msg1, lasterr); + +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! msg1 = "textscan: HeaderLines must be numeric"; +%! try +%! A = textscan (fid, "", "headerlines", "hh"); +%! end_try_catch; +%! fclose (fid); +%! unlink (f); +%! assert (msg1, lasterr); + +%!test +%! ## Skip headerlines +%! A = textscan ("field 1 field2\n 1 2\n3 4", "", "headerlines", 1, "collectOutput", 1); +%! assert (A, {[1 2; 3 4]}); + +%!test +%! ## Skip headerlines with non-default EOL +%! A = textscan ("field 1 field2\r 1 2\r3 4", "", "headerlines", 2, "collectOutput", 1, "EndOfLine", '\r'); +%! assert (A, {[3 4]}); + +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid,"some_string"); +%! fseek (fid, 0, "bof"); +%! msg1 = "textscan: EndOfLine must be at most one character or '\\r\\n'"; +%! try +%! A = textscan (fid, "%f", "EndOfLine", "\n\r"); +%! end_try_catch; +%! fclose (fid); +%! unlink (f); +%! assert (msg1, lasterr); + +%!test +%! f = tempname (); +%! fid = fopen (f, "w+"); +%! fprintf (fid,"some_string"); +%! fseek (fid, 0, "bof"); +%! msg1 = "textscan: EndOfLine must be at most one character or '\\r\\n'"; +%! try +%! A = textscan (fid, "%f", "EndOfLine", 33); +%! end_try_catch; +%! fclose (fid); +%! unlink (f); +%! assert (msg1, lasterr); + +## Bug #41824 +%!test +%! assert (textscan ("123", "", "whitespace", " "){:}, 123); + +## Bug #42343-1, just test supplied emptyvalue +%!test +%! assert (textscan (",NaN", "", "delimiter", "," ,"emptyValue" ,Inf), {Inf, NaN}); + +## Bug #42343-2, test padding with supplied emptyvalue +%!test +%! a = textscan (",1,,4\nInf, ,NaN", "", "delimiter", ",", "emptyvalue", -10); +%! assert (cell2mat (a), [-10, 1, -10, 4; Inf, -10, NaN, -10]); + +## Bug #42528 +%!test +%! assert (textscan ("1i", ""){1}, 0+1i); +%! assert (cell2mat (textscan ("3, 2-4i, NaN\n -i, 1, 23.4+2.2i\n 1+1 1+1j", "", "delimiter", ",")), [3+0i, 2-4i, NaN+0i; 0-i, 1+0i, 23.4+2.2i; 1 1 1+1i]); + + +%!test +%! ## TreatAsEmpty +%! C = textscan ("1,2,3,NN,5,6\n", "%d%d%d%f", "delimiter", ",", "TreatAsEmpty", "NN"); +%! assert (C{3}(1), int32 (3)); +%! assert (C{4}(1), NaN); + +## MultipleDelimsAsOne +%!test +%! str = "11, 12, 13,, 15\n21,, 23, 24, 25\n,, 33, 34, 35"; +%! C = textscan (str, "%f %f %f %f", "delimiter", ",", "multipledelimsasone", 1, "endofline", "\n"); +%! assert (C{1}', [11, 21, 33]); +%! assert (C{2}', [12, 23, 34]); +%! assert (C{3}', [13, 24, 35]); +%! assert (C{4}', [15, 25, NaN]); + +## Bug #44750 +%!test +%! assert (textscan ("/home/foo/", "%s", "delimiter", "/", "MultipleDelimsAsOne", 1){1}, ... +%! {"home"; "foo"}); + +### Allow cuddling %sliteral but warn it is ambiguous +#%!test +#%! C = textscan ("abcxyz51\nxyz83\n##xyz101", "%s xyz %d"); +#%! assert (C{1}([1 3]), {"abc"; "##"}); +#%! assert (isempty (C{1}{2}), true); +#%! assert (C{2}, int32([51; 83; 101])); +### Literals are not delimiters. + +## Test for false positives in check for non-supported format specifiers +%!test +%! assert (textscan ("Total: 32.5 % (of cm values)", "Total: %f %% (of cm values)"){1}, 32.5, 1e-5); + +## Test various forms of string format specifiers (bug #45712) +%!test +%! str = "14 :1 z:2 z:3 z:5 z:11"; +%! C = textscan (str, "%f %s %*s %3s %*3s %f", "delimiter", ":"); +%! assert (C, {14, {"1 z"}, {"3 z"}, 11}); + +%% Bit width, fixed width conv. specifiers +%!test +%! str2 = "123456789012345 "; +%! str2 = [str2 str2 str2 str2 str2 str2 str2 str2]; +%! str2 = [str2 "123456789.01234 1234567890.1234 12345.678901234 12345.678901234"]; +%! pttrn = "%3u8%*s %5u16%*s %10u32%*s %15u64 %3d8%*s %5d16%*s %10d32%*s %15d64 %9f32%*s %14f64%*s %10.2f32%*s %12.2f64%*s"; +%! C = textscan (str2, pttrn, "delimiter", " "); +%! assert (C{1}, uint8 (123)); +%! assert (C{2}, uint16 (12345)); +%! assert (C{3}, uint32 (1234567890)); +%! assert (C{4}, uint64 (123456789012345)); +%! assert (C{5}, int8 (123)); +%! assert (C{6}, int16 (12345)); +%! assert (C{7}, int32 (1234567890)); +%! assert (C{8}, int64 (123456789012345)); +%! assert (C{9}, single (123456789), 1e-12); +%! assert (C{10}, double (1234567890.123), 1e-15); +%! assert (C{11}, single (12345.68), 1e-5); +%! assert (C{12}, double (12345.68), 1e-11); + +%% Bit width, fixed width conv. specifiers -- check the right amount is left +%!test +%! str2 = "123456789012345 "; +%! str2 = [str2 str2 "123456789.01234"]; +%! pttrn = "%3u8 %5u16 %10u32 %3d8 %5d16 %10d32 %9f32 %9f"; +%! C = textscan (str2, pttrn, "delimiter", " "); +%! assert (C{1}, uint8 (123)); +%! assert (C{2}, uint16 (45678)); +%! assert (C{3}, uint32 (9012345)); +%! assert (C{4}, int8 (123)); +%! assert (C{5}, int16 (45678)); +%! assert (C{6}, int32 (9012345)); +%! assert (C{7}, single (123456789), 1e-12); +%! assert (C{8}, double (0.01234), 1e-12); + +%!test +%! C = textscan ("123.123", "%2f %3f %3f"); +%! assert (C{1}, 12); +%! assert (C{2}, 3.1, 1e-11); +%! assert (C{3}, 23); + +%!test +%! C = textscan ("123.123", "%3f %3f %3f"); +%! assert (C{1}, 123); +%! assert (C{2}, 0.12, 1e-11); +%! assert (C{3}, 3); + +%!test +%! C = textscan ("123.123", "%4f %3f"); +%! assert (C{1}, 123); +%! assert (C{2}, 123); + +%% field width interrupts exponent. (Matlab incorrectly gives [12, 2e12]) +%!test +%! assert (textscan ("12e12", "%4f"), {[120; 2]}); +%! assert (textscan ("12e+12", "%5f"), {[120; 2]}); +%! assert (textscan ("125e-12","%6f"), {[12.5; 2]}); + +%% %[] tests +%% Plain [..] and *[..] +%!test +%! ar = "abcdefguvwxAny\nacegxyzTrailing\nJunk"; +%! C = textscan (ar, "%[abcdefg] %*[uvwxyz] %s"); +%! assert (C{1}, {"abcdefg"; "aceg"; ""}); +%! assert (C{2}, {"Any"; "Trailing"; "Junk"}); + +%!test +%! assert (textscan ("A2 B2 C3", "%*[ABC]%d", 3), {int32([2; 2; 3])}); + +%% [^..] and *[^..] +%!test +%! br = "abcdefguvwx1Any\nacegxyz2Trailing\n3Junk"; +%! C = textscan (br, "%[abcdefg] %*[^0123456789] %s"); +%! assert (C{1}, {"abcdefg"; "aceg"; ""}); +%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"}); + +%% [..] and [^..] containing delimiters +%!test +%! cr = "ab cd efguv wx1Any\na ce gx yz2Trailing\n 3Junk"; +%! C = textscan (cr, "%[ abcdefg] %*[^0123456789] %s", "delimiter", " \n", "whitespace", ""); +%! assert (C{1}, {"ab cd efg"; "a ce g"; " "}); +%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"}); + +%% Bug #36464 +%!test +%! assert (textscan ('1 2 3 4 5 6', "%*n%n%*[^\n]"){1}, 2); + +%% test %[]] and %[^]] +%!test +%! assert (textscan ('345]', "%*[123456]%[]]"){1}{1}, "]"); +%! assert (textscan ('345]', "%*[^]]%s"){1}{1}, "]"); + + + +## Needed tests: +# 1. Check that it fails at an appropriate point +# 3. Checks for %[...] and %[^...] +# 4. Checks for file vs string +# 5. Checks for mixing headerlines and comments +# 7. Checks for ReturnOnError +# 8. Checks for all types of invalid parameter +# 9. Checks derived from searching the internet for "= textscan" + +## Check of what Matlab does: +# 1. mixing headerlines and comments (just skip lines, even if comments) +# 2. Empty EndOfLine (acceptable) +# 3. Complex NaN (NaN+0i. However 1+NaNi gives 1+NaNi) +# 4. delimiters at the start of lines with "multiple delimiters as one" + (delim merges with EOL. Without multiple...one, a trailing delimiter + before a newline is ignored) + +# 5. If N not specified, are parameters still checked? + + +#ToDO: +# 1. Common skip whitespace for everything except %[...] and %[^...] +# 2. Refactor octave_scan to separate integers from floating point? +# 3. Fail when the first mismatch occurs. Tests for this +# 4. Remove commented out code +# 5. Treatment of missing final end-of-line -- unequal line lengths. +# 6. String delimiter as multichar, not as list of delims. (Check Matlab.) +# 7. Minimise duplication with scanf and printf? Use as basis for textread? +# 8. List tests modified from textscan.m +# 9. Profile