/*
Copyright (C) 2015-2016 Lachlan Andrew, Monash University
This file is part of Octave.
Octave is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3 of the License, or (at your
option) any later version.
Octave is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with Octave; see the file COPYING. If not, see
.
*/
/** @file libinterp/dldfcn/ytextscan
* Implement the 'ytextscan' octave command
*/
//#ifdef HAVE_CONFIG_H
#include
//#endif
#include "list"
#include "Array.cc"
#include "Cell.h"
#include "ytextscan.h"
#include "oct-stream.h"
#include "ovl.h"
#include "utils.h"
#include "defun-dld.h"
#include "ov-re-mat.h"
DEFUN_DLD (ytextscan, args, ,
"-*- texinfo -*-\n\
@deftypefn {} {@var{C} =} ytextscan (@var{fid}, @var{format})\n\
@deftypefnx {} {@var{C} =} ytextscan (@var{fid}, @var{format}, @var{repeat})\n\
@deftypefnx {} {@var{C} =} ytextscan (@var{fid}, @var{format}, @var{param}, @var{value}, @dots{})\n\
@deftypefnx {} {@var{C} =} ytextscan (@var{fid}, @var{format}, @var{repeat}, @var{param}, @var{value}, @dots{})\n\
@deftypefnx {} {@var{C} =} ytextscan (@var{str}, @dots{})\n\
@deftypefnx {} {[@var{C}, @var{position}] =} ytextscan (@dots{})\n\
Read data from a text file or string.\n\
\n\
The string @var{str} or file associated with @var{fid} is read from and\n\
parsed according to @var{format}. The function is an extension\n\
of @code{strread} and @code{textread}. Differences include: the ability to\n\
read from either a file or a string, additional options, and additional\n\
format specifiers.\n\
\n\
The input is interpreted as a sequence of \"words\", delimiters (such as\n\
whitespace) and literals. The characters that form delimiters and whitespace\n\
are determined by the options. The format consists of format specifiers\n\
interspersed between literals. In the format, whitespace forms a delimiter\n\
between consecutive literals, but is otherwise ignored.\n\
\n\
The output @var{C} is a cell array whose second dimension is determined\n\
by the number of format specifiers.\n\
\n\
The first word of the input is matched to the first specifier of the\n\
format and placed in the first column of the output; the second is matched\n\
to the second specifier and placed in the second column and so forth.\n\
If there are more words than specifiers, the process is repeated until all\n\
words have been processed or the limit imposed by @var{repeat} has been met\n\
(see below).\n\
\n\
The string @var{format} describes how the words in @var{str} should be\n\
parsed. As in @var{fscanf},\n\
any (non-whitespace) text in the format that is not one of these specifiers\n\
is considered a literal; if there is a literal between two format specifiers\n\
then that same literal must appear in the input stream between the matching\n\
words.\n\
\n\
The following specifiers are valid:\n\
\n\
@table @code\n\
@item %f\n\
@itemx %f64\n\
@itemx %n\n\
The word is parsed as a number and converted to double.\n\
\n\
@item %f32\n\
The word is parsed as a number and converted to single (float).\n\
\n\
@item %d\n\
@itemx %d8\n\
@itemx %d16\n\
@itemx %d32\n\
@itemx %d64\n\
The word is parsed as a number and converted to int8, int16, int32 or int64.\n\
If not size is specified, int32 is used.\n\
\n\
@item %u\n\
@itemx %u8\n\
@itemx %u16\n\
@itemx %u32\n\
@itemx %u64\n\
The word is parsed as a number and converted to uint8, uint16, uint32 or\n\
uint64. If not size is specified, uint32 is used.\n\
\n\
@item %s\n\
The word is parsed as a string, ending at the last character before\n\
whitespace, an end-of-line or a delimiter specified in the options.\n\
\n\
@item %q\n\
The word is parsed as a \"quoted string\". If the first character of the\n\
string is a double quote (\") then the string includes everything until a\n\
matching double quote, including whitespace, delimiters and end of line\n\
characters. If a pair of consecutive double quotes appears in the input,\n\
it is replaced in the output by a single double quote. That is, the input\n\
\"He said \"\"Hello\"\"\" would return the value 'He said \"Hello\"'.\n\
\n\
@item %c\n\
The next character of the input is read. This includes delimiters,\n\
whitespace and end of line characters.\n\
\n\
@item %[...]\n\
@itemx %[^...]\n\
In the first form, the word consists of the longest run consisting of only\n\
characters between the brackets. Ranges of characters can be specified by\n\
a hyphen; for example, %[0-9a-zA-Z] matches all alphanumeric characters\n\
(if the underlying character set is ASCII). To include '-' in the set,\n\
it should appear first or last in the brackets; to include ']', it should\n\
be the first character. If the first character is '^' then the word\n\
consists of characters NOT listed.\n\
\n\
For %s, %c %d, %f, %n, %u, an\n\
optional width can be specified as %Ns, etc. where N is an integer > 1.\n\
For %c, this causes exactly the next N characters to be read instead of\n\
a single character. For the other specifiers, it is an upper bound on the\n\
number of characters read; normal delimiters can cause fewer characters to\n\
be read. For complex numbers, this limit applies to the real and imaginary\n\
components individually\n\
For %f and %n, format specifiers like %N.Mf are allowed, where M is an upper\n\
bound on number of characters after the decimal point to be considered;\n\
subsequent digits are skipped. For example, the specifier %8.2f would\n\
read 12.345e6 as 1.234e7.\n\
\n\
@item %*...\n\
The word specified by the remainder of the conversion specifier is skipped.\n\
\n\
@item literals\n\
In addition the format may contain literal character strings; these will be\n\
skipped during reading. If the input string does not match this literal,\n\
the processing terminates.\n\
@end table\n\
\n\
Parsed words corresponding to the first specifier are returned in the first\n\
output argument and likewise for the rest of the specifiers.\n\
\n\
By default, @var{format} is @t{\"%f\"}, meaning that numbers are read from\n\
@var{str}. This will do if @var{str} contains only numeric fields.\n\
\n\
For example, the string\n\
\n\
@example\n\
@group\n\
@var{str} = \"\\\n\
Bunny Bugs 5.5\n\\\n\
Duck Daffy -7.5e-5\n\\\n\
Penguin Tux 6\"\n\
@end group\n\
@end example\n\
\n\
@noindent\n\
can be read using\n\
\n\
@example\n\
[@var{a}] = ytextscan (@var{str}, \"%s %s %f\");\n\
@end example\n\
\n\
Optional numeric argument @var{repeat} can be used for limiting the\n\
number of items read:\n\
\n\
@table @asis\n\
@item -1\n\
(default) read all of the string or file until the end.\n\
\n\
@item N\n\
Read until the first of two conditions occurs: the format has been processed\n\
N times, or N lines of the input have been processed. Zero (0) is an\n\
acceptable value for @var{repeat}. Currently, end-of-line characters\n\
inside %q, %c, and %[...]$ conversions do not contribute to the line count.\n\
This is incompatible with Matlab and may change in future.\n\
@end table\n\
\n\
The behavior of @code{ytextscan} can be changed via property-value pairs. The\n\
following properties are recognized:\n\
\n\
@table @asis\n\
@item @qcode{\"CollectOutput\"}\n\
A value of 1 or true instructs ytextscan to concatenate consecutive columns\n\
of the same class in the output cell array. A value of 0 or false (default)\n\
leaves output in distinct columns.\n\
\n\
@item @qcode{\"CommentStyle\"}\n\
Parts of @var{str} are considered comments and will be skipped.\n\
@var{value} is the comment style and can be\n\
either\n\
@c any of the following.\n\
@c \n\
@c @itemize\n\
@c @item @qcode{\"shell\"}\n\
@c Everything from @code{#} characters to the nearest end-of-line is skipped.\n\
@c \n\
@c @item @qcode{\"c\"}\n\
@c Everything between @code{/*} and @code{*/} is skipped.\n\
@c \n\
@c @item @qcode{\"c++\"}\n\
@c Everything from @code{//} characters to the nearest end-of-line is\n\
@c skipped.\n\
@c \n\
@c @item @qcode{\"matlab\"}\n\
@c Everything from @code{%} characters to the nearest end-of-line is skipped.\n\
@c \n\
@c @item user-supplied. Two options:\n\
(1) One string, or 1x1 cell string, to skip everything to the right of it;\n\
(2) A cell array of two strings, to skip everything between the first and\n\
second strings.\n\
@c @end itemize\n\
Comments are only parsed where whitespace is accepted, and do not act as\n\
delimiters.\n\
\n\
@item @qcode{\"Delimiter\"}\n\
If @var{value} is a string, any character in @var{value} will be used to\n\
split @var{str} into words. If @var{value} is a cell array of strings,\n\
any string in the array will be used to split @var{str} into words.\n\
(default value = any whitespace).\n\
\n\
@item @qcode{\"EmptyValue\"}\n\
Value to return for empty numeric values in non-whitespace delimited data.\n\
The default is NaN@. When the data type does not support NaN (int32 for\n\
example), then default is zero.\n\
\n\
@item @qcode{\"EndOfLine\"}\n\
@var{value} can be either a emtpy or one character specifying the end of line\n\
character, or the pair\n\
@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} (CRLF).\n\
In the latter case, any of\n\
@qcode{\"@xbackslashchar{}r\"}, @qcode{\"@xbackslashchar{}n\"} or\n\
@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is counted as a (single)\n\
newline.\n\
If no value is given, @qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is\n\
used.\n\
@c If set to \"\" (empty string) EOLs are ignored as delimiters and added\n\
@c to whitespace.\n\
\n\
@c When reading from a character string, optional input argument @var{n}\n\
@c specifies the number of times @var{format} should be used (i.e., to limit\n\
@c the amount of data read).\n\
@c When reading from file, @var{n} specifies the number of data lines to read;\n\
@c in this sense it differs slightly from the format repeat count in strread.\n\
\n\
@item @qcode{\"HeaderLines\"}\n\
The first @var{value} number of lines of @var{fid} are skipped.\n\
\n\
@item @qcode{\"MultipleDelimsAsOne\"}\n\
If @var{value} is non-zero,\n\
treat a series of consecutive delimiters, without whitespace in between,\n\
as a single delimiter. Consecutive delimiter series need not be vertically\n\
@qcode{\"aligned\"}.\n\
Without this option, a single delimiter before the end of the line does\n\
not cause the line to be considered to start with an empty value, but a\n\
single delimiter at the start of a line causes the line to be considered\n\
to start with an empty value.\n\
\n\
@item @qcode{\"TreatAsEmpty\"}\n\
Treat single occurrences (surrounded by delimiters or whitespace) of the\n\
string(s) in @var{value} as missing values.\n\
\n\
@item @qcode{\"ReturnOnError\"}\n\
If set to numerical 1 or true (default), return normally when read errors\n\
have been encountered. If set to 0 or false, return an error and no data.\n\
As the string or file is read by columns rather than by rows, and because\n\
ytextscan is fairly forgiving as regards read errors, setting this option\n\
may have little or no actual effect.\n\
\n\
@item @qcode{\"Whitespace\"}\n\
Any character in @var{value} will be interpreted as whitespace and trimmed;\n\
The default value for whitespace is\n\
@c Note: the next line specifically has a newline which generates a space\n\
@c in the output of qcode, but keeps the next line < 80 characters.\n\
@qcode{\"\n\
@xbackslashchar{}b@xbackslashchar{}r@xbackslashchar{}n@xbackslashchar{}t\"}\n\
(note the space). Unless whitespace is set to @qcode{\"\"} (empty) AND at\n\
least one @qcode{\"%s\"} format conversion specifier is supplied, a space is\n\
always part of whitespace.\n\
\n\
@end table\n\
\n\
When the number of words in @var{str} or @var{fid} doesn't match an exact\n\
multiple of\n\
the number of format conversion specifiers,\n\
ytextscan fills the last line with empty strings, NaNs or 0s (for integer\n\
fields). In the case that the last line does not end with an end-of-line\n\
character, this is incompatible with Matlab and subject to change.\n\
@c ytextscan's behavior depends on\n\
@c whether the last line of the string or file is an end-of-line as specified\n\
@c by the EndOfLine option:\n\
@c \n\
@c @table @asis\n\
@c @item last character = end-of-line\n\
@c Data columns are padded with empty fields, NaN or 0 so that all columns have\n\
@c equal length\n\
@c \n\
@c @item last character is not end-of-line\n\
@c Data columns are not padded; ytextscan returns columns of unequal length\n\
@c @end table\n\
\n\
If the format string is explicitly empty (not omitted) and the file contains\n\
only\n\
numeric data (excluding headerlines), ytextscan will return data in a number\n\
of columns matching the number of fields on the first data line of\n\
the input.\n\
\n\
The second output, @var{position}, provides the position, in characters\n\
from the beginning of the file or string, at which the processing stopped.\n\
\n\
@seealso{dlmread, fscanf, load, strread, textread}\n\
@end deftypefn")
{
octave_value_list retval;
std::string format;
int params = 1;
if (args.length () < 1)
print_usage ();
else if (args.length () == 1)
format = "%f"; // ommited format = %f. explicit "" = width from file
else if (args(1).is_string ())
{
format = args(1).string_value ();
if (args(1).is_sq_string ())
format = do_string_escapes (format);
params++;
}
else
error ("textscan: FORMAT must be a string, not <%s>",
args(1).class_name ().c_str ());
octave_idx_type ntimes = -1;
ytextscan tscanner;
if (args.length () >= 3)
{
if (args(2).is_numeric_type ())
{
ntimes = args(2).idx_type_value ();
if (ntimes < args(2).double_value ())
error ("textscan: N = %g is too large", args(2).double_value ());
params = 3;
}
}
ytextscan_format_list fmt_list (format);
tscanner.parse_options (args, params, fmt_list);
if (args(0).is_string ())
{
std::istringstream is (args(0).string_value ());
octave_value tmp = tscanner.scan (&is, fmt_list, ntimes);
retval(0) = tmp;
std::ios::iostate state = is.rdstate ();
is.clear ();
retval(1) = octave_value (static_cast(is.tellg ()));
is.setstate (state);
}
else
{
octave_stream os = octave_stream_list::lookup (args(0), "ytextscan");
octave_value tmp = tscanner.scan (os.input_stream (), fmt_list, ntimes);
retval(0) = tmp;
// FIXME -- warn if stream is not opened in binary mode?
std::ios::iostate state = os.input_stream ()->rdstate ();
os.input_stream ()->clear ();
retval(1) = os.tell ();
os.input_stream ()->setstate (state);
}
return retval;
}
/********************/
/* Calculate x^n */
/* Used for ...e+nn so that, for example, 1e2 is exactly 100 and 5e-1 is 1/2 */
double pown (double x, unsigned int n)
{
double retval = 1;
for (unsigned int d = n; d; d >>= 1)
{
if (d & 1)
retval *= x;
x *= x;
}
return retval;
}
/********************/
/* Read a double considering the "precision" field of fmt and the */
/* exp_chars option of options. */
double
ytextscan::read_double (dstr& is, const ytextscan_format_elt& fmt) const
{
int sign = 1;
unsigned int width_left = fmt.width;
double retval = 0;
bool valid = false; // syntactically correct double?
int ch = is.peek ();
if (ch == '+')
{
is.get ();
ch = is.peek ();
if (width_left)
width_left--;
}
else if (ch == '-')
{
sign = -1;
is.get ();
ch = is.peek ();
if (width_left)
width_left--;
}
// Read integer part
if (ch != '.')
{
if (ch >= '0' && ch <= '9') // valid if at least one digit
valid = true;
while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
retval = retval * 10 + (ch - '0');
width_left++;
}
// Read fractional part, up to specified precision
if (ch == '.' && width_left)
{
double multiplier = 1;
int precision = fmt.prec;
int i;
if (width_left)
width_left--; // Consider width of '.'
if (precision == -1)
precision = 1<<30; // FIXME Should be MAXINT
if (!valid) // if there was nothing before '.'...
is.get (); // ...ch was a "peek", not "get".
for (i = 0; i < precision; i++)
{
if (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
retval += (ch - '0') * (multiplier *= 0.1);
else
{
width_left++;
break;
}
}
// round up if we truncated and the next digit is >= 5
if ((i == precision || !width_left) && (ch = is.get ()) >= '5'
&& ch <= '9')
retval += multiplier;
if (i > 0)
valid = true; // valid if at least one digit after '.'
// skip remainder after '.', to field width, to look for exponent
if (i == precision)
while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
; /* discard */
width_left++;
}
// look for exponent part in, e.g., 6.023E+23
const char *ec = exp_chars.c_str ();
bool used_exp = false;
if (valid && width_left > 1 && strchr (ec, ch))
{
int ch1 = is.peek ();
if (ch1 == '-' || ch1 == '+' || (ch1 >= '0' && ch1 <= '9'))
{ // if 1.0e+$ or some such, this will set failbit, as we want
width_left--; // count "E"
int exp = 0;
int exp_sign = 1;
if (ch1 == '+')
{
if (width_left)
width_left--;
is.get ();
}
else if (ch1 == '-')
{
exp_sign = -1;
is.get ();
if (width_left)
width_left--;
}
valid = false;
while (width_left-- && is && (ch = is.get ()) >= '0' && ch1 <= '9')
{
exp = exp*10 + ch - '0';
valid = true;
}
width_left++;
if (ch != EOF && width_left)
is.putback (ch);
double multiplier = pown (10, exp);
if (exp_sign > 0)
retval *= multiplier;
else
retval /= multiplier;
used_exp = true;
}
}
is.clear ();
if (!used_exp && ch != EOF && width_left)
is.putback (ch);
// Check for +/- inf and NaN
if (!valid && width_left >= 3)
{
int i = lookahead (is, inf_nan, 3, false); // false -> case insensitive
if (i == 0)
{
retval = octave_Inf;
valid = true;
}
else if (i == 1)
{
retval = octave_NaN;
valid = true;
}
}
// Check for +/- inf and NaN
if (!valid && width_left >= 3)
{
int i = lookahead (is, inf_nan, 3, false); // false -> case insensitive
if (i == 0)
{
retval = octave_Inf;
valid = true;
}
else if (i == 1)
{
retval = octave_NaN;
valid = true;
}
}
if (!valid)
is.setstate (std::ios_base::failbit);
else
is.setstate (is.rdstate () & ~std::ios_base::failbit);
return retval * sign;
}
/********************/
// Read a single number: real, complex, inf, NaN,
// possibly with limited precision.
// Calls to this should be preceded by skip_whitespace.
// Calling that inside scan_complex would violate its const declaration.
void
ytextscan::scan_complex (dstr& is, const ytextscan_format_elt& fmt,
Complex& val) const
{
double im = 0;
double re = 0;
bool as_empty = false; // did we fail but match a "treat_as_empty" string?
bool inf = false;
int ch = is.peek ();
if (ch == '+' || ch == '-') // check for [+-][ij] with no coefficients
{
ch = is.get ();
int ch2 = is.peek ();
if (ch2 == 'i' || ch2 == 'j')
{
double value = 1;
is.get ();
// Check not -inf
if (is.peek () == 'n')
{
char *pos = is.tellg ();
std::ios::iostate state = is.rdstate ();
is.get ();
ch2 = is.get ();
if (ch2 == 'f')
{
inf = true;
re = (ch == '+') ? octave_Inf : -octave_Inf;
value = 0;
}
else
{
is.clear (state);
is.seekg (pos); // reset to position before look-ahead
}
}
im = (ch == '+') ? value : -value;
}
else
is.putback (ch);
}
if (!im && !inf) // if not [+-][ij] or [+-]inf, read real normally
{
char *pos = is.tellg ();
std::ios::iostate state = is.rdstate ();
//re = octave_read_value (is);
re = read_double (is, fmt);
// check for "treat as empty" string
if (treat_as_empty.numel ()
&& (is.fail () || octave_is_NaN_or_NA (Complex (re))
|| re == octave_Inf))
{
for (int i = 0; i < treat_as_empty.numel (); i++)
{
if (ch == treat_as_empty (i).string_value ()[0])
{
as_empty = true; // first char matches, so read the lot
break;
}
}
if (as_empty) // if first char matched...
{
as_empty = false; // ...look for the whole string
is.clear (state); // treat_as_empty "-" causes partial read
is.seekg (pos); // reset to position before failed read
// treat_as_empty strings may be different sizes.
// Read ahead longest, put it all back, then re-read the string
// that matches.
char *look, look_buf [treat_as_empty_len + 1];
// prefill, in case EOF means part-filled.
memset (look_buf, '\0', treat_as_empty_len);
look = is.read (look_buf, treat_as_empty_len, pos);
is.clear (state);
is.seekg (pos); // reset to position before look-ahead
// FIXME -- is.read could invalidate pos
for (int i = 0; i < treat_as_empty.numel (); i++)
{
std::string s = treat_as_empty (i).string_value ();
if (!strncmp (s.c_str (), look, s.size ()))
{
as_empty = true;
// read just the right amount
is.read (look_buf, s.size (), pos);
break;
}
}
}
}
if (!is.eof () && !as_empty)
{
state = is.rdstate (); // before tellg, since that fails at EOF
pos = is.tellg ();
ch = is.peek (); // ch == EOF if read failed; no need to chk fail
if (ch == 'i' || ch == 'j') // pure imaginary
{
is.get ();
im = re;
re = 0;
}
else if (ch == '+' || ch == '-') // see if it is real+imag[ij]
{
// save stream state in case we have to restore it
pos = is.tellg ();
state = is.rdstate ();
//im = octave_read_value (is);
im = read_double (is, fmt);
if (is.fail ())
im = 1;
if (is.peek () == 'i' || is.peek () == 'j')
is.get ();
else
{
im = 0; // no valid imaginary part. Restore state
is.clear (state); // eof shouldn't cause fail.
is.seekg (pos);
}
}
else if (is.eof ()) // we've read enough to be a "valid" read
is.clear (state); // failed peek shouldn't cause fail
}
}
if (as_empty)
val = empty_value.scalar_value ();
else
val = Complex (re, im);
}
/********************/
// Return in val the run of characters from is NOT contained in pattern.
int
ytextscan::scan_caret (dstr& is, const char *pattern, std::string& val)
const
{
int c1 = EOF;
std::ostringstream obuf; // Is this optimised for growing?
while (is && (c1 = (is && !is.eof ()) ? is.get_undelim () : EOF) != EOF
&& !strchr (pattern, c1))
obuf << static_cast (c1);
val = obuf.str ();
if (c1 != EOF)
is.putback (c1);
return c1;
}
/********************/
// Read until one of the strings in delimiters is found.
// For efficiency, ends is a list of the last character of each delimiter
std::string
ytextscan::read_until (dstr& is, const Cell& delimiters,
const std::string& ends) const
{
std::string retval ("");
bool done = false;
do
{ // find sequence ending with an ending char
std::string next;
scan_caret (is, ends.c_str (), next);
retval = retval + next; // FIXME -- could use repeated doubling of size
int last = (!is.eof ()) ? is.get_undelim () : EOF;
if (last != EOF)
{
retval = retval + static_cast (last);
for (int i = 0; i < delimiters.numel (); i++)
{
std::string delim = delimiters(i).string_value ();
int start = retval.length () - delim.length ();
if (start < 0)
start = 0;
std::string may_match = retval.substr (start);
if (may_match == delim)
{
done = true;
retval = retval.substr (0, start);
break;
}
}
}
}
while (!done && is && !is.eof ());
return retval;
}
/********************/
// Read stream until either fmt.width chars have been read, or options.delimiter
// has been found
// Does *not* rely on fmt being 's' -- used by formats like %6f to limit to 6
void
ytextscan::scan_string (dstr& is, const ytextscan_format_elt& fmt, std::string& val) const
{
if (delim_list.numel () == 0)
{
unsigned int i = 0;
unsigned int width = fmt.width;
for (i = 0; i < width; i++)
{
if (i+1 > val.length ())
val = val + val + ' '; // grow even if empty
int ch = is.get ();
if (is_delim (ch) || ch == EOF)
{
is.putback (ch);
break;
}
else
val[i] = ch;
}
val = val.substr (0, i); // trim pre-allocation
}
else // Cell array of multi-character delimiters
{
std::string ends ("");
for (int i = 0; i < delim_list.numel (); i++)
{
std::string tmp = delim_list(i).string_value ();
ends += tmp.substr (tmp.length () - 1);
}
val = ytextscan::read_until (is, delim_list, ends);
}
}
/********************/
// Return in val the run of characters from is contained in pattern.
int
ytextscan::scan_bracket (dstr& is, const char *pattern, std::string& val)
const
{
int c1 = EOF;
std::ostringstream obuf; // Is this optimised for growing?
while (is && strchr (pattern, (c1 = is.get_undelim ())))
obuf << static_cast (c1);
val = obuf.str ();
if (c1 != EOF)
is.putback (c1);
return c1;
}
/********************/
// Return in val a string, either delimited by whitespace/delimiters, or
// enclosed in a pair of double quotes ("..."). Enclosing quotes are removed.
// A consecutive pair "" is inserted into val as a single ".
void
ytextscan::scan_qstring (dstr& is, const ytextscan_format_elt& fmt,
std::string& val)
{
skip_whitespace (is);
if (is.peek () != '\"')
scan_string (is, fmt, val);
else
{
is.get ();
scan_caret (is, "\"", val); // read everything until "
is.get (); // swallow "
while (is && is.peek () == '\"') // if double ", insert one in stream,
{ // and keep looking for single "
is.get ();
std::string val1;
scan_caret (is, "\"", val1);
val = val + "\"" + val1;
is.get_undelim ();
}
}
}
/********************/
// Read from is into val a string of the next fmt.width characters,
// including any whitespace or delimiters.
void
ytextscan::scan_cstring (dstr& is, const ytextscan_format_elt& fmt,
std::string& val) const
{
int ch;
val.resize (fmt.width);
for (unsigned int i = 0; is && i < fmt.width; i++)
if ((ch = is.get_undelim ()) != EOF)
val[i] = ch;
else
{
val.resize (i);
break;
}
}
/**
* Read a single '%...' conversion and place it in position row of ov.
*/
void
ytextscan::scan_one (dstr& is, const ytextscan_format_elt& fmt,
octave_value& ov, Array row)
{
skip_whitespace (is);
is.clear ();
octave_value val;
if (fmt.numeric)
{
if (fmt.type == 'f' || fmt.type == 'n')
{
Complex v;
skip_whitespace (is);
scan_complex (is, fmt, v);
if (!fmt.discard && !is.fail ())
{
if (fmt.bitwidth == 64)
{
if (ov.is_real_type () && v.imag () == 0)
ov.internal_rep ()->fast_elem_insert (row(0), v.real ());
else
{
if (ov.is_real_type ()) // cat does type conversion
ov = do_cat_op (ov, octave_value (v), row);
else
ov.internal_rep ()->fast_elem_insert (row(0), v);
}
}
else
{
if (ov.is_real_type () && v.imag () == 0)
ov.internal_rep ()->fast_elem_insert (row(0),
float (v.real ()));
else
{
if (ov.is_real_type ()) // cat does type conversion
ov = do_cat_op (ov, octave_value (v), row);
else
ov.internal_rep ()->fast_elem_insert (row(0),
FloatComplex (v));
}
}
}
}
else
{
double v; // Matlab docs say 1e30 etc should be valid for %d and
// 1000 as a %d8 should be 127, so read as double.
// Some loss of precision for d64 and u64.
skip_whitespace (is);
v = read_double (is, fmt);
if (!fmt.discard && !is.fail ())
switch (fmt.bitwidth)
{
case 64:
switch (fmt.type)
{
case 'd':
{
octave_int64 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
case 'u':
{
octave_uint64 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
}
break;
case 32:
switch (fmt.type)
{
case 'd':
{
octave_int32 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
case 'u':
{
octave_uint32 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
}
break;
case 16:
if (fmt.type == 'd')
{
octave_int16 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
else
{
octave_uint16 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
case 8:
if (fmt.type == 'd')
{
octave_int8 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
else
{
octave_uint8 vv = v;
ov.internal_rep ()->fast_elem_insert (row(0), vv);
}
break;
}
}
if (is.fail ())
{
if (!fmt.discard)
ov = do_cat_op (ov, empty_value, row);
std::string dummy; // FIXME: should fail instead of skipping
std::ios::iostate state = is.rdstate ();
is.clear (); // clear to allow read pointer to advance
scan_string (is, fmt, dummy);
is.setstate (state);
}
}
else
{
std::string vv (" "); // initial buffer. Grows as needed
switch (fmt.type)
{
case 's':
scan_string (is, fmt, vv);
break;
case 'q':
scan_qstring (is, fmt, vv);
break;
case 'c':
scan_cstring (is, fmt, vv);
break;
case '[':
scan_bracket (is, fmt.char_class.c_str (), vv);
break;
case '^':
scan_caret (is, fmt.char_class.c_str (), vv);
break;
}
if (!fmt.discard)
ov.internal_rep ()->fast_elem_insert (row (0),
Cell (octave_value (vv)));
// FIXME -- why does failbit get set at EOF, instead of eofbit?
if (vv.length () != 0)
is.clear (is.rdstate () & ~std::ios_base::failbit);
}
is.field_done ();
}
/**
* Read data corresponding to the entire format string once,
* placing the values in row row of retval.
*/
int
ytextscan::read_format_once (dstr& is,
ytextscan_format_list& fmt_list,
std::list & retval,
Array row)
{
const ytextscan_format_elt *elem = fmt_list.first ();
std::list::iterator out=retval.begin ();
bool no_conversions = true;
bool done = false;
int i;
for (i = 0; i < fmt_list.numel (); i++)
{
is.clear (); // clear fail of previous numeric conversions
switch (elem->type)
{
case 'C':
case 'D':
std::cerr << "ytextscan: Conversion %" << elem->type
<< " not yet implemented\n";
break;
case 'u':
case 'd':
case 'f':
case 'n':
case 's':
case '[':
case '^':
case 'q':
case 'c':
scan_one (is, *elem, *out, row);
break;
case ytextscan_format_elt::literal_conversion :
match_literal (is, *elem);
break;
default:
error ("Unknown format element '%c'", elem->type);
}
if (!is.fail () && !elem->discard)
no_conversions = false;
if (is.eof ())
done = true; // note EOF, but process others to get empty_val
else
is.clear (is.rdstate () & ~std::ios::failbit);
if (!elem->discard)
out++;
elem = fmt_list.next ();
// FIXME -- these conversions "ignore delimiters". Should they include
// delimiters at the start of the conversion, or can those be skipped?
if (elem->type != ytextscan_format_elt::literal_conversion
// && elem->type != '[' && elem->type != '^' && elem->type != 'c'
)
skip_delim (is);
}
if (done)
is.setstate (std::ios::eofbit);
// returning -2 means "error, and no columns read this row
if (is.eof ())
return (-1 - no_conversions);
return 1;
}
/********************/
void
ytextscan::parse_options (const octave_value_list& args, int first_param,
ytextscan_format_list& fmt_list)
{
int last = args.length ();
int n = last - first_param;
if (n & 1)
error ("ytextscan: %d parameters given, but only %d values", n-n/2, n/2);
delim_len = 1;
bool have_delims = false;
for (int i = first_param; i < last; i += 2)
{
if (!args(i).is_string ())
error ("ytextscan: Invalid paramter type <%s> for parameter %d",
args(i).type_name ().c_str (), (i-first_param)/2 + 1);
std::string param = args(i).string_value ();
std::transform (param.begin (), param.end (),
param.begin (), ::tolower);
if (param == "delimiter")
{
bool invalid = true;
if (args(i+1).is_string ())
{
invalid = false;
have_delims = true;
delims = args(i+1).string_value ();
}
else if (args(i+1).is_cell ())
{
invalid = false;
delim_list = args(i+1).cell_value ();
delim_table = " "; // non-empty, to flag non-default delim
// Check that all elements are strings, and find max length
for (int j = 0; j < delim_list.numel (); j++)
{
if (!delim_list(j).is_string ())
invalid = true;
else
{
octave_idx_type len = delim_list(j).string_value ()
.length ();
delim_len = std::max (static_cast(len), delim_len);
}
}
}
if (invalid)
error ("ytextscan: Delimiters must be either a string or "
"cell array of strings");
}
else if (param == "commentstyle")
{
if (args(i+1).is_string ())
{ // check here for names like "C++", "C", "shell", ...?
comment_style = Cell (args(i+1));
}
else if (args(i+1).is_cell ())
{
comment_style = args(i+1).cell_value ();
int len = comment_style.numel ();
if ((len >= 1 && !comment_style (0).is_string ())
|| (len >= 2 && !comment_style (1).is_string ())
|| (len >= 3))
error ("ytextscan: CommentStyle must be either a string or "
"cell array of one or two strings");
}
else
error ("ytextscan: CommentStyle must be either a string"
" or cell array of one or two strings, not <%s>",
args(i+1).class_name ().c_str ());
// How far ahead do we need to look to detect an open comment
// and which character do we look for?
if (comment_style.numel () >= 1)
{
comment_len = comment_style (0).string_value ().size ();
comment_char = comment_style (0).string_value ()[0];
}
}
else if (param == "treatasempty")
{
bool invalid = false;
if (args(i+1).is_string ())
{
treat_as_empty = Cell (args(i+1));
treat_as_empty_len = args(i+1).string_value ().size ();
}
else if (args(i+1).is_cell ())
{
treat_as_empty = args(i+1).cell_value ();
for (int j = 0; j < treat_as_empty.numel (); j++)
if (!treat_as_empty (j).is_string ())
invalid = true;
else
{
int k = treat_as_empty (j).string_value ().size ();
if (k > treat_as_empty_len)
treat_as_empty_len = k;
}
}
if (invalid)
error ("ytextscan: TreatAsEmpty must be either a string or "
"cell array of one or two strings");
// FIXME Ensure none is a prefix of a later one. Sort by length?
}
else if (param == "collectoutput")
{
if (args(i+1).is_numeric_type ())
collect_output = args(i+1).bool_value ();
else
error ("ytextscan: CollectOutput must be logical or numeric");
}
else if (param == "emptyvalue")
{
if (args(i+1).is_numeric_type ())
empty_value = args(i+1).scalar_value ();
else
error ("ytextscan: EmptyValue must be numeric, not <%s>",
args(i+1).class_name ().c_str ());
}
else if (param == "headerlines")
{
if (args(i+1).is_numeric_type ())
header_lines = args(i+1).scalar_value ();
else
error ("ytextscan: HeaderLines must be numeric");
}
else if (param == "multipledelimsasone")
{
if (args(i+1).is_numeric_type ())
{
if (args(i+1).bool_value ())
multiple_delims_as_one = true;
}
else
error ("ytextscan: MultipleDimsAsOne must be logical or numeric");
}
else if (param == "returnonerror")
{
if (args(i+1).is_numeric_type ())
{
if (!args(i+1).bool_value ())
return_on_error = false;
}
else
error ("ytextscan: ReturnOnError must be logical or numeric");
}
else if (param == "whitespace")
{
if (args(i+1).is_string ())
whitespace = args(i+1).string_value ();
else
error ("ytextscan: Whitespace must be a character string");
}
else if (param == "expchars")
{
if (args(i+1).is_string ())
{
exp_chars = args(i+1).string_value ();
default_exp = false;
}
else
error ("ytextscan: ExpChars must be a character string");
}
else if (param == "endofline")
{
bool valid = true;
if (args(i+1).is_string ())
{
std::string s = args(i+1).string_value ();
if (args(i+1).is_sq_string ())
s = do_string_escapes (s);
int l = s.length ();
if (l == 0)
eol1 = eol2 = -2;
else if (l == 1)
eol1 = eol2 = s.c_str ()[0];
else if (l == 2)
{
eol1 = s.c_str ()[0];
eol2 = s.c_str ()[1];
if (eol1 != '\r' || eol2 != '\n') // Why limit it?
valid = false;
}
else
valid = false;
}
else
valid = false;
if (!valid)
error ("ytextscan: EndOfLine must be at most one character "
"or '\\r\\n'");
}
else
error ("ytextscan: Unrecognised option '%s'", param.c_str ());
}
whitespace_table = std::string (256, '\0');
for (unsigned int i = 0; i < whitespace.length (); i++)
whitespace_table[whitespace[i]] = '1';
// For Matlab compatibility, add 0x20 to whitespace, unless
// whitespace is explicitly ignored.
if (! (whitespace.length () == 0 && fmt_list.has_string))
whitespace_table[' '] = '1';
// Create look-up table of delimiters, based on 'delimiter'
delim_table = std::string (256, '\0');
delim_table[eol1] = '1'; // EOL is always a delimiter
delim_table[eol2] = '1'; // EOL is always a delimiter
if (!have_delims)
for (unsigned int i = 0; i < 256; i++)
{
if (isspace (i))
delim_table[i] = '1';
}
else
for (unsigned int i = 0; i < delims.length (); i++)
delim_table[delims[i]] = '1';
}
/********************/
// skip comments, and characters specified by the "Whitespace" option
// If EOLstop==true, doesn't skip end of line
int
ytextscan::skip_whitespace (dstr& is, bool EOLstop)
{
int c1 = EOF;
bool found_comment = false;
do
{
found_comment = false;
int prev = -1;
while (is && (c1 = is.get_undelim ()) != EOF
&& ( ( (c1 == eol1 || c1 == eol2) && ++lines && !EOLstop)
|| isspace (c1)))
{
if (prev == eol1 && eol1 != eol2 && c1 == eol2)
lines--;
prev = c1;
}
if (c1 == comment_char) // see if we match an open comment
{
// save stream state in case we have to restore it
char *pos = is.tellg ();
std::ios::iostate state = is.rdstate ();
char *look, buf [comment_len];
look = is.read (buf, comment_len-1, pos); // already read first char
if (is && !strncmp (comment_style(0).string_value ().substr (1)
.c_str (), look, comment_len-1))
{
found_comment = true;
std::string dummy;
char eol [3] = {static_cast (eol1),
static_cast (eol2),
'\0'};
if (comment_style.numel () == 1) // skip to end of line
{
scan_caret (is, eol, dummy);
c1 = is.get_undelim ();
if (c1 == eol1 && eol1 != eol2 && is.peek_undelim () == eol2)
is.get_undelim ();
lines++;
}
else // matching pair
{
std::string end_c = comment_style(1).string_value ();
// last char of end-comment sequence
char last = *(end_c.substr (end_c.length ()-1).c_str ());
std::string may_match ("");
do
{ // find sequence ending with last char
scan_caret (is, &last, dummy);
is.get_undelim (); // (read last itself)
may_match = may_match + dummy + last;
int start = may_match.length () - end_c.length ();
if (start < 0)
start = 0;
may_match = may_match.substr (start);
}
while (may_match != end_c && is && !is.eof ());
}
}
else // wasn't really a comment; restore state
{
is.clear (state);
is.seekg (pos);
}
}
}
while (found_comment);
if (c1 != EOF)
is.putback (c1);
return c1;
}
/********************/
// See if the next few characters match one of the strings in target.
// For efficiency, max_len is the cached longest length of any target.
// Returns -1 if none is found, or the inde of the match.
int
ytextscan::lookahead (dstr& is, const Cell& targets, int max_len,
bool case_sensitive) const
{
// target strings may be different sizes.
// Read ahead longest, put it all back, then re-read the string
// that matches.
char *pos = is.tellg ();
char *look, buf [max_len + 1];
memset (buf, '\0', max_len); // prefill, in case EOF means part-filled.
look = is.read (buf, max_len, pos);
is.clear ();
is.seekg (pos); // reset to position before look-ahead
// FIXME pos may be corrupted by is.read
int i;
int (*compare)(const char *, const char *, size_t);
compare = case_sensitive ? strncmp : strncasecmp;
for (i = 0; i < targets.numel (); i++)
{
std::string s = targets (i).string_value ();
if (!(*compare) (s.c_str (), look, s.size ()))
{
is.read (buf, s.size (), pos); // read just the right amount
break;
}
}
if (i == targets.numel ())
i = -1;
return i;
}
/********************/
// Skip delimiters -- multiple if MultipleDelimsAsOne specified.
int
ytextscan::skip_delim (dstr& is)
{
int c1 = skip_whitespace (is, true); // 'true': stop once EOL is read
if (delim_list.numel () == 0) // single character delimiter
{
if (is_delim (c1) || c1 == eol1 || c1 == eol2)
{
is.get ();
if (c1 == eol1 && is.peek_undelim () == eol2)
is.get_undelim (); // if \r\n, skip the \n too.
if (multiple_delims_as_one)
{
int prev = -1;
// skip multiple delims.
// Increment lines for each end-of-line seen; for \r\n, decrement
while (is && (c1 = is.get_undelim ()) != EOF
&& (((c1 == eol1 || c1 == eol2) && ++lines)
|| isspace (c1) || is_delim (c1)))
{
if (prev == eol1 && eol1 != eol2 && c1 == eol2)
lines--;
prev = c1;
}
if (c1 != EOF)
is.putback (c1);
}
}
}
else // multi-character delimiter
{
int first_match;
if (c1 == eol1 || c1 == eol2
|| (-1 != (first_match = lookahead (is, delim_list, delim_len))))
{
if (c1 == eol1)
{
is.get_undelim ();
if (is.peek_undelim () == eol2)
is.get_undelim ();
}
else if (c1 == eol2)
{
is.get_undelim ();
}
if (multiple_delims_as_one)
{
int prev = -1;
// skip multiple delims.
// Increment lines for each end-of-line seen; for \r\n, decrement
while (is && (c1 = skip_whitespace (is, true)) != EOF
&& (((c1 == eol1 || c1 == eol2) && ++lines)
|| -1 != lookahead (is, delim_list, delim_len)))
{
if (prev == eol1 && eol1 != eol2 && c1 == eol2)
lines--;
prev = c1;
}
}
}
}
return c1;
}
/********************/
// Read in as much of the input as coincides with the literal in
// the format string.
// Return "true" if the entire literal is matched, else false (and set failbit)
bool
ytextscan::match_literal (dstr& is, const ytextscan_format_elt& fmt)
{
// "false" -> treat EOL as normal space
// since a delimiter at the start of a line is a mismatch, not empty field
skip_whitespace (is, false);
for (unsigned int i = 0; i < fmt.width; i++)
{
int ch = is.get_undelim ();
if (ch != fmt.text[i])
{
if (ch != EOF)
is.putback (ch);
is.setstate (std::ios_base::failbit);
return false;
}
}
return true;
}
/********************/
ytextscan_format_list::ytextscan_format_list (const std::string& s)
: set_from_first (false), has_string (false), nconv (0), curr_idx (0),
list (dim_vector (16, 1)), buf (0)
{
octave_idx_type num_elts = 0;
size_t n = s.length ();
size_t i = 0;
unsigned int width = -1; // Unspecified width = max (except %c)
int prec = -1;
int bitwidth = 0;
bool discard = false;
char type = '\0';
bool have_more = true;
if (s.length () == 0)
{
buf = new std::ostringstream ("%f");
bitwidth = 64;
type = 'f';
add_elt_to_list (width, prec, bitwidth, octave_value (NDArray ()),
discard, type, num_elts);
have_more = false;
set_from_first = true;
nconv = 1;
}
else
{
set_from_first = false;
while (i < n)
{
have_more = true;
if (! buf)
buf = new std::ostringstream ();
if (s[i] == '%' && (i+1 == n || s[i+1] != '%'))
{
// Process percent-escape conversion type.
process_conversion (s, i, n, num_elts);
have_more = (buf != 0);
}
else if (isspace (s[i]))
{
while (++i < n && isspace (s[i]))
/* skip whitespace */;
have_more = false;
}
else
{
type = ytextscan_format_elt::literal_conversion;
width = 0;
prec = -1;
bitwidth = 0;
discard = true;
while (i < n && ! isspace (s[i])
&& (s[i] != '%' || (i+1 < n && s[i+1] == '%')))
{
if (s[i] == '%') // if double %, skip one
i++;
*buf << s[i++];
width++;
}
add_elt_to_list (width, prec, bitwidth, octave_value (), discard,
type, num_elts);
have_more = false;
}
if (nconv < 0)
{
have_more = false;
break;
}
}
}
if (have_more)
add_elt_to_list (width, prec, bitwidth, octave_value (), discard, type, num_elts);
list.resize (dim_vector (num_elts, 1));
delete buf;
}
/********************/
ytextscan_format_list::~ytextscan_format_list (void)
{
octave_idx_type n = list.numel ();
for (octave_idx_type i = 0; i < n; i++)
{
ytextscan_format_elt *elt = list(i);
delete elt;
}
}
/********************/
void
ytextscan_format_list::add_elt_to_list (unsigned int width, int prec,
int bitwidth, octave_value val_type,
bool discard, char type,
octave_idx_type& num_elts,
const std::string& char_class)
{
if (buf)
{
std::string text = buf->str ();
if (! text.empty ())
{
ytextscan_format_elt *elt
= new ytextscan_format_elt (text.c_str (), width, prec, bitwidth,
discard, type, char_class);
if (num_elts == list.numel ())
{
list.resize (dim_vector (2 * num_elts, 1));
}
if (!discard)
{
output_container.push_back (val_type);
}
list(num_elts++) = elt;
}
delete buf;
buf = 0;
}
}
/********************/
void
ytextscan_format_list::process_conversion (const std::string& s, size_t& i,
size_t n, octave_idx_type& num_elts)
{
unsigned width = 0;
int prec = -1;
int bitwidth = 0;
bool discard = false;
octave_value val_type;
char type = '\0';
*buf << s[i++];
bool have_width = false;
while (i < n)
{
switch (s[i])
{
case '*':
if (discard)
nconv = -1;
else
{
discard = true;
*buf << s[i++];
}
break;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
if (have_width)
nconv = -1;
else
{
char c = s[i++];
width = width * 10 + c - '0';
have_width = true;
*buf << c;
while (i < n && isdigit (s[i]))
{
c = s[i++];
width = width * 10 + c - '0';
*buf << c;
}
if (i < n && s[i] == '.')
{
*buf << s[i++];
prec = 0;
while (i < n && isdigit (s[i]))
{
c = s[i++];
prec = prec * 10 + c - '0';
*buf << c;
}
}
}
break;
case 'd': case 'u':
{
bool done = true;
*buf << (type = s[i++]);
if (i < n)
{
if (s[i] == '8')
{
bitwidth = 8;
if (type == 'd')
val_type = octave_value (int8NDArray ());
else
val_type = octave_value (uint8NDArray ());
*buf << s[i++];
}
else if (s[i] == '1' && i+1 < n && s[i+1] == '6')
{
bitwidth = 16;
if (type == 'd')
val_type = octave_value (int16NDArray ());
else
val_type = octave_value (uint16NDArray ());
*buf << s[i++];
*buf << s[i++];
}
else if (s[i] == '3' && i+1 < n && s[i+1] == '2')
{
done = false; // use default size below
*buf << s[i++];
*buf << s[i++];
}
else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
{
bitwidth = 64;
if (type == 'd')
val_type = octave_value (int64NDArray ());
else
val_type = octave_value (uint64NDArray ());
*buf << s[i++];
*buf << s[i++];
}
else
done = false;
}
else
done = false;
if (!done)
{
bitwidth = 32;
if (type == 'd')
val_type = octave_value (int32NDArray ());
else
val_type = octave_value (uint32NDArray ());
}
goto fini;
}
case 'f':
*buf << (type = s[i++]);
bitwidth = 64;
if (i < n)
{
if (s[i] == '3' && i+1 < n && s[i+1] == '2')
{
bitwidth = 32;
val_type = octave_value (FloatNDArray ());
*buf << s[i++];
*buf << s[i++];
}
else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
{
val_type = octave_value (NDArray ());
*buf << s[i++];
*buf << s[i++];
}
else
val_type = octave_value (NDArray ());
}
else
val_type = octave_value (NDArray ());
goto fini;
case 'n':
*buf << (type = s[i++]);
bitwidth = 64;
val_type = octave_value (NDArray ());
goto fini;
case 's': case 'q': case '[': case 'c':
if (!discard)
val_type = octave_value (Cell ());
*buf << (type = s[i++]);
has_string = true;
goto fini;
fini:
{
if (!have_width)
{
if (type == 'c') // %c defaults to one character
width = 1;
else
width = static_cast (-1); // others: unlimited
}
if (finish_conversion (s, i, n, width, prec, bitwidth, val_type,
discard, type, num_elts) == 0)
return;
}
break;
default:
error ("ytextscan: '%%%c' is not a valid format specifier", s[i]);
}
if (nconv < 0)
break;
}
nconv = -1;
}
/**
* Parse [...] and [^...]
* Matlab does not expand expressions like A-Z, but they are useful, and so
* we parse them "carefully". We treat '-' as a usual character unless
* both start and end characters are from the same class (upper case, lower
* case, numeric), or this is not the first '-' in the pattern.
*
* Keeps both a running list of characters and a mask of which chars have
* occurred. The first is efficient for patterns with few characters. The
* latter is efficient for [^...] patterns and makes it simpler to
*/
static std::string
textscan_char_class (const std::string& pattern)
{
int len = pattern.length ();
if (len == 0)
return "";
std::string retval (256, '\0');
std::string mask (256, '\0'); // number of times chr has been seen
int in = 0, out = 0;
unsigned char ch, prev = 0;
bool flip = false;
ch = pattern[in];
if (ch == '^')
{
in++;
flip = true;
}
mask[pattern[in]] = '\1';
retval[out++] = pattern[in++]; // even copy ']' if it is first
for (; in < len; in++)
{
bool was_range = false;
ch = pattern[in];
if (ch == ']')
break;
if (prev == '-' && in > 1 && isalnum (ch))
{
unsigned char start_of_range = pattern[in-2];
if (start_of_range < ch
&& ((isupper (ch) && isupper (start_of_range))
|| (islower (ch) && islower (start_of_range))
|| (isdigit (ch) && isdigit (start_of_range))
|| mask['-'] > 1)) // not the first '-'
{
was_range = true;
out--;
mask['-']--;
for (int i = start_of_range; i <= ch; i++)
{
if (mask[i] == '\0')
{
mask[i] = '\1';
retval[out++] = i;
}
}
}
}
if (!was_range)
{
if (mask[ch]++ == 0)
retval[out++] = ch;
else
warning_with_id ("octave:textscan-pattern",
"textscan: [...] contains two '%c'%s", ch,
(ch == '-') ? " outside range expressions" : "");
}
}
if (flip) // [^...]
{
out = 0;
for (int i = 0; i < 256; i++)
if (!mask[i])
retval[out++] = i;
}
retval.resize (out);
return retval;
}
/********************/
int
ytextscan_format_list::finish_conversion (const std::string& s, size_t& i,
size_t n, unsigned int& width,
int& prec, int& bitwidth,
octave_value& val_type, bool discard,
char& type, octave_idx_type& num_elts)
{
int retval = 0;
std::string char_class;
size_t beg_idx = std::string::npos;
size_t end_idx = std::string::npos;
if (type != '%')
{
nconv++;
if (type == '[')
{
if (i < n)
{
beg_idx = i;
if (s[i] == '^')
{
type = '^';
*buf << s[i++];
if (i < n)
{
beg_idx = i;
if (s[i] == ']')
*buf << s[i++];
}
}
else if (s[i] == ']')
*buf << s[i++];
}
while (i < n && s[i] != ']')
*buf << s[i++];
if (i < n && s[i] == ']')
{
end_idx = i-1;
*buf << s[i++];
}
if (s[i-1] != ']')
retval = nconv = -1;
}
}
if (nconv >= 0)
{
if (beg_idx != std::string::npos && end_idx != std::string::npos)
char_class = textscan_char_class (s.substr (beg_idx,
end_idx - beg_idx + 1));
add_elt_to_list (width, prec, bitwidth, val_type, discard, type,
num_elts, char_class);
}
return retval;
}
/********************
void
ytextscan_format_list::printme (void) const
{
octave_idx_type n = list.numel ();
for (octave_idx_type i = 0; i < n; i++)
{
ytextscan_format_elt *elt = list(i);
std::cerr
<< "width: " << elt->width << "\n"
<< "digits " << elt->prec << "\n"
<< "bitwidth: " << elt->bitwidth << "\n"
<< "discard: " << elt->discard << "\n"
<< "type: ";
if (elt->type == ytextscan_format_elt::literal_conversion)
std::cerr << "literal text\n";
else if (elt->type == ytextscan_format_elt::whitespace_conversion)
std::cerr << "whitespace\n";
else
std::cerr << elt->type << "\n";
std::cerr
<< "char_class: `" << undo_string_escapes (elt->char_class) << "'\n"
<< "text: `" << undo_string_escapes (elt->text) << "'\n\n";
}
}
* */
/********************/
// If FORMAT is explicitly "", it is assumed to be "%f" repeated enough times
// to read the first row of the file. Set it now.
void
ytextscan_format_list::read_first_row (dstr& is, ytextscan& ts)
{
// Read first line and strip end-of-line, which may be two characters
std::string first_line (20, ' ');
is.getline (first_line, static_cast (ts.eol2));
if (first_line.length () > 0
&& first_line[first_line.length () - 1] == ts.eol1)
first_line.resize (first_line.length () - 1);
std::istringstream strstr (first_line);
dstr ds (strstr, is);
dim_vector dv (1,1); // initial size of each output_container
Complex val;
octave_value val_type;
nconv = 0;
int max_empty = 1000; // failsafe, if ds fails but not with eof
const char *pos;
// read line, creating output_container as we go
while (!ds.eof ())
{
ts.skip_whitespace (ds);
ds.progress_benchmark ();
ts.scan_complex (ds, *list(0), val);
if (ds.fail ())
{
if (ds.eof ())
break;
else // allow missing values in first row
{
ds.clear (strstr.rdstate () & ~std::ios_base::failbit);
val = ts.empty_value.scalar_value ();
if (!--max_empty)
break;
}
}
if (val.imag () == 0)
val_type = octave_value (NDArray (dv, val.real ()));
else
val_type = octave_value (ComplexNDArray (dv, val));
output_container.push_back (val_type);
ts.skip_delim (ds);
if (ds.no_progress ())
break;
nconv++;
}
output_container.pop_front (); // discard empty element from constructor
//Create fmt now that the size is known
list.resize (dim_vector (nconv, 1));
for (octave_idx_type i = 1; i < nconv; i++)
list(i) = new ytextscan_format_elt (*list(0));
}
/**
* Perform actual ytextscan: read data from stream, and create cell array
*/
octave_value
ytextscan::scan (std::istream *isp, ytextscan_format_list& fmt_list,
octave_idx_type ntimes)
{
if (!isp)
error ("internal error: ytextscan called with invalid istream");
if (fmt_list.num_conversions () == -1)
error ("ytextscan: invalid format specified");
if (fmt_list.num_conversions () == 0)
error ("ytextscan: no valid format conversion specifiers\n");
octave_value retval = Cell ();
// skip the first header_lines
std::string dummy;
for (int i = 0; i < header_lines && *isp; i++)
getline (*isp, dummy, static_cast (eol2));
// Create our own buffered stream, for fast get/putback/tell/seek.
// First, see how far ahead it should let us look.
int max_lookahead = std::max (std::max (comment_len, treat_as_empty_len),
std::max (delim_len, 3)); // 3 for NaN and Inf
// Next, choose a buffer size to avoid reading too much, or too often.
octave_idx_type buf_size = 4096;
if (ntimes > 0)
{
buf_size = 80 * ntimes;
if (buf_size < ntimes) // if overflow...
buf_size = ntimes;
buf_size = std::max (ntimes, std::min (buf_size, 4096));
}
// Finally, create the stream.
dstr is (*isp, whitespace + delims, max_lookahead, buf_size);
// Grow retval dynamically. "size" is half the initial size
// (FIXME -- Should we start smaller if ntimes is large?)
octave_idx_type size = ((ntimes < 8 && ntimes >= 0) ? ntimes : 1);
Array row_idx (dim_vector (1,2));
row_idx(1) = 0;
int err = 0;
octave_idx_type row = 0;
if (multiple_delims_as_one) // bug #44750?
skip_delim (is);
// If FORMAT explicitly "", read first line and see how many "%f" match
if (fmt_list.set_from_first)
{
row = 1; // the above puts the first line into fmt_list.out_buf ()
fmt_list.read_first_row (is, *this);
lines = 1;
}
std::list out = fmt_list.out_buf ();
// Check which columns to merge. Reals may become complex, and so we can't
// trust types after reading in data.
bool merge_with_prev[fmt_list.numel ()];
int conv = 0;
if (collect_output)
{
int prev_type = -1;
for (std::list::iterator col = out.begin ();
col != out.end (); col++)
{
if (col->type_id () == prev_type)
merge_with_prev [conv++] = true;
else
merge_with_prev [conv++] = false;
prev_type = col->type_id ();
}
}
// This should be caught by earlier code, but this avoids a possible
// infinite loop below.
if (fmt_list.num_conversions () == 0)
error ("ytextscan: No conversions specified");
// Read the data
for (/* row set ~25 lines above */; row < ntimes || ntimes == -1; row++)
{
if (row == 0 || row >= size)
{
size += size+1;
for (std::list::iterator col = out.begin ();
col != out.end (); col++)
*col = (*col).resize (dim_vector (size, 1), 0);
}
row_idx(0) = row;
err = read_format_once (is, fmt_list, out, row_idx);
if (err < 0 || !is || (lines >= ntimes && ntimes > -1))
break;
}
// convert return value to Cell array
int i = 0;
Array ra_idx (dim_vector (1,2));
// err = -2 means "error, and no columns read this row
int valid_rows = (row == ntimes) ? ntimes : ((err == -2) ? row : row+1);
dim_vector dv (valid_rows, 1);
ra_idx(0) = 0;
if (!collect_output)
{
for (std::list::iterator col = out.begin ();
col != out.end (); col++, i++)
{
// resize col to size row
ra_idx(1) = i;
retval = do_cat_op (retval,
octave_value (Cell (col->resize (dv,0))),
ra_idx);
}
}
else // group adjacent cells of the same type into a single cell
{
octave_value cur; // current cell, accumulating columns
octave_idx_type group_size = 0; // columns in this cell
int prev_type = -1;
conv = 0;
for (std::list::iterator col = out.begin ();
col != out.end (); col++)
{
if (!merge_with_prev [conv++]) // including first time
{
if (prev_type != -1)
{
ra_idx(1) = i++;
retval = do_cat_op (retval, octave_value (Cell(cur)),
ra_idx);
}
cur = octave_value (col->resize (dv,0));
group_size = 1;
prev_type = col->type_id ();
}
else
{
ra_idx(1) = group_size++;
cur = do_cat_op (cur, octave_value (col->resize (dv,0)),
ra_idx);
}
}
ra_idx(1) = i;
retval = do_cat_op (retval, octave_value (Cell (cur)), ra_idx);
}
return retval;
}
/********************/
/**
* Create a delimited stream, reading from is, with delimiters delims,
* and allowing reading of up to tellg() + longest_lookeahead.
* When is is at EOF, lookahead may be padded by ASCII nuls.
*/
dstr::dstr (std::istream& is, const std::string& delimiters,
int longest_lookahead, octave_idx_type bsize)
: bufsize (bsize), i_stream (is), longest (longest_lookahead),
delims (delimiters),
flags (std::ios_base::failbit & ~std::ios_base::failbit) // can't cast 0
{
buf = new char[bufsize];
eob = buf + bufsize;
idx = eob; // refresh_buf shouldn't try to copy old data
progress_marker = idx;
refresh_buf (); // load the first batch of data
}
// Used to create a stream from a strstream from data read from a dstr.
// FIXME: Find a more efficient approach. Perhaps derived dstrstream
dstr::dstr (std::istream& is, const dstr& ds)
: bufsize (ds.bufsize), i_stream (is), longest (ds.longest),
delims (ds.delims),
flags (std::ios_base::failbit & ~std::ios_base::failbit) // can't cast 0
{
buf = new char[bufsize];
eob = buf + bufsize;
idx = eob; // refresh_buf shouldn't try to copy old data
progress_marker = idx;
refresh_buf (); // load the first batch of data
}
dstr::~dstr ()
{
// Seek to the correct position in i_stream.
if (!eof ())
{
i_stream.clear ();
i_stream.seekg (buf_in_file);
i_stream.read (buf, idx - buf);
}
delete [] buf;
}
/** Read a character from the buffer, refilling the buffer from the
* file if necessary.
*/
int
dstr::get_undelim ()
{
int retval;
if (eof ())
{
setstate (std::ios_base::failbit);
return EOF;
}
if (idx < eob)
retval = *idx++;
else
{
refresh_buf ();
if (eof ())
{
setstate (std::ios_base::eofbit);
retval = EOF;
}
else
retval = *idx++;
}
if (idx >= last)
delimited = false;
return retval;
}
/** Return the next character to be read without incrementing the pointer,
* refilling the buffer from the file if necessary.
*/
int
dstr::peek_undelim ()
{
int retval = get_undelim ();
putback ();
return retval;
}
/**
* Copy remaining unprocessed data to the start of the buffer and load
* new data to fill it.
* Returns EOF if the file is at EOF before reading any data and all of the
* data that has been read has been processed.
*/
int
dstr::refresh_buf (void)
{
if (eof ())
return EOF;
int retval;
int old_remaining = eob - idx;
if (old_remaining < 0)
{
idx = eob;
old_remaining = 0;
}
octave_quit (); // allow ctrl-C
if (old_remaining > 0)
memmove (buf, idx, old_remaining);
progress_marker -= idx - buf; // where original idx would have been
idx = buf;
int gcount; // chars read
if (!i_stream.eof ())
{
buf_in_file = i_stream.tellg (); // record for destructor
i_stream.read (buf + old_remaining, bufsize - old_remaining);
gcount = i_stream.gcount ();
}
else
gcount = 0;
eob = buf + old_remaining + gcount;
last = eob;
if (gcount == 0)
{
delimited = false;
if (eob != buf) // no more data in file, but still some to go
retval = 0;
else
retval = EOF; // file and buffer are both done.
}
else
{
delimited = true;
for (last = eob - longest; last - buf >= 0; last--)
{
if (strchr (delims.c_str (), *last))
break;
}
if (last - buf < 0)
delimited = false;
retval = 0;
}
if (retval == EOF) // Ensure fast peek doesn't give valid char
*idx = '\0'; // FIXME - check that no TreatAsEmpty etc starts w. \0?
return retval;
}
/** Return a pointer to a block of data of size size, assuming that a
* sufficiently large buffer is available in buffer, if required.
* If called when delimited==true, and size is no greater than
* longest_lookahead then this will not call refresh_buf(), so seekg()
* still works. Otherwise, seekg may be invalidated.
*/
char *
dstr::read (char *buffer, int size, char* &prior_tell)
{
char *retval;
if (eob - idx > size)
{
retval = idx;
idx += size;
if (idx > last)
delimited = false;
}
else
{
// If there was a tellg pointing to an earlier point than the current
// read position, try to keep it in the active buffer.
// In the current code, prior_tell==idx for each call,
// so this is not necessary, just a precaution.
if (eob - prior_tell + size < bufsize)
{
octave_idx_type gap = idx - prior_tell;
idx = prior_tell;
refresh_buf ();
idx += gap;
}
else // can't keep the tellg in range. May skip some data.
{
refresh_buf ();
}
prior_tell = buf;
if (eob - idx > size)
{
retval = idx;
idx += size;
if (idx > last)
delimited = false;
}
else
{
if (size <= bufsize) // small read, but reached EOF
{
retval = idx;
bzero (eob, size - (eob - idx));
idx += size;
}
else // Reading more than the whole buf; return it in buffer
{
retval = buffer;
// FIXME -- read bufsize at a time
int i;
for (i = 0; i < size && !eof (); i++)
*buffer++ = get_undelim ();
if (eof ())
bzero (buffer, size - i);
}
}
}
return retval;
}
/**
* Return in out an entire line, terminated by delim.
* On input, out must have length at least 1.
*/
int
dstr::getline (std::string& out, char delim)
{
int len = out.length (), used = 0;
int ch;
while ((ch = get_undelim ()) != delim && ch != EOF)
{
out[used++] = ch;
if (used == len)
{
len <<= 1;
out.resize (len);
}
}
out.resize (used);
field_done ();
return ch;
}
/*
* Skip an entire line, terminated by delim.
int
dstr::skipline (char delim)
{
int ch;
while ((ch = get_undelim ()) != delim && ch != EOF)
;
field_done ();
return ch;
}
*/
/*
%!test
%! str = "1, 2, 3, 4\n 5, , , 8\n 9, 10, 11, 12";
%! fmtstr = "%f %d %f %s";
%! c = ytextscan (str, fmtstr, 2, "delimiter", ",", "emptyvalue", -Inf);
%! assert (isequal (c{1}, [1;5]));
%! assert (length (c{1}), 2);
%! assert (iscellstr (c{4}));
%! assert (isequal (c{3}, [3; -Inf]));
%!test
%! b = [10:10:100];
%! b = [b; 8*b/5];
%! str = sprintf ("%g miles/hr = %g kilometers/hr\n", b);
%! fmt = "%f miles/hr = %f kilometers/hr";
%! c = ytextscan (str, fmt);
%! assert (c{1}, b(1,:)', 1e-5);
%! assert (c{2}, b(2,:)', 1e-5);
%!test
%! str = "13, -, NA, str1, -25\r\n// Middle line\r\n36, na, 05, str3, 6";
%! a = ytextscan (str, "%d %n %f %s %n", "delimiter", ",",
%! "treatAsEmpty", {"NA", "na", "-"},"commentStyle", "//");
%! assert (a{1}, int32 ([13; 36]));
%! assert (a{2}, [NaN; NaN]);
%! assert (a{3}, [NaN; 5]);
%! assert (a{4}, {"str1"; "str3"});
%! assert (a{5}, [-25; 6]);
%!test
%! str = "Km:10 = hhhBjjj miles16hour\r\n";
%! str = [str "Km:15 = hhhJjjj miles241hour\r\n"];
%! str = [str "Km:2 = hhhRjjj miles3hour\r\n"];
%! str = [str "Km:25 = hhhZ\r\n"];
%! fmt = "Km:%d = hhh%1sjjj miles%dhour";
%! a = ytextscan (str, fmt, "delimiter", " ");
%! assert (a{1}', int32 ([10 15 2 25]));
%! assert (a{2}', {'B' 'J' 'R' 'Z'});
%! assert (a{3}', int32 ([16 241 3 0]));
## Test with default endofline parameter
%!test
%! c = ytextscan ("L1\nL2", "%s");
%! assert (c{:}, {"L1"; "L2"});
## Test with endofline parameter set to "" (empty) - newline should be in word
%!test
%! c = ytextscan ("L1\nL2", "%s", "endofline", "");
%! assert (int8 ([c{:}{:}]), int8 ([ 76, 49, 10, 76, 50 ]));
### Matlab fails this test. A literal after a conversion is not a delimiter
#%!test
#%! ## No delimiters at all besides EOL. Skip fields, even empty fields
#%! str = "Text1Text2Text\nTextText4Text\nText57Text";
#%! c = ytextscan (str, "Text%*dText%dText");
#%! assert (c{1}, int32 ([2; 4; 0]));
## CollectOutput test
%!test
%! b = [10:10:100];
%! b = [b; 8*b/5; 8*b*1000/5];
%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
%! c = ytextscan (str, fmt, "collectoutput", 1);
%! assert (size (c{3}), [10, 2]);
%! assert (size (c{2}), [10, 2]);
## CollectOutput test with uneven column length files
%!test
%! b = [10:10:100];
%! b = [b; 8*b/5; 8*b*1000/5];
%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
%! str = [str "110 miles/hr"];
%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
%! c = ytextscan (str, fmt, "collectoutput", 1);
%! assert (size (c{1}), [11, 1]);
%! assert (size (c{3}), [11, 2]);
%! assert (size (c{2}), [11, 2]);
%! assert (c{3}(end), NaN);
%! assert (c{2}{11, 1}, "/hr");
%! assert (isempty (c{2}{11, 2}), true);
## Double quoted string
%!test
%! str = 'First "the second called ""the middle""" third';
%! fmt = "%q";
%! c = ytextscan (str, fmt);
%! assert (c{1}, {"First"; 'the second called "the middle"'; "third"});
## Arbitrary character
%!test
%! c = ytextscan ("a first, \n second, third", "%s %c %11c", 'delimiter', ' ,');
%! assert (c{1}, {"a"; "ond"});
%! assert (c{2}, {"f"; "t"});
%! assert (c{3}, {"irst, \n sec"; "hird"});
## Field width and non-standard delimiters
%!test
%! str = "12;34;123456789;7";
%! c = ytextscan (str, "%4d %4d", "delimiter", ";", "collectOutput", 1);
%! assert (c, {[12 34; 1234 5678; 9 7]});
## Field width and non-standard delimiters
%!test
%! str = "12;34;123456789;7";
%! c = ytextscan (str, "%4f %f", "delimiter", ";", "collectOutput", 1);
%! assert (c, {[12 34; 1234 56789; 7 NaN]});
## Ignore trailing delimiter, but use leading one
%!test
%! str = "12.234e+2,34, \n12345.789-9876j,78\n,10|3";
%! c = ytextscan (str, "%10.2f %f", "delimiter", ",", "collectOutput", 1,
%! "expChars", "e|");
%! assert (c, {[1223 34; 12345.79-9876j 78; NaN 10000]}, 1e-6);
%!test
%! ## Multi-character delimiter
%! str = "99end2 space88gap 4564";
%! c = ytextscan (str, "%d %s", "delimiter", {"end", "gap", "space"});
%! assert (c{1}, int32([99; 88]));
%! assert (c{2}, {"2 "; "4564"});
### Delimiters as part of literals, and following literals
#%!test
#%! str = "12 R&D & 7";
#%! c = ytextscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1, "EmptyValue", -99);
#%! assert (c, {[12 -99; 7 -99]});
#
### Delimiters as part of literals, and before literals
#%!test
#%! str = "12 & R&D 7";
#%! c = ytextscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1);
#%! assert (c, {[12 7]});
%!test
%! ## Check number of lines read, not number of passes through format string
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid, "1\n2\n3\n4\n5\n6");
%! fseek (fid, 0, "bof");
%! c = ytextscan (fid, "%f %f", 2);
%! E = feof (fid);
%! fclose (fid);
%! unlink (f);
%! assert (c, {1, 2});
%! assert (!E);
%!test
%! ## Check number of lines read, not number of passes through format string
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid, "1\r\n2\r3\n4\r\n5\n6");
%! fseek (fid, 0, "bof");
%! c = ytextscan (fid, "%f %f", 4);
%! fclose (fid);
%! unlink (f);
%! assert (c, {[1;3], [2;4]})
%!test
%! ## Check number of lines read, with multiple delimiters
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid, "1-\r\n-2\r3-\n-4\r\n5\n6");
%! fseek (fid, 0, "bof");
%! c = ytextscan (fid, "%f %f", 4, "delimiter", "-", "multipleDelimsAsOne", 1);
%! fclose (fid);
%! unlink (f);
%! assert (c, {[1;3], [2;4]})
## Test input validation
%!error ytextscan ()
%!error ytextscan (single (40))
%!error ytextscan ({40})
%!error ytextscan ("Hello World", 2)
#%!error [C, pos] = ytextscan ("Hello World")
%!error ytextscan ("Hello World", '%s', 'EndOfLine', 3)
%!error <'%z' is not a valid format specifier> ytextscan ("1.0", "%z");
%!error ytextscan ("1.0", "foo");
## Test incomplete first data line
%! R = ytextscan (['Empty1' char(10)], 'Empty%d %f');
%! assert (R{1}, int32 (1));
%! assert (isempty (R{2}), true);
## bug #37023
%!test
%! data = ytextscan(" 1. 1 \n 2 3\n", '%f %f');
%! assert (data{1}, [1; 2], 1e-15);
%! assert (data{2}, [1; 3], 1e-15);
## Whitespace test (bug #37333) using delimiter ";"
%!test
%! tc = [];
%! tc{1, 1} = "C:/code;";
%! tc{1, end+1} = "C:/code/meas;";
%! tc{1, end+1} = " C:/code/sim;";
%! tc{1, end+1} = "C:/code/utils;";
%! string = [tc{:}];
%! c = ytextscan (string, "%s", "delimiter", ";");
%! for k = 1:max (numel (c{1}), numel (tc))
%! lh = c{1}{k};
%! rh = tc{k};
%! rh(rh == ";") = "";
%! rh = strtrim (rh);
%! assert (strcmp (lh, rh));
%! end
## Whitespace test (bug #37333), adding multipleDelimsAsOne true arg
%!test
%! tc = [];
%! tc{1, 1} = "C:/code;";
%! tc{1, end+1} = " C:/code/meas;";
%! tc{1, end+1} = "C:/code/sim;;";
%! tc{1, end+1} = "C:/code/utils;";
%! string = [tc{:}];
%! c = ytextscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 1);
%! for k = 1:max (numel (c{1}), numel (tc))
%! lh = c{1}{k};
%! rh = tc{k};
%! rh(rh == ";") = "";
%! rh = strtrim (rh);
%! assert (strcmp (lh, rh));
%! end
## Whitespace test (bug #37333), adding multipleDelimsAsOne false arg
%!test
%! tc = [];
%! tc{1, 1} = "C:/code;";
%! tc{1, end+1} = " C:/code/meas;";
%! tc{1, end+1} = "C:/code/sim;;";
%! tc{1, end+1} = "";
%! tc{1, end+1} = "C:/code/utils;";
%! string = [tc{:}];
%! c = ytextscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 0);
%! for k = 1:max (numel (c{1}), numel (tc))
%! lh = c{1}{k};
%! rh = tc{k};
%! rh(rh == ";") = "";
%! rh = strtrim (rh);
%! assert (strcmp (lh, rh));
%! end
## Whitespace test (bug #37333) whitespace "" arg
%!test
%! tc = [];
%! tc{1, 1} = "C:/code;";
%! tc{1, end+1} = " C:/code/meas;";
%! tc{1, end+1} = "C:/code/sim;";
%! tc{1, end+1} = "C:/code/utils;";
%! string = [tc{:}];
%! c = ytextscan (string, "%s", "delimiter", ";", "whitespace", "");
%! for k = 1:max (numel (c{1}), numel (tc))
%! lh = c{1}{k};
%! rh = tc{k};
%! rh(rh == ";") = "";
%! assert (strcmp (lh, rh));
%! end
## Whitespace test (bug #37333), whitespace " " arg
%!test
%! tc = [];
%! tc{1, 1} = "C:/code;";
%! tc{1, end+1} = " C:/code/meas;";
%! tc{1, end+1} = "C:/code/sim;";
%! tc{1, end+1} = "C:/code/utils;";
%! string = [tc{:}];
%! c = ytextscan (string, "%s", "delimiter", ";", "whitespace", " ");
%! for k = 1:max (numel (c{1}), numel (tc))
%! lh = c{1}{k};
%! rh = tc{k};
%! rh(rh == ";") = "";
%! rh = strtrim (rh);
%! assert (strcmp (lh, rh));
%! end
## Tests reading with empty format, should return proper nr of columns
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid, " 1 2 3 4\n5 6 7 8");
%! fseek (fid, 0, "bof");
%! A = ytextscan (fid, "");
%! E = feof (fid);
%! fclose (fid);
%! unlink (f);
%! assert (A{1}, [1 ; 5], 1e-6);
%! assert (A{2}, [2 ; 6], 1e-6);
%! assert (A{3}, [3 ; 7], 1e-6);
%! assert (A{4}, [4 ; 8], 1e-6);
%! assert (E);
## Tests reading with empty format; empty fields & incomplete lower row
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid, " ,2,,4\n5,6");
%! fseek (fid, 0, "bof");
%! A = ytextscan (fid, "", "delimiter", ",", "EmptyValue", 999, "CollectOutput" , 1);
%! fclose (fid);
%! unlink (f);
%! assert (A{1}, [999, 2, 999, 4; 5, 6, 999, 999], 1e-6);
## Error message tests
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! msg1 = "ytextscan: 1 parameters given, but only 0 values";
%! try
%! A = ytextscan (fid, "", "headerlines");
%! end_try_catch;
%! assert (!feof (fid));
%! fclose (fid);
%! unlink (f);
%! assert (msg1, lasterr);
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! msg1 = "ytextscan: HeaderLines must be numeric";
%! try
%! A = ytextscan (fid, "", "headerlines", "hh");
%! end_try_catch;
%! fclose (fid);
%! unlink (f);
%! assert (msg1, lasterr);
%!test
%! ## Skip headerlines
%! A = ytextscan ("field 1 field2\n 1 2\n3 4", "", "headerlines", 1, "collectOutput", 1);
%! assert (A, {[1 2; 3 4]});
%!test
%! ## Skip headerlines with non-default EOL
%! A = ytextscan ("field 1 field2\r 1 2\r3 4", "", "headerlines", 2, "collectOutput", 1, "EndOfLine", '\r');
%! assert (A, {[3 4]});
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid,"some_string");
%! fseek (fid, 0, "bof");
%! msg1 = "ytextscan: EndOfLine must be at most one character or '\\r\\n'";
%! try
%! A = ytextscan (fid, "%f", "EndOfLine", "\n\r");
%! end_try_catch;
%! fclose (fid);
%! unlink (f);
%! assert (msg1, lasterr);
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid,"some_string");
%! fseek (fid, 0, "bof");
%! msg1 = "ytextscan: EndOfLine must be at most one character or '\\r\\n'";
%! try
%! A = ytextscan (fid, "%f", "EndOfLine", 33);
%! end_try_catch;
%! fclose (fid);
%! unlink (f);
%! assert (msg1, lasterr);
## Bug #41824
%!test
%! assert (ytextscan ("123", "", "whitespace", " "){:}, 123);
## Bug #42343-1, just test supplied emptyvalue
%!test
%! assert (ytextscan (",NaN", "", "delimiter", "," ,"emptyValue" ,Inf), {Inf, NaN});
## Bug #42343-2, test padding with supplied emptyvalue
%!test
%! a = ytextscan (",1,,4\nInf, ,NaN", "", "delimiter", ",", "emptyvalue", -10);
%! assert (cell2mat (a), [-10, 1, -10, 4; Inf, -10, NaN, -10]);
## Bug #42528
%!test
%! assert (ytextscan ("1i", ""){1}, 0+1i);
%! assert (cell2mat (ytextscan ("3, 2-4i, NaN\n -i, 1, 23.4+2.2i\n 1+1 1+1j", "", "delimiter", ",")), [3+0i, 2-4i, NaN+0i; 0-i, 1+0i, 23.4+2.2i; 1 1 1+1i]);
%!test
%! ## TreatAsEmpty
%! C = ytextscan ("1,2,3,NN,5,6\n", "%d%d%d%f", "delimiter", ",", "TreatAsEmpty", "NN");
%! assert (C{3}(1), int32 (3));
%! assert (C{4}(1), NaN);
## MultipleDelimsAsOne
%!test
%! str = "11, 12, 13,, 15\n21,, 23, 24, 25\n,, 33, 34, 35";
%! C = ytextscan (str, "%f %f %f %f", "delimiter", ",", "multipledelimsasone", 1, "endofline", "\n");
%! assert (C{1}', [11, 21, 33]);
%! assert (C{2}', [12, 23, 34]);
%! assert (C{3}', [13, 24, 35]);
%! assert (C{4}', [15, 25, NaN]);
## Bug #44750
%!test
%! assert (ytextscan ("/home/foo/", "%s", "delimiter", "/", "MultipleDelimsAsOne", 1){1}, ...
%! {"home"; "foo"});
### Allow cuddling %sliteral but warn it is ambiguous
#%!test
#%! C = ytextscan ("abcxyz51\nxyz83\n##xyz101", "%s xyz %d");
#%! assert (C{1}([1 3]), {"abc"; "##"});
#%! assert (isempty (C{1}{2}), true);
#%! assert (C{2}, int32([51; 83; 101]));
### Literals are not delimiters.
## Test for false positives in check for non-supported format specifiers
%!test
%! assert (ytextscan ("Total: 32.5 % (of cm values)", "Total: %f %% (of cm values)"){1}, 32.5, 1e-5);
## Test various forms of string format specifiers (bug #45712)
%!test
%! str = "14 :1 z:2 z:3 z:5 z:11";
%! C = ytextscan (str, "%f %s %*s %3s %*3s %f", "delimiter", ":");
%! assert (C, {14, {"1 z"}, {"3 z"}, 11});
%% Bit width, fixed width conv. specifiers
%!test
%! str2 = "123456789012345 ";
%! str2 = [str2 str2 str2 str2 str2 str2 str2 str2];
%! str2 = [str2 "123456789.01234 1234567890.1234 12345.678901234 12345.678901234"];
%! pttrn = "%3u8%*s %5u16%*s %10u32%*s %15u64 %3d8%*s %5d16%*s %10d32%*s %15d64 %9f32%*s %14f64%*s %10.2f32%*s %12.2f64%*s";
%! C = ytextscan (str2, pttrn, "delimiter", " ");
%! assert (C{1}, uint8 (123));
%! assert (C{2}, uint16 (12345));
%! assert (C{3}, uint32 (1234567890));
%! assert (C{4}, uint64 (123456789012345));
%! assert (C{5}, int8 (123));
%! assert (C{6}, int16 (12345));
%! assert (C{7}, int32 (1234567890));
%! assert (C{8}, int64 (123456789012345));
%! assert (C{9}, single (123456789), 1e-12);
%! assert (C{10}, double (1234567890.123), 1e-15);
%! assert (C{11}, single (12345.68), 1e-5);
%! assert (C{12}, double (12345.68), 1e-11);
%% Bit width, fixed width conv. specifiers -- check the right amount is left
%!test
%! str2 = "123456789012345 ";
%! str2 = [str2 str2 "123456789.01234"];
%! pttrn = "%3u8 %5u16 %10u32 %3d8 %5d16 %10d32 %9f32 %9f";
%! C = ytextscan (str2, pttrn, "delimiter", " ");
%! assert (C{1}, uint8 (123));
%! assert (C{2}, uint16 (45678));
%! assert (C{3}, uint32 (9012345));
%! assert (C{4}, int8 (123));
%! assert (C{5}, int16 (45678));
%! assert (C{6}, int32 (9012345));
%! assert (C{7}, single (123456789), 1e-12);
%! assert (C{8}, double (0.01234), 1e-12);
%!test
%! C = ytextscan ("123.123", "%2f %3f %3f");
%! assert (C{1}, 12);
%! assert (C{2}, 3.1, 1e-11);
%! assert (C{3}, 23);
%!test
%! C = ytextscan ("123.123", "%3f %3f %3f");
%! assert (C{1}, 123);
%! assert (C{2}, 0.12, 1e-11);
%! assert (C{3}, 3);
%!test
%! C = ytextscan ("123.123", "%4f %3f");
%! assert (C{1}, 123);
%! assert (C{2}, 123);
%% field width interrupts exponent. (Matlab incorrectly gives [12, 2e12])
%!test
%! assert (ytextscan ("12e12", "%4f"), {[120; 2]});
%! assert (ytextscan ("12e+12", "%5f"), {[120; 2]});
%! assert (ytextscan ("125e-12","%6f"), {[12.5; 2]});
%% %[] tests
%% Plain [..] and *[..]
%!test
%! ar = "abcdefguvwxAny\nacegxyzTrailing\nJunk";
%! C = ytextscan (ar, "%[abcdefg] %*[uvwxyz] %s");
%! assert (C{1}, {"abcdefg"; "aceg"; ""});
%! assert (C{2}, {"Any"; "Trailing"; "Junk"});
%!test
%! assert (ytextscan ("A2 B2 C3", "%*[ABC]%d", 3), {int32([2; 2; 3])});
%% [^..] and *[^..]
%!test
%! br = "abcdefguvwx1Any\nacegxyz2Trailing\n3Junk";
%! C = ytextscan (br, "%[abcdefg] %*[^0123456789] %s");
%! assert (C{1}, {"abcdefg"; "aceg"; ""});
%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"});
%% [..] and [^..] containing delimiters
%!test
%! cr = "ab cd efguv wx1Any\na ce gx yz2Trailing\n 3Junk";
%! C = ytextscan (cr, "%[ abcdefg] %*[^0123456789] %s", "delimiter", " \n", "whitespace", "");
%! assert (C{1}, {"ab cd efg"; "a ce g"; " "});
%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"});
%% Bug #36464
%!test
%! assert (ytextscan ('1 2 3 4 5 6', "%*n%n%*[^\n]"){1}, 2);
%% test %[]] and %[^]]
%!test
%! assert (ytextscan ('345]', "%*[123456]%[]]"){1}{1}, "]");
%! assert (ytextscan ('345]', "%*[^]]%s"){1}{1}, "]");
%% Test that "-i" checks the next two characters
%!test
%! C = ytextscan ("-i -in -inf -infinity", "%f %f%s %f %f %s");
%! assert (C, {-i, -i, {"n"}, -Inf, -Inf, {"inity"}});
%% Again for "+i", this time with custom parser
%!test
%! C = ytextscan ("+i +in +inf +infinity", "%f %f%s %f %f %s", "ExpChars", "eE");
%! assert (C, {i, i, {"n"}, Inf, Inf, {"inity"}});
%% Check single quoted format interprets control sequences
%!test
%! C = ytextscan ("1 2\t3 4", '%f %[^\t] %f %f');
%! assert (C, {1, {"2"}, 3, 4});
%% Check overflow and underflow of integer types
%!test
%! a = "-1e90 ";
%! b = "1e90 ";
%! fmt = "%d8 %d16 %d32 %d64 %u8 %u16 %u32 %u64 ";
%! C = ytextscan ([a a a a a a a a b b b b b b b b], fmt);
%! assert (C{1}, int8 ([-128; 127]));
%! assert (C{2}, int16([-32768; 32767]));
%! assert (C{3}, int32([-2147483648; 2147483647]));
%! assert (C{4}, int64([-9223372036854775808; 9223372036854775807]));
%! assert (C{5}, uint8 ([0; 255]));
%! assert (C{6}, uint16([0; 65535]));
%! assert (C{7}, uint32([0; 4294967295]));
%! assert (C{8}, uint64([0; 18446744073709551615]));
%% Tests from Matlab (does The MathWorks have any copyright over the input?)
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid,"09/12/2005 Level1 12.34 45 1.23e10 inf Nan Yes 5.1+3i\n");
%! fprintf (fid,"10/12/2005 Level2 23.54 60 9e19 -inf 0.001 No 2.2-.5i\n");
%! fprintf (fid,"11/12/2005 Level3 34.90 12 2e5 10 100 No 3.1+.1i\n");
%! fseek (fid, 0, "bof");
%! C = ytextscan(fid,"%s %s %f32 %d8 %u %f %f %s %f");
%! %assert (C{1}, {"09/12/2005";"10/12/2005";"11/12/2005"});
%! assert (C{2}, {"Level1";"Level2";"Level3"});
%! assert (C{3}, [single(12.34);single(23.54);single(34.90)]);
%! assert (C{4}, [int8(45);int8(60);int8(12)]);
%! assert (C{5}, [uint32(4294967295);uint32(4294967295);uint32(200000)]);
%! assert (C{6}, [inf;-inf;10]);
%! assert (C{7}, [NaN;0.001;100], eps);
%! assert (C{8}, {"Yes";"No";"No"});
%! assert (C{9}, [5.1+3i;2.2-0.5i;3.1+0.1i]);
%! fseek (fid, 0, "bof");
%! C = ytextscan(fid,"%s Level%d %f32 %d8 %u %f %f %s %f");
%! assert (C{2}, [int32(1);int32(2);int32(3)]);
%! assert (C{3}, [single(12.34);single(23.54);single(34.90)]);
%! fseek (fid, 0, "bof");
%! C = ytextscan(fid,'%s %*[^\n]');
%! fclose (fid);
%! unlink (f);
%! assert (C, {{"09/12/2005";"10/12/2005";"11/12/2005"}});
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid,"1, 2, 3, 4, , 6\n");
%! fprintf (fid,"7, 8, 9, , 11, 12\n");
%! fseek (fid, 0, "bof");
%! C = ytextscan(fid,"%f %f %f %f %u8 %f", "Delimiter",",","EmptyValue",-Inf);
%! fclose (fid);
%! unlink (f);
%! assert (C{4}, [4; -Inf]);
%! assert (C{5}, uint8 ([0; 11]));
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! fprintf (fid,"abc, 2, NA, 3, 4\n");
%! fprintf (fid,"// Comment Here\n");
%! fprintf (fid,"def, na, 5, 6, 7\n");
%! fseek (fid, 0, "bof");
%! C = ytextscan(fid,"%s %n %n %n %n","Delimiter",",","TreatAsEmpty",{"NA","na"},"CommentStyle","//");
%! fclose (fid);
%! unlink (f);
%! assert (C{1}, {"abc";"def"});
%! assert (C{2}, [2; NaN]);
%! assert (C{3}, [NaN; 5]);
%! assert (C{4}, [3; 6]);
%! assert (C{5}, [4; 7]);
%!test
%!## Test start of comment as string
%! c = ytextscan ("1 / 2 // 3", "%n %s %u8", "CommentStyle", {"//"});
%! assert (c, {1, "/", 2});
*/
// These tests have end-comment sequences, so can't just be in a comment
#if 0
%!test
%!## Test unfinished comment
%! c = ytextscan ("1 2 /* half comment", "%n %u8", "CommentStyle", {"/*", "*/"});
%! assert (c, {1, 2});
## Test reading from a real file
%!test
%! f = tempname ();
%! fid = fopen (f, "w+");
%! d = rand (1, 4);
%! fprintf (fid, " %f %f /* comment */ %f %f ", d);
%! fseek (fid, 0, "bof");
%! A = ytextscan (fid, "%f %f", "CommentStyle", {"/*", "*/"});
%! E = feof (fid);
%! fclose (fid);
%! unlink (f);
%! assert (A{1}, [d(1); d(3)], 1e-6);
%! assert (A{2}, [d(2); d(4)], 1e-6);
%! assert (E);
#endif
/*
## Needed tests:
# 1. Check that it fails at an appropriate point
# 2. Checks for file vs string
# 3. Checks for ReturnOnError
# 4. Checks for all types of invalid parameter
#ToDO:
# 1. Fail when the first mismatch occurs. Tests for this
# 2. Treatment of missing final end-of-line -- unequal line lengths.
# 3. Minimise duplication with scanf and printf? Use as basis for textread?
# 4. Implement ReturnOnError. Perhaps:
# omitted = permissive, present = Matlab behaviour.
*/