# HG changeset patch # User John W. Eaton # Date 1589513263 14400 # Thu May 14 23:27:43 2020 -0400 # Branch stable # Node ID cd17a3cced24a1206f84db4d1f4dbccf33da382d # Parent 7096f672f611055881a619c6a2262e33bfd376a6 accept multi-line interactive input (bug #58370) * lex.h, lex.ll (base_lexer::input_buffer::m_pos): Delete. Use offset from beginning of buffer instead. (base_lexer::input_buffer::m_offset): New data member. (base_lexer::input_buffer::copy_chunk): New argument, BY_LINES. Perform line buffering on input if BY_LINES is true. (push_lexer::fill_flex_buffer): Call copy_chunk with BY_LINES = true. * parse.h, oct-parse.yy (base_parser::statement_list): Append new statements to existing m_stmt_list. (base_parser::run (const std::string&, bool)): Also continue parsing input if lexer input buffer is not empty. diff --git a/libinterp/parse-tree/lex.h b/libinterp/parse-tree/lex.h --- a/libinterp/parse-tree/lex.h +++ b/libinterp/parse-tree/lex.h @@ -542,13 +542,13 @@ namespace octave public: input_buffer (void) - : m_buffer (), m_pos (nullptr), m_chars_left (0), m_eof (false) + : m_buffer (), m_offset (0), m_chars_left (0), m_eof (false) { } void fill (const std::string& input, bool eof_arg); // Copy at most max_size characters to buf. - int copy_chunk (char *buf, size_t max_size); + int copy_chunk (char *buf, size_t max_size, bool by_lines = false); bool empty (void) const { return m_chars_left == 0; } @@ -557,7 +557,7 @@ namespace octave private: std::string m_buffer; - const char *m_pos; + size_t m_offset; size_t m_chars_left; bool m_eof; }; diff --git a/libinterp/parse-tree/lex.ll b/libinterp/parse-tree/lex.ll --- a/libinterp/parse-tree/lex.ll +++ b/libinterp/parse-tree/lex.ll @@ -2365,24 +2365,37 @@ namespace octave { m_buffer = input; m_chars_left = m_buffer.length (); - m_pos = m_buffer.c_str (); + m_offset = 0; m_eof = eof_arg; } + // If BY_LINES is true, return chunks to the lexer line by line. int - base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size) + base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size, + bool by_lines) { static const char * const eol = "\n"; - size_t len = max_size > m_chars_left ? m_chars_left : max_size; + size_t len = 0; + if (by_lines) + { + size_t newline_pos = m_buffer.find ('\n', m_offset); + len = (newline_pos != std::string::npos + ? newline_pos - m_offset + 1 + : (max_size > m_chars_left ? m_chars_left : max_size)); + } + else + len = max_size > m_chars_left ? m_chars_left : max_size; + assert (len > 0); - - memcpy (buf, m_pos, len); + memcpy (buf, m_buffer.c_str () + m_offset, len); m_chars_left -= len; - m_pos += len; - - // Make sure input ends with a new line character. + m_offset += len; + + // Make sure the final input returned to the lexer ends with a new + // line character. + if (m_chars_left == 0 && buf[len-1] != '\n') { if (len < max_size) @@ -2394,10 +2407,17 @@ namespace octave else { // There isn't enough room to plug the newline character - // in the buffer so arrange to have it returned on the next - // call to base_lexer::read. - m_pos = eol; + // in BUF so arrange to have it returned on the next call + // to base_lexer::read. + + // At this point we've exhausted the original input + // (m_chars_left is zero) so we can overwrite the initial + // buffer with a single newline character to be returned on + // the next call. + + m_buffer = eol; m_chars_left = 1; + m_offset = 0; } } @@ -3831,7 +3851,7 @@ namespace octave // character to the input. if (! m_input_buf.empty ()) - status = m_input_buf.copy_chunk (buf, max_size); + status = m_input_buf.copy_chunk (buf, max_size, true); else status = YY_NULL; } diff --git a/libinterp/parse-tree/oct-parse.yy b/libinterp/parse-tree/oct-parse.yy --- a/libinterp/parse-tree/oct-parse.yy +++ b/libinterp/parse-tree/oct-parse.yy @@ -385,7 +385,8 @@ input : simple_list '\n' YYUSE ($2); $$ = nullptr; - parser.statement_list (std::shared_ptr ($1)); + std::shared_ptr tmp_lst ($1); + parser.statement_list (tmp_lst); YYACCEPT; } | simple_list END_OF_INPUT @@ -394,7 +395,8 @@ input : simple_list '\n' $$ = nullptr; lexer.m_end_of_input = true; - parser.statement_list (std::shared_ptr ($1)); + std::shared_ptr tmp_lst ($1); + parser.statement_list (tmp_lst); YYACCEPT; } | parse_error @@ -2437,6 +2439,23 @@ namespace octave } void + base_parser::statement_list (std::shared_ptr& lst) + { + if (m_stmt_list) + { + // Append additional code to existing statement list. + + while (! lst->empty ()) + { + m_stmt_list->push_back (lst->front ()); + lst->pop_front (); + } + } + else + m_stmt_list = lst; + } + + void base_parser::end_token_error (token *tok, token::end_tok_type expected) { std::string msg = ("'" + end_token_as_string (expected) @@ -4709,7 +4728,7 @@ namespace octave error ("unexpected exception while parsing %s", file.c_str ()); } } - while (status == YYPUSH_MORE); + while (status == YYPUSH_MORE || ! m_lexer.at_end_of_buffer ()); if (status != 0) parse_error ("%s", m_parse_error_msg.c_str ()); diff --git a/libinterp/parse-tree/parse.h b/libinterp/parse-tree/parse.h --- a/libinterp/parse-tree/parse.h +++ b/libinterp/parse-tree/parse.h @@ -172,10 +172,7 @@ namespace octave return m_classdef_object; } - void statement_list (const std::shared_ptr& lst) - { - m_stmt_list = lst; - } + void statement_list (std::shared_ptr& lst); std::shared_ptr statement_list (void) const {