%top {
/* This file is part of Mailfromd.
   Copyright (C) 2005-2020 Sergey Poznyakoff

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3, or (at your option)
   any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
}

%option nounput

%{
#include <sys/types.h>	
#include <sys/stat.h>
#include "mailfromd.h"
#include <mailutils/yyloc.h>
#include "gram.h"
#include "prog.h"
#include "srvcfg.h"

static mu_linetrack_t trk;

static struct mu_locus_point start_locus; /* Location when the last state switch
					    occurred */
static mu_opool_t string_pool;   /* Opool for constructing string values */
static char *multiline_delimiter; /* End of here-document delimiter */ 
static size_t multiline_delimiter_len; /* Length of multiline_delimiter_len */
static int multiline_unescape;         /* Unescape here-document contents */
static int (*char_to_strip)(char);     /* Strip matching characters of each
					  here-document line */
struct mu_locus_range const *
get_locus(void)
{
	return &yylloc;
}

static int
is_tab(char c)
{
	return c == '\t';
}
 
static int
is_space(char c)
{
	return c == '\t' || c == ' ';
}

static unsigned char
c_unescape(unsigned char inc)
{
	int outc = mu_wordsplit_c_unquote_char(inc);
        return outc ? outc : inc;
}

static void
switch_origin(struct mu_locus_point *pt)
{
	mu_linetrack_origin(trk, pt);
	mu_locus_point_deinit(pt);
}

#define line_begin string_begin
#define line_add string_add
#define line_add_char string_add_char
static void line_finish(void);
static void string(const char *str, size_t len);
static int isemptystr(char *text);
static int builtin_const(const char *s, size_t len);


/* Input context flags */
#define INCTX_MODULE     0x1  /* current input is a MFL module */
#define INCTX_HADINPUT   0x2  /* some statements has already been processed */
#define INCTX_IGNORE_BYE 0x4  /* ignore eventual `bye' statement, because
                                 this is an #included module */

/* Input context stack */
struct inctx {                     /* input context structure */
	struct inctx *parent;      /* parent context */
	struct mu_locus_range locus;/* locus where the context was pushed */
	mu_linetrack_t trk;
	struct input_file_ident id;/* file id structure to prevent recursion */
	FILE *file;                /* saved yyin */ 
	pid_t pp_pid;              /* preprocessor pid, if used */
	int inctx_flags;           /* input context flags */ 
	YY_BUFFER_STATE buf;       /* lex buffer state */
};

struct inctx *inctx_tos;           /* stack of input contexts */

/* Current input context: */
static pid_t pp_pid;               /* preprocessor pid */
static struct input_file_ident input_file_id;
static int inctx_flags;            /* input context flags */

/* If not 0, emit_token keeps a token that yylex must return on the
   next call. See YY_USER_ACTION below. */
static int emit_token;

/* Find on stack an input context that matches the given file id.
   Return pointer to the context. */
struct inctx *
inctx_locate(struct input_file_ident *id)
{
	struct inctx *ctx;

	for (ctx = inctx_tos; ctx; ctx = ctx->parent)
		if (ctx->id.device == id->device
		    && ctx->id.i_node == id->i_node)
			break;
	return ctx;
}
	
/* Push input context */	
static void
inctx_push()
{
	struct inctx *ctx = mu_alloc(sizeof(*ctx));

	mu_locus_range_init(&ctx->locus);
	mu_locus_range_copy(&ctx->locus, &yylloc);
	ctx->trk = trk;
	ctx->id = input_file_id;
	ctx->file = yyin;
	ctx->pp_pid = pp_pid;
	ctx->inctx_flags = inctx_flags;
	ctx->buf = YY_CURRENT_BUFFER;
	ctx->parent = inctx_tos;
	inctx_tos = ctx;
}

/* Pop input context from the top of the stack into the current
   input context. Return 1 if there are no more contexts left. */
int
inctx_pop()
{
	struct inctx *ctx = inctx_tos;
	
	if (!ctx)
		return 1;

	inctx_tos = ctx->parent;
	
	mu_locus_range_deinit(&ctx->locus);
	trk = ctx->trk;
	yyin = ctx->file; 
	input_file_id = ctx->id;
	inctx_flags = ctx->inctx_flags;
	yy_delete_buffer(YY_CURRENT_BUFFER);
	yy_switch_to_buffer(ctx->buf);
	pp_pid = ctx->pp_pid;
	free(ctx);
	return 0;
}

/* Setup the scanner for input from the file NAME. Return 0 on success,
   and an appropriate EX_ code on error. See lex_new_source below. */
static int
lex_new_source_0(const char *name)
{
	if (!yyin)
		mu_locus_range_init (&yylloc);
	if (ext_pp) {
		yyin = pp_extrn_start(name, &pp_pid);
		if (!yyin) {
			parse_error(_("unable to start external "
				      "preprocessor `%s': %s"),
				     ext_pp,
				     mu_strerror(errno));
			return EX_OSFILE;
		}
	} else {
		yyin = fopen(name, "r");
		if (!yyin) {
			parse_error(_("cannot open %s: %s"), name,
				    mu_strerror(errno));
			return EX_NOINPUT;
		}
	}
	yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));
	MU_ASSERT(mu_linetrack_create(&trk, name, 2));
	mu_locus_point_set_file(&yylloc.beg, name);
	yylloc.beg.mu_line = 1;
	yylloc.beg.mu_col = 1;
	mu_locus_point_copy(&yylloc.end, &yylloc.beg);
	mu_stream_ioctl(mu_strerr, MU_IOCTL_LOGSTREAM,
			MU_IOCTL_LOGSTREAM_SET_LOCUS_RANGE, &yylloc);
	return EX_OK;
}

/* Save the current input context on stack and set up the scanner for
   input from the file NAME. Return 0 on success,
   and an appropriate EX_ code on error. */
int
lex_new_source(const char *name, int flag)
{
	int rc;
	struct stat st;
	struct input_file_ident id;
	struct inctx *pctx;
	
	if (stat(name, &st)) {
		parse_error(_("cannot open `%s': %s"),
			    name, mu_strerror(errno));
		return EX_NOINPUT;
	} else if (!S_ISREG(st.st_mode)) {
		parse_error(_("`%s' is not a regular file"), name);
		return EX_NOINPUT;
	}
	
	id.device = st.st_dev;
	id.i_node = st.st_ino;
	
	pctx = inctx_locate(&id);
	if (pctx) {
		parse_error(_("recursive inclusion"));
		if (pctx->parent)
			parse_error_locus(&pctx->parent->locus,
					  _("`%s' already included here"),
					  name);
		else
			parse_error(_("`%s' already included at top level"),
				    name);
		return 1;
	}
	
	if (flag == LEX_ONCE && source_lookup(&id))
		return 0;

	if (yyin)
		inctx_push();
	rc = lex_new_source_0(name);
	if (rc)
		inctx_pop();
	else {
		input_file_id = id;
		if (flag == LEX_MODULE) {
			inctx_flags = INCTX_MODULE;
			emit_token = T_MODBEG;
		} else
			inctx_flags = 0;
	}
	return rc;
}


/* Return constant or variable token corresponding to the current
   value of yylval.literal->text. */
static int
variable_or_const()
{
	struct variable *vptr;
	const struct constant *cptr;

	cptr = constant_lookup(yylval.literal->text);
	if (cptr) {
		const struct value *value_ptr = &cptr->value;
		switch (value_ptr->type) {
		case dtype_number:
			yylval.number = value_ptr->v.number;
			return T_NUMBER;

		case dtype_string:
			yylval.literal = value_ptr->v.literal;
			return T_STRING;
			
		default:
			abort();
		}
	}

	vptr = variable_lookup(yylval.literal->text);
	if (!vptr) {
		parse_error(_("variable %s is not defined"),
			    yylval.literal->text);
		return T_BOGUS;
	}
	add_xref(vptr, &yylloc);
	yylval.var = vptr;
	return T_VARIABLE;
}
	
/* Saved lexer state before entering COMMENT */
static int save_state;
/* Same as BEGIN, but also saves the current locus. It is then used
   to report unclosed constructs at the end of file. */
#define BEGIN_X(s)							\
	do {								\
		BEGIN(s);						\
		mu_locus_point_copy (&start_locus, &yylloc.beg);	\
	} while(0)

/* If emit_token is not 0, push back current input and return the value
   of emit_token. Clear emit_token before returning. */
#define YY_USER_ACTION							\
	if (emit_token) {						\
		int tok = emit_token;					\
		emit_token = 0;						\
		if (yy_flex_debug)					\
			fprintf(stderr, "--emitting %d (module %s)\n",	\
				tok,top_module->name);			\
		mu_linetrack_retreat(trk, yyleng);			\
		yyless(0);						\
		return tok;						\
  	} else {							\
	        mu_linetrack_advance(trk, &yylloc, yytext, yyleng);	\
	        mu_stream_ioctl(mu_strerr, MU_IOCTL_LOGSTREAM,		\
			        MU_IOCTL_LOGSTREAM_SET_LOCUS_RANGE, &yylloc);\
   	}

/* Read next input chunk. */
#define YY_INPUT(buf,result,max_size)					\
	if (yyin == NULL)						\
		result = YY_NULL;					\
	else if (((result = fread(buf, 1, max_size, yyin)) == 0)	\
		 && ferror(yyin))					\
		YY_FATAL_ERROR("input in flex scanner failed");

/* Redeclare main entry point. Actual yylex is defined in the code section
   below. */
#define YY_DECL static int lexscan(void)


/* String composer */
static struct mu_locus_point string_beg;
static NODE *string_head, *string_tail;

static void
compose_add_node(NODE *node)
{
	if (string_tail)
		string_tail->next = node;
	else
		string_head = node;
	string_tail = node;
}

static void
compose_start(int state)
{
	if (string_head) {
		parse_error("INTERNAL ERROR: previous composition has not "
			    "finished when a new one started");
		abort();
	}
	mu_locus_point_copy(&string_beg, &yylloc.beg);
	BEGIN_X(state);
}

static int
compose_finish()
{
	if (string_tail != string_head) {
		while (string_head->next) {
			NODE *cat = alloc_node(node_type_concat,
					       &string_head->locus);
			NODE *next = string_head->next;
			cat->next = next->next;
			cat->v.concat.arg[0] = cast_to(dtype_string,
						       string_head);
			cat->v.concat.arg[1] = cast_to(dtype_string, next);
			string_head->next = next->next = NULL;
			string_head = cat;
		}
	}
	yylval.node = string_head;
	mu_locus_point_copy(&yylloc.beg, &string_beg);
	mu_locus_point_deinit(&string_beg);
	string_head = string_tail = NULL;
	return T_COMPOSE;
}

static void
compose_add_literal(struct literal *lit)
{
	NODE *node = alloc_node(node_type_string, &yylloc);
	node->v.literal = lit;
	compose_add_node(node);
	if (yy_flex_debug)
		fprintf(stderr, "--add literal: '%s'\n", lit->text);
}

static void
compose_add_string(const char *text, size_t length)
{
	compose_add_literal(string_alloc(text, length));
}

static void
compose_add_number(long num)
{
	NODE *node = alloc_node(node_type_number, &yylloc);
	node->v.number = num;
	compose_add_node(node);
}

void
compose_add_builtin_const(const char *s, size_t len)
{
	const char *sval;
	long nval;
	
	switch (builtin_const_value(s, len, &sval, &nval)) {
	case dtype_number:
		compose_add_number(nval);
		break;

	case dtype_string:
		compose_add_string(sval, strlen(sval));
		break;

	default:
		abort();
	}
}

void
compose_add_variable_or_const(int what)
{
	switch (what) {
	case T_NUMBER:
		compose_add_number(yylval.number);
		break;

	case T_STRING:
		compose_add_literal(yylval.literal);
		break;

	case T_VARIABLE:
		compose_add_node(create_node_variable(yylval.var, &yylloc));
	}
}

%}

/* Exclusive states:

   SHELLMAGIC     Initial shell-magic boilerplate (#!... !#)
   COMMENT        Within a C-style comment;
   XIDENT         Expected identifier;
   STR            Processing a complex string;
   ML             Within a multi-line aggregator.  The line being built
                  requires stripping leading whitespace (if requested).
   CML            Continuation within a multi-line aggregator.  The line
                  being built does not require stripping leading whitespace.
   QML            Quoted multi-line aggregator.  No variable substitution and
                  unquoting is needed.

   Inclusive states:
   
   ONBLOCK	  Lexical tie-in after an `on' keyword.  In ONBLOCK state
                  the strigns `as', `host', `from', and `poll' are
		  recognized as keywords.  The string `for' also acquires
		  special meaning.
*/

%x COMMENT STR ML CML QML XIDENT SHELLMAGIC
%s ONBLOCK

N [0-9][0-9]*
P [1-9][0-9]*
X [0-9a-fA-F]
O [0-7]
WS [ \t][ \t]*
IDENT [a-zA-Z_][a-zA-Z_0-9]*
LOCUS __file__|__line__|__function__|__module__
VCONST __package__|__version__|__major__|__minor__|__patch__|__git__
STATEDIR __(def)?statedir__
PREPROC  __(def)?preproc__	
ICONST {LOCUS}|{VCONST}|{STATEDIR}|{PREPROC}
%%
         /* C-style comments */
<INITIAL,XIDENT>"/*"    { save_state = YYSTATE;  BEGIN_X(COMMENT); }
<COMMENT>[^*\n]*        /* eat anything that's not a '*' */
<COMMENT>"*"+[^*/\n]*   /* eat up '*'s not followed by '/'s */
<COMMENT>\n             ;
<COMMENT>"*"+"/"        BEGIN(save_state);
         /* Shell magic sequence */
^#!" "?"/".*\n {
	if (yylloc.beg.mu_line == 1) {
		BEGIN_X(SHELLMAGIC);
	}
      }
<SHELLMAGIC>^"!#"[ \t]*\n  { BEGIN(INITIAL); }
<SHELLMAGIC>.*\n   ;
         /* Configuration directives */
^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n {
	struct mu_locus_point pt;
	if (parse_line_cpp(yytext, &pt) == 0)
		switch_origin(&pt);
}
        /* Normally, everything within a comment should be ignored, so
	   the exclusive condition for the rule below is an error.
	   Unfortunately, GNU m4 in versions up to 1.4.9 outputs line
	   synchronisation directives in comments, which makes any decent
	   compiler lost trace of which input line it is on.  To avoid
	   this, mfd handles #line directives even within a C-comment
	   block.

	   This bug was fixed in m4 version 1.4.10.  I prefer to keep
	   this kludge until I am pretty sure there are no 1.4.9 around.
	*/
<INITIAL,COMMENT>^[ \t]*#[ \t]*line[ \t].*\n {
	struct mu_locus_point pt;
	if (parse_line(yytext, &pt) == 0)
		switch_origin(&pt);
}
^[ \t]*#[ \t]*pragma[ \t].*\n {
	mu_linetrack_retreat(trk, 1);
	yyless(yyleng-1);
	parse_pragma(yytext);
}
^[ \t]*#[ \t]*error[ \t].*\n  {
         yytext[yyleng-1] = 0; /* Kill trailing newline */
         parse_error("%s", yytext);
}
^[ \t]*#[ \t]*include_once[ \t].*\n {
	parse_include(yytext, 1);
}
^[ \t]*#[ \t]*include[ \t].*\n {
	parse_include(yytext, 0);
}
         /* End-of-line comments */
#.*\n   ;
#.*     /* end-of-file comment */;
        /* expected identified (after 'require' or initial 'from') */
<XIDENT>{IDENT} { BEGIN(INITIAL);
	          string(yytext, yyleng);
		  return T_IDENTIFIER; }
<XIDENT>{WS}    ;
<XIDENT>\n      ;
<XIDENT>.       { BEGIN(INITIAL);
	 mu_linetrack_retreat(trk, yyleng);
                  yyless(0); }
         /* Reserved words */
accept    return T_ACCEPT;
reject    return T_REJECT;
tempfail  return T_TEMPFAIL;
continue  return T_CONTINUE;
discard   return T_DISCARD;
add       return T_ADD;
replace   return T_REPLACE;
delete    return T_DELETE;
if        return T_IF;
fi        return T_FI;
else      return T_ELSE;
elif      return T_ELIF;
on        return T_ON;
do        return T_DO;
done      return T_DONE;
matches   return T_MATCHES;
fnmatches return T_FNMATCHES;
mx{WS}matches return T_MXMATCHES;
mx{WS}fnmatches return T_MXFNMATCHES;
when      return T_WHEN;
or        return T_OR;
and       return T_AND;
not       return T_NOT;
next      return T_NEXT;
prog      return T_PROG;
set       return T_SET;
catch     return T_CATCH;
try       return T_TRY;
echo      return T_ECHO;
return    return T_RETURN;
returns   return T_RETURNS;
func      return T_FUNC;
switch    return T_SWITCH;
case      return T_CASE;
default   return T_DEFAULT;
string/[ \t]*\( { yylval.type = dtype_string; return T_TYPECAST; }
number/[ \t]*\( { yylval.type = dtype_number; return T_TYPECAST; }
string    { yylval.type = dtype_string; return T_TYPE; }
number    { yylval.type = dtype_number; return T_TYPE; }
const     return T_CONST;
throw     return T_THROW;
loop      return T_LOOP;
while     return T_WHILE;
for       return T_FOR;
break     return T_BREAK;
pass      return T_PASS;
begin     return T_BEGIN;
end       return T_END;
alias     return T_ALIAS;
vaptr     return T_VAPTR;
precious  return T_PRECIOUS;
require   return T_REQUIRE;
import    return T_IMPORT;
from      return T_FROM;
static    return T_STATIC;
public    return T_PUBLIC;
module    {  if (inctx_tos && inctx_flags == 0) {
                int c;
		int flen;
		const char *fname = strrchr(yylloc.beg.mu_file, '/');

		if (fname)
			fname++;
		else
			fname = yylloc.beg.mu_file;
		flen = strlen(fname);
		
		if (flen > 3 && strcmp(fname + flen - 3, ".mf") == 0)
			flen -= 3;
		parse_warning_locus(&inctx_tos->locus,
				    _("including a module file is unreliable and may cause subtle errors"));
		/* TRANSLATORS: Do not translate `require %*.*s' */
		parse_warning_locus(&inctx_tos->locus,
				    _("use `require %*.*s' instead"),
				    flen, flen, fname);
		inctx_flags |= INCTX_IGNORE_BYE;
		mu_locus_point_copy(&start_locus, &yylloc.beg);
		while ((c = input()) != '.') {
		    if (c == 0) {
  		        mu_diag_at_locus_point(MU_DIAG_ERROR, &start_locus,
					       "%s",
					       _("unexpected end of file"));
		     	return 0;
		    } else {
			char ch = c;
			mu_linetrack_advance(trk, &yylloc, &ch, 1);
		    }
		}
	     } else if (inctx_flags & INCTX_HADINPUT) 
		     parse_error(_("misplaced `module'"));
	    return T_MODULE;
          }
bye       return T_BYE;
dclex     return T_DCLEX;

{ICONST}   { return builtin_const(yytext, yyleng); } 
<ONBLOCK>poll      return T_POLL;
<ONBLOCK>host      return T_HOST;
<ONBLOCK>as        return T_AS;
         /* Variables */
<INITIAL,ONBLOCK>\%({ICONST}) {
	return builtin_const(yytext + 1, yyleng - 1);
}
<ML,CML,STR>\%({ICONST}) {
	compose_add_builtin_const(yytext + 1, yyleng - 1);
}	
<INITIAL,ONBLOCK>\%\{({ICONST})\} {
	return builtin_const(yytext + 2, yyleng - 3);
}
<ML,CML,STR>\%\{({ICONST})\} {
	compose_add_builtin_const(yytext + 2, yyleng - 3);
}
<ML,CML,STR>\%{IDENT}     {
	string(yytext + 1, yyleng - 1);
	compose_add_variable_or_const(variable_or_const());
}
<ML,CML,STR>\%\{{IDENT}\} {
	string(yytext + 2, yyleng - 3);
	compose_add_variable_or_const(variable_or_const());
}
         /* Positional arguments */
<INITIAL,ONBLOCK>\$# {
	return T_ARGCOUNT;
}
<ML,CML,STR>\$# {
	compose_add_node(create_node_argcount(&yylloc));
}
<INITIAL,ONBLOCK>\${P}        {
	yylval.number = strtol(yytext + 1, NULL, 0);
	return T_ARG; }
<ML,CML,STR>\${P}        {
	compose_add_node(create_node_arg(strtol(yytext + 1, NULL, 0), &yylloc));
}
<INITIAL,ONBLOCK>\$/"("        { return T_ARGX; }
         /* Sendmail variables */
<INITIAL,ONBLOCK>\${IDENT} {
	        if (yyleng == 2)
	           string(yytext + 1, 1);
                else {
                   line_begin();
                   line_add("{", 1);
                   line_add(yytext + 1, yyleng - 1);
                   line_add("}", 1);
                   line_finish();
                }
                return T_SYMBOL;
              }
<ML,CML,STR>\${IDENT} {
	        if (yyleng == 2)
	           string(yytext + 1, 1);
                else {
                   line_begin();
                   line_add("{", 1);
                   line_add(yytext + 1, yyleng - 1);
                   line_add("}", 1);
                   line_finish();
                }
		compose_add_node(create_node_symbol(yylval.literal, &yylloc));
              }
<INITIAL,ONBLOCK>\$\{{IDENT}\} {
	string(yytext+1, yyleng - 1); return T_SYMBOL; }
<ML,CML,STR>\$\{{IDENT}\} {
	string(yytext+1, yyleng - 1);
	compose_add_node(create_node_symbol(yylval.literal, &yylloc)); }
         /* Back-references */
<INITIAL,ONBLOCK>\\{P} {
	 yylval.number = strtoul(yytext+1, NULL, 0);
	 return T_BACKREF; }
<ML,CML,STR>\\{P} {
	compose_add_node(create_node_backref(strtoul(yytext+1, NULL, 10),
					     &yylloc)); }
         /* Numeric strings */
0[xX]{X}{X}*  { yylval.number = strtoul(yytext, NULL, 16); return T_NUMBER; };
0{O}{O}*      { yylval.number = strtoul(yytext, NULL, 8); return T_NUMBER; };
0|{P}         { yylval.number = strtoul(yytext, NULL, 10); return T_NUMBER; };
         /* Identifiers */
{IDENT}  {
	const struct constant *cptr;

	cptr = constant_lookup(yytext);
	if (cptr) {
		const struct value *value_ptr = &cptr->value;
		switch (value_ptr->type) {
		case dtype_number:
			yylval.number = value_ptr->v.number;
			return T_NUMBER;

		case dtype_string:
			yylval.literal = value_ptr->v.literal;
			return T_STRING;
			
		default:
			abort();
		}
	}
	
	if (yylval.builtin = builtin_lookup(yytext))
		return T_BUILTIN;
	else if (yylval.function = function_lookup(yytext))
		return T_FUNCTION;
	else {
                struct variable *vptr;
		string(yytext, yyleng);
                vptr = variable_lookup(yylval.literal->text);
	        if (!vptr) {
			return T_IDENTIFIER;
		}
		add_xref(vptr, &yylloc);
		yylval.var = vptr;
		return T_VARIABLE;
	}
}
         /* Strings */
'[^\n']*'             { string(yytext+1, yyleng-2); return T_STRING; }
\"[^\\\"$%\n]*\"      { string(yytext+1, yyleng-2); return T_STRING; }
\"[^\\\"$%\n]*\\\n    { compose_start(STR);
                        compose_add_string(yytext + 1, yyleng - 3); }
\"[^\\\"$%\n]*/[\\$%] { compose_start(STR);
                        compose_add_string(yytext+1, yyleng-1); }
\"\\x{X}{X}/[\\$%] {
         compose_start(STR);
         line_add_char(strtoul(yytext + 3, NULL, 16));
         compose_add_literal(string_finish());
}
\"\\x{X}{X} {
         compose_start(STR);
         line_add_char(strtoul(yytext + 3, NULL, 16));
}

\"\\0{O}{1,3}/[\\$%] {
         compose_start(STR);
         line_add_char(strtoul(yytext + 3, NULL, 8));
         compose_add_literal(string_finish());
}
\"\\0{O}{1,3} {
         compose_start(STR);  
         line_add_char(strtoul(yytext + 3, NULL, 8));
}

\"\\[^1-9]/[\\$%]  {
         compose_start(STR);
         line_add_char(c_unescape(yytext[2]));
         compose_add_literal(string_finish());
}
\"\\[^1-9]  {
         compose_start(STR);
         line_add_char(c_unescape(yytext[2]));
}
<STR>[^\\\"$%\n]*\\\n { line_add(yytext, yyleng - 2); }
<STR>[^\\\"$%\n]*\"   { BEGIN(INITIAL);
                        if (yyleng > 1)
                          line_add(yytext, yyleng - 1);
                        compose_add_literal(string_finish());
                        return compose_finish(); }
<STR>[^\\\"$%\n]+/[\\$%] {
                       line_add(yytext, yyleng);
                       compose_add_literal(string_finish());
                       line_begin();
                 }
<STR,ML,CML>\\x{X}{X}/[\\$%] {
         line_add_char(strtoul(yytext + 2, NULL, 16));
         compose_add_literal(string_finish());       
         line_begin();
}
<STR,ML,CML>\\x{X}{X} {
         line_add_char(strtoul(yytext + 2, NULL, 16));
}
<STR,ML,CML>\\0{O}{1,3}/[\\$%] {
         line_add_char(strtoul(yytext + 2, NULL, 8));
         compose_add_literal(string_finish());
         line_begin();
}
<STR,ML,CML>\\0{O}{1,3} {
         line_add_char(strtoul(yytext + 2, NULL, 8));
}
<STR,ML,CML>\\[^1-9]/[\\$%] {
         line_add_char(c_unescape(yytext[1]));
         compose_add_literal(string_finish());
         line_begin();
}
<STR,ML,CML>\\[^1-9] {
         line_add_char(c_unescape(yytext[1]));
}

         /* Multi-line strings */
"<<"(-" "?)?\\?{IDENT}[ \t]*.*\n |
"<<"(-" "?)?'{IDENT}'[ \t]*.*\n {
        char *p;

        char_to_strip = NULL;
        multiline_unescape = 1;

        line_begin();
        p = yytext + 2;
        if (*p == '-') {
            ++p;
            if (*p == ' ') {
                ++p;
                char_to_strip = is_space;
            } else
                char_to_strip = is_tab;
        }
        
        if (*p == '\\') {
            p++;
            multiline_unescape = 0;
        }
        if (*p == '\'') {
            char *q;
 
            p++;
            multiline_unescape = 0;
            q = strchr(p, '\'');
            multiline_delimiter_len = q - p;
        } else
            multiline_delimiter_len = strcspn(p, " \t");

        multiline_delimiter = mu_alloc(multiline_delimiter_len + 1);
        memcpy(multiline_delimiter, p, multiline_delimiter_len);
        multiline_delimiter[multiline_delimiter_len] = 0;
        if (multiline_unescape)
            compose_start(ML);
        else
            compose_start(QML);
}
        /* Quoted multilines */
<QML>[^\n]*\n {
        char *p;

        p = yytext;
        if (char_to_strip)
               for (; char_to_strip (*p); p++)
                      ;

        if (strlen(p) >= multiline_delimiter_len
            && memcmp(p, multiline_delimiter, multiline_delimiter_len) == 0
            && isemptystr(p + multiline_delimiter_len)) {
	       free (multiline_delimiter);
	       multiline_delimiter = NULL;
               multiline_delimiter_len = 0;
	       BEGIN(INITIAL);
               compose_add_literal(string_finish());
               return compose_finish();
        }
        line_add(p, strlen(p));
}

        /* Unquoted multilines */
<ML>[^\\$%\n]+/[\\$%] {
        char *p = yytext;
        if (char_to_strip)
               for (; char_to_strip (*p); p++)
                      ;
        line_add(p, strlen(p));
        compose_add_literal(string_finish());
        BEGIN_X(CML);
}
<CML>[^\\$%\n]+/[\\$%] {
        line_add(yytext, yyleng);
        compose_add_literal(string_finish());
        line_begin();
}
<CML,STR>"%%"|"$$" {
        line_add(yytext, 1);
        compose_add_literal(string_finish());
        line_begin();
}
<ML,CML,STR>[$%] {
        line_add(yytext, yyleng);
}
<ML>[^\\$%\n]*\n/% {
        char *p;

        p = yytext;
        if (char_to_strip)
               for (; char_to_strip (*p); p++)
                      ;

        if (strlen(p) >= multiline_delimiter_len
            && memcmp(p, multiline_delimiter, multiline_delimiter_len) == 0
            && isemptystr(p + multiline_delimiter_len)) {
	       free (multiline_delimiter);
	       multiline_delimiter = NULL;
               multiline_delimiter_len = 0;
	       BEGIN(INITIAL);
               compose_add_literal(string_finish());
               return compose_finish();
        }
        line_add(p, strlen(p));
        compose_add_literal(string_finish());
}
<ML>[^\\$%\n]*\n {
        char *p;

        p = yytext;
        if (char_to_strip)
               for (; char_to_strip (*p); p++)
                      ;

        if (strlen(p) >= multiline_delimiter_len
            && memcmp(p, multiline_delimiter, multiline_delimiter_len) == 0
            && isemptystr(p + multiline_delimiter_len)) {
	       free (multiline_delimiter);
	       multiline_delimiter = NULL;
               multiline_delimiter_len = 0;
	       BEGIN(INITIAL);
               compose_add_literal(string_finish());
               return compose_finish();
        }
        line_add(p, strlen(p));
}
<CML>[^\\$%\n]*\n {
        if (yyleng >= multiline_delimiter_len
            && memcmp(yytext, multiline_delimiter,
                      multiline_delimiter_len) == 0
            && isemptystr(yytext + multiline_delimiter_len)) {
	       free (multiline_delimiter);
	       multiline_delimiter = NULL;
               multiline_delimiter_len = 0;
	       BEGIN(INITIAL);
               compose_add_literal(string_finish());
               return compose_finish();
        }
        line_add(yytext, yyleng);
        BEGIN_X(ML);
}
         /* Other tokens */
{WS}     ;
\n       ;
"="|"==" return T_EQ;
"!=" return T_NE;
"<"  return T_LT;
"<=" return T_LE;
">"  return T_GT;
">=" return T_GE;
"&"  return T_LOGAND;
"|"  return T_LOGOR;
"^"  return T_LOGXOR;
"~"  return T_LOGNOT;
"<<" return T_SHL;
">>" return T_SHR;
"..." return T_DOTS;
<ML,QML>.   /* If a here-document is not closed and its next line does not
               end with a \n, prevent it from being displayed by ECHO */;
. return yytext[0];
%%

int
yylex()
{
        int rc = lexscan();
	if (rc != T_MODBEG && rc != T_MODEND)
		inctx_flags |= INCTX_HADINPUT;
	return rc;
}

void
init_string_space()
{
        mu_opool_create(&string_pool, MU_OPOOL_ENOMEMABRT);
}

void
free_string_space()
{
	mu_opool_destroy(&string_pool);
}

char *
mf_strdup(const char *str)
{
	string_add(str, strlen(str) + 1);
	return mu_opool_finish(string_pool, NULL);
}

struct literal *
string_alloc(const char *str, size_t len)
{
	string_begin();
	string_add(str, len);
	return string_finish();
}

static void
string(const char *str, size_t len)
{
	yylval.literal = string_alloc(str, len);
}

void
string_begin()
{
	/* nothing */
}

struct literal *
string_finish()
{
	char *ptr;
	struct literal *lit;
	mu_opool_append_char(string_pool, 0);
	ptr = mu_opool_finish(string_pool, NULL);
	lit = literal_lookup(ptr);
	if (lit->text != ptr)
		mu_opool_free(string_pool, ptr);
	return lit;
}

static void
line_finish()
{	
	yylval.literal = string_finish();
	if (yy_flex_debug)
		fprintf(stderr, "constructed line: %s\n",
			yylval.literal->text);
}		

void
string_add(const char *str, size_t len)
{
	mu_opool_append(string_pool, str, len);
}

void
string_add_char(unsigned char c)
{
	mu_opool_append_char(string_pool, c);
}

void
parse_warning(const char *fmt, ...)
{
	va_list ap;
	va_start(ap, fmt);
        mu_diag_voutput (MU_DIAG_WARNING, fmt, ap);
	va_end(ap);
}

void
parse_warning_locus(struct mu_locus_range const *loc, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
        mu_vdiag_at_locus_range(MU_DIAG_WARNING, loc, fmt, ap);
	va_end(ap);
}

void
parse_error(const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
        mu_diag_voutput(MU_DIAG_ERROR, fmt, ap);
	va_end(ap);
	error_count++;
}

void
parse_error_locus(struct mu_locus_range const *loc, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
        mu_vdiag_at_locus_range(MU_DIAG_ERROR, loc, fmt, ap);
	va_end(ap);
	error_count++;
}



static int
lex_close_source()
{
	if (!yyin)
		return 1;
	if (ext_pp)
 	        pp_extrn_shutdown(yyin, pp_pid);
	else
		fclose(yyin);
	yyin = NULL;
        mu_linetrack_destroy(&trk);
	
	switch (YYSTATE) {
	case INITIAL:
		/* ok */
		break;

	case COMMENT:
                mu_diag_at_locus_point (MU_DIAG_ERROR, &start_locus,
                                        "%s", _("end of file in comment"));
		break;

	case STR:
	case ML:
	case CML:
	case QML:
                mu_diag_at_locus_point (MU_DIAG_ERROR, &start_locus,
		                        "%s", _("end of file in string"));
                break;

	case SHELLMAGIC:
                mu_diag_at_locus_point (MU_DIAG_ERROR, &start_locus,
				        "%s", _("end of file in shell block"));
		break;
	}
	return inctx_pop();
}

void
lex_drain_input()
{
	while (input() != EOF)
		;
}

void
lex_bye()
{
	lex_drain_input();
	lex_close_source();
}

int
yywrap()
{
	if (yy_flex_debug && top_module)
		fprintf(stderr, "--eof in module %s\n", top_module->name);
	if (inctx_flags & INCTX_MODULE)
		emit_token = T_MODEND;
	return lex_close_source();
}

static int
isemptystr(char *text)
{
	for (; *text && mu_isspace (*text); text++)
		;
	return *text == 0;
}

void
tie_in_onblock(int enable)
{
	if (enable)
		BEGIN(ONBLOCK);
	else
		BEGIN(INITIAL);
}

data_type_t
builtin_const_value(const char *s, size_t len, const char **sval, long *nval)
{
	if (strncmp(s, "__file__", len) == 0) {
		struct literal *lit = literal_lookup(yylloc.beg.mu_file);
		*sval = lit->text;
		return dtype_string;
	} else if (strncmp(s, "__line__", len) == 0) {
		*nval = yylloc.beg.mu_line;
		return dtype_number;
	} else if (strncmp(s, "__function__", len) == 0) {
		*sval = function_name();
		return dtype_string;
	} else if (strncmp(s, "__package__", len) == 0) {
		*sval = PACKAGE_TARNAME;
		return dtype_string;
	} else if (strncmp(s, "__version__", len) == 0) {
		*sval = PACKAGE_VERSION;
		return dtype_string;
	} else if (strncmp(s, "__major__", len) == 0) {
		*nval = MAILFROMD_VERSION_MAJOR;
		return dtype_number;
	} else if (strncmp(s, "__minor__", len) == 0) {
		*nval = MAILFROMD_VERSION_MINOR;
		return dtype_number;
	} else if (strncmp(s, "__patch__", len) == 0) {
		*nval = MAILFROMD_VERSION_PATCH;
		return dtype_number;
	} else if (strncmp(s, "__git__", len) == 0) {
#ifdef GIT_DESCRIBE
		*sval = GIT_DESCRIBE;
#else
		*sval = "";
#endif
		return dtype_string;
	} else if (strncmp(s, "__statedir__", len) == 0) {
                *sval = mailfromd_state_dir;
		return dtype_string;
	} else if (strncmp(s, "__defstatedir__", len) == 0) {
		*sval = DEFAULT_STATE_DIR;
		return dtype_string;
	} else if (strncmp(s, "__preproc__", len) == 0) {
		*sval = ext_pp ? ext_pp : "";
		return dtype_string;
	} else if (strncmp(s, "__defpreproc__", len) == 0) {
		*sval = DEF_EXT_PP ? DEF_EXT_PP : "";
		return dtype_string;
	} else if (strncmp(s, "__module__", len) == 0) {
		*sval = top_module->dclname ? top_module->dclname : top_module->name;
		return dtype_string;
	}
        return dtype_unspecified;  
}

int
builtin_const(const char *s, size_t len)
{
	const char *sval;
	long nval;
	
	switch (builtin_const_value(s, len, &sval, &nval)) {
	case dtype_number:
		yylval.number = nval;
		return T_NUMBER;

	case dtype_string:
		string(sval, strlen(sval));
		return T_STRING;

	default:
		abort();
	}
	return T_BOGUS;
}

const char *
symbit_to_qualifier(unsigned f)
{
        switch (f) {
	case SYM_PRECIOUS:
		return "precious";
	case SYM_STATIC:
		return "static";
	case SYM_PUBLIC:
		return "public";
	}
	return NULL;
}

/* End of lex.l */