AwkParser

package io.jawk.frontend;

/*-
 * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
 * Jawk
 * ჻჻჻჻჻჻
 * Copyright (C) 2006 - 2026 MetricsHub
 * ჻჻჻჻჻჻
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Lesser Public License for more details.
 *
 * You should have received a copy of the GNU General Lesser Public
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/lgpl-3.0.html>.
 * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
 */

import java.io.File;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import io.jawk.AwkSandboxException;
import io.jawk.NotImplementedError;
import io.jawk.backend.AVM;
import io.jawk.ext.ExtensionFunction;
import io.jawk.intermediate.Address;
import io.jawk.intermediate.AwkTuples;
import io.jawk.intermediate.BuiltinFunction;
import io.jawk.jrt.JRT;
import io.jawk.intermediate.Tuple;
import io.jawk.util.ScriptFileSource;
import io.jawk.util.ScriptSource;
import io.jawk.frontend.ast.LexerException;
import io.jawk.frontend.ast.ParserException;

/**
 * Converts the AWK script into a syntax tree,
 * which is useful the backend that either compiles or interprets the script.
 * <p>
 * It contains the internal state of the parser and the lexer.
 *
 * @author Danny Daglas
 */
public class AwkParser {

	/**
	 * Flags that describe special behaviours of AST nodes. These replace the
	 * previous marker interfaces such as {@code Breakable} and
	 * {@code NonStatementAst}.
	 */
	private enum AstFlag {
		BREAKABLE,
		NEXTABLE,
		CONTINUEABLE,
		RETURNABLE,
		NON_STATEMENT
	}

	/** Lexer token values. */
	enum Token {
		EOF,
		NEWLINE,
		SEMICOLON,
		ID,
		FUNC_ID,
		INTEGER,
		DOUBLE,
		STRING,

		EQUALS,

		AND,
		OR,

		EQ,
		GT,
		GE,
		LT,
		LE,
		NE,
		NOT,
		PIPE,
		QUESTION_MARK,
		COLON,
		APPEND,

		PLUS,
		MINUS,
		MULT,
		DIVIDE,
		MOD,
		POW,
		COMMA,
		MATCHES,
		NOT_MATCHES,
		DOLLAR,

		INC,
		DEC,

		PLUS_EQ,
		MINUS_EQ,
		MULT_EQ,
		DIV_EQ,
		MOD_EQ,
		POW_EQ,

		OPEN_PAREN,
		CLOSE_PAREN,
		OPEN_BRACE,
		CLOSE_BRACE,
		OPEN_BRACKET,
		CLOSE_BRACKET,

		BUILTIN_FUNC_NAME,

		EXTENSION,
		TYPED_REGEXP,
		INDIRECT,
		DIRECTIVE_INCLUDE,
		DIRECTIVE_NAMESPACE,
		DIRECTIVE_UNSUPPORTED,

		KW_FUNCTION,
		KW_BEGIN,
		KW_END,
		KW_BEGINFILE,
		KW_ENDFILE,
		KW_IN,
		KW_IF,
		KW_ELSE,
		KW_WHILE,
		KW_FOR,
		KW_DO,
		KW_RETURN,
		KW_EXIT,
		KW_NEXT,
		KW_NEXTFILE,
		KW_CONTINUE,
		KW_DELETE,
		KW_BREAK,
		KW_PRINT,
		KW_PRINTF,
		KW_GETLINE
	}

	/**
	 * Contains a mapping of Jawk keywords to their
	 * token values.
	 * They closely correspond to AWK keywords, but with
	 * a few added extensions.
	 * <p>
	 * Keys are the keywords themselves, and values are the
	 * token values (equivalent to yytok values in lex/yacc).
	 * <p>
	 * <strong>Note:</strong> whether built-in AWK function names
	 * and special AWK variable names are formally keywords or not,
	 * they are not stored in this map. They are separated
	 * into other maps.
	 */
	private static final Map<String, Token> KEYWORDS = new HashMap<String, Token>();

	static {
		// special keywords
		KEYWORDS.put("function", Token.KW_FUNCTION);
		KEYWORDS.put("BEGIN", Token.KW_BEGIN);
		KEYWORDS.put("END", Token.KW_END);
		KEYWORDS.put("BEGINFILE", Token.KW_BEGINFILE);
		KEYWORDS.put("ENDFILE", Token.KW_ENDFILE);
		KEYWORDS.put("in", Token.KW_IN);

		// statements
		KEYWORDS.put("if", Token.KW_IF);
		KEYWORDS.put("else", Token.KW_ELSE);
		KEYWORDS.put("while", Token.KW_WHILE);
		KEYWORDS.put("for", Token.KW_FOR);
		KEYWORDS.put("do", Token.KW_DO);
		KEYWORDS.put("return", Token.KW_RETURN);
		KEYWORDS.put("exit", Token.KW_EXIT);
		KEYWORDS.put("next", Token.KW_NEXT);
		KEYWORDS.put("nextfile", Token.KW_NEXTFILE);
		KEYWORDS.put("continue", Token.KW_CONTINUE);
		KEYWORDS.put("delete", Token.KW_DELETE);
		KEYWORDS.put("break", Token.KW_BREAK);

		// special-form functions
		KEYWORDS.put("print", Token.KW_PRINT);
		KEYWORDS.put("printf", Token.KW_PRINTF);
		KEYWORDS.put("getline", Token.KW_GETLINE);
	}

	private static final int SP_IDX = 257;
	/**
	 * Contains a mapping of Jawk special variables to their
	 * variable token values.
	 * As of this writing, they correspond exactly to
	 * standard AWK variables, no more, no less.
	 * <p>
	 * Keys are the variable names themselves, and values are the
	 * variable token values.
	 */
	private static final Map<String, Integer> SPECIAL_VAR_NAMES = new HashMap<String, Integer>();

	static {
		SPECIAL_VAR_NAMES.put("NR", SP_IDX);
		SPECIAL_VAR_NAMES.put("FNR", SP_IDX);
		SPECIAL_VAR_NAMES.put("NF", SP_IDX);
		SPECIAL_VAR_NAMES.put("FS", SP_IDX);
		SPECIAL_VAR_NAMES.put("RS", SP_IDX);
		SPECIAL_VAR_NAMES.put("OFS", SP_IDX);
		SPECIAL_VAR_NAMES.put("ORS", SP_IDX);
		SPECIAL_VAR_NAMES.put("RSTART", SP_IDX);
		SPECIAL_VAR_NAMES.put("RLENGTH", SP_IDX);
		SPECIAL_VAR_NAMES.put("FILENAME", SP_IDX);
		SPECIAL_VAR_NAMES.put("SUBSEP", SP_IDX);
		SPECIAL_VAR_NAMES.put("CONVFMT", SP_IDX);
		SPECIAL_VAR_NAMES.put("OFMT", SP_IDX);
		SPECIAL_VAR_NAMES.put("ENVIRON", SP_IDX);
		SPECIAL_VAR_NAMES.put("ARGC", SP_IDX);
		SPECIAL_VAR_NAMES.put("ARGV", SP_IDX);
		SPECIAL_VAR_NAMES.put("IGNORECASE", SP_IDX);
		SPECIAL_VAR_NAMES.put("ERRNO", SP_IDX);
		SPECIAL_VAR_NAMES.put("ARGIND", SP_IDX);
	}

	/**
	 * Defined as concrete implementation class (not an
	 * interface reference) as to not clutter the interface
	 * with methods appropriate for private access, only.
	 */
	private final AwkSymbolTableImpl symbolTable = new AwkSymbolTableImpl();

	private final Map<String, ExtensionFunction> extensions;

	/** POSIX compile-time mode: rejects gawk syntax such as arrays of arrays and typed regexps. */
	private final boolean posix;

	/** Whether the compiling engine permits source inclusion from the filesystem. */
	private final boolean sourceIncludeAllowed;

	/**
	 * <p>
	 * Constructor for AwkParser.
	 * </p>
	 *
	 * @param extensions a {@link java.util.Map} object
	 * @param posix {@code true} to enforce POSIX compile-time behavior
	 */
	public AwkParser(Map<String, ExtensionFunction> extensions, boolean posix) {
		this(extensions, posix, true);
	}

	/**
	 * Creates a parser with an explicit source-inclusion policy.
	 *
	 * @param extensions extension functions available during parsing
	 * @param posix {@code true} to enforce POSIX compile-time behavior
	 * @param sourceIncludeAllowed {@code true} to permit {@code @include}
	 */
	public AwkParser(
			Map<String, ExtensionFunction> extensions,
			boolean posix,
			boolean sourceIncludeAllowed) {
		this.extensions = extensions == null ?
				Collections.emptyMap() : Collections.unmodifiableMap(new HashMap<>(extensions));
		this.posix = posix;
		this.sourceIncludeAllowed = sourceIncludeAllowed;
	}

	/**
	 * Returns whether the keyword is disabled in the current compile-time
	 * mode. BEGINFILE and ENDFILE are gawk extensions: in POSIX mode they are
	 * not special and lex as plain identifiers, exactly like
	 * {@code gawk --posix}.
	 *
	 * @param keywordToken the keyword token to inspect
	 * @return {@code true} when the keyword must be treated as an identifier
	 */
	private boolean isDisabledKeyword(Token keywordToken) {
		return posix && (keywordToken == Token.KW_BEGINFILE || keywordToken == Token.KW_ENDFILE);
	}

	private boolean isAwkNamespaceIdentifier(String identifier) {
		int separator = identifier.indexOf("::");
		if (separator >= 0) {
			return "awk".equals(identifier.substring(0, separator));
		}
		return "awk".equals(currentNamespace);
	}

	private String awkNamespaceComponent(String identifier) {
		return identifier.startsWith("awk::") ? identifier.substring("awk::".length()) : identifier;
	}

	private String qualifyGlobalIdentifier(String identifier) {
		int separator = identifier.indexOf("::");
		if (separator >= 0) {
			return "awk".equals(identifier.substring(0, separator)) ? identifier.substring(separator + 2) : identifier;
		}
		if ("awk".equals(currentNamespace) || isAllUppercaseIdentifier(identifier)) {
			return identifier;
		}
		return currentNamespace + "::" + identifier;
	}

	private boolean isAllUppercaseIdentifier(String identifier) {
		if (identifier.isEmpty()) {
			return false;
		}
		for (int i = 0; i < identifier.length(); i++) {
			char ch = identifier.charAt(i);
			if (ch < 'A' || ch > 'Z') {
				return false;
			}
		}
		return true;
	}

	private List<ScriptSource> scriptSources;
	private int scriptSourcesCurrentIndex;
	private ScriptSource currentScriptSource;
	private LineNumberReader reader;
	private int c;
	private Token token;
	private String pendingIndirectIdentifier;
	private boolean pendingColon;
	private String currentNamespace = "awk";
	private long conditionPairCount;
	private final Deque<SourceState> includedSourceStack = new ArrayDeque<SourceState>();
	private final Set<Path> includedSourcePaths = new HashSet<Path>();
	private final Set<Path> topLevelSourcePaths = new HashSet<Path>();

	private StringBuffer text = new StringBuffer();
	private StringBuffer string = new StringBuffer();
	private StringBuffer regexp = new StringBuffer();

	private static final class SourceState {
		private final ScriptSource scriptSource;
		private final LineNumberReader reader;
		private final int currentCharacter;
		private final String namespace;

		private SourceState(
				ScriptSource scriptSourceParam,
				LineNumberReader readerParam,
				int currentCharacterParam,
				String namespaceParam) {
			scriptSource = scriptSourceParam;
			reader = readerParam;
			currentCharacter = currentCharacterParam;
			namespace = namespaceParam;
		}
	}

	private void read() throws IOException {
		text.append((char) c);
		c = reader.read();
		// completely bypass \r's
		while (c == '\r') {
			c = reader.read();
		}
	}

	/**
	 * Consumes the exponent part of a numeric constant ({@code e} or {@code E},
	 * an optional sign, and at least one digit) when the current character
	 * starts one. When the characters after the {@code e}/{@code E} do not form
	 * a valid exponent, nothing is consumed, so the {@code e}/{@code E} starts
	 * the next token: {@code 1e} is the number {@code 1} followed by the
	 * identifier {@code e}, as in gawk.
	 *
	 * @return whether an exponent part was consumed
	 * @throws IOException upon an I/O error while reading the script source
	 */
	private boolean readExponent() throws IOException {
		if (c != 'e' && c != 'E') {
			return false;
		}
		// Peek past the 'e'/'E' (and an optional sign) for a digit before
		// consuming anything: the lexer keeps only one character of lookahead
		// in c, so the decision must be made without advancing.
		reader.mark(3);
		int first = reader.read();
		boolean validExponent;
		if (first == '+' || first == '-') {
			validExponent = Character.isDigit(reader.read());
		} else {
			validExponent = Character.isDigit(first);
		}
		reader.reset();
		if (!validExponent) {
			return false;
		}
		read();
		if (c == '+' || c == '-') {
			read();
		}
		while (c > 0 && Character.isDigit(c)) {
			read();
		}
		return true;
	}

	/**
	 * Advances to the next readable source at a token boundary when the current
	 * reader has reached end-of-file. Deferring this transition until the current
	 * token is complete preserves the source and namespace used to classify its
	 * final token. Included files are unwound first in LIFO order, restoring the
	 * including reader, its unread character, and its namespace. Once the include
	 * stack is empty, parsing continues with the next top-level script source,
	 * whose namespace starts at {@code awk}.
	 *
	 * @throws IOException if a source cannot be closed or read
	 */
	private void advancePastEndOfSource() throws IOException {
		while (c < 0) {
			if (!includedSourceStack.isEmpty()) {
				reader.close();
				SourceState previous = includedSourceStack.pop();
				currentScriptSource = previous.scriptSource;
				reader = previous.reader;
				c = previous.currentCharacter;
				currentNamespace = previous.namespace;
			} else if ((scriptSourcesCurrentIndex + 1) < scriptSources.size()) {
				scriptSourcesCurrentIndex++;
				currentScriptSource = scriptSources.get(scriptSourcesCurrentIndex);
				reader = new LineNumberReader(currentScriptSource.getReader());
				currentNamespace = "awk";
				c = reader.read();
				while (c == '\r') {
					c = reader.read();
				}
			} else {
				return;
			}
		}
	}

	/**
	 * Skip all whitespaces and comments
	 *
	 * @throws IOException
	 */
	private void skipWhitespaces() throws IOException {
		while (c == ' ' || c == '\t' || c == '#' || c == '\n') {
			if (c == '#') {
				while (c >= 0 && c != '\n') {
					read();
				}
			}
			read();
		}
	}

	/**
	 * Parse the script streamed by script_reader. Build and return the
	 * root of the abstract syntax tree which represents the Jawk script.
	 *
	 * @param localScriptSources List of script sources
	 * @return The abstract syntax tree of this script.
	 * @throws java.io.IOException upon an IO error.
	 */
	public AstNode parse(List<ScriptSource> localScriptSources) throws IOException {
		if (localScriptSources == null || localScriptSources.isEmpty()) {
			throw new IOException("No script sources supplied");
		}
		this.scriptSources = Collections.unmodifiableList(new ArrayList<>(localScriptSources));
		scriptSourcesCurrentIndex = 0;
		currentScriptSource = this.scriptSources.get(scriptSourcesCurrentIndex);
		reader = new LineNumberReader(currentScriptSource.getReader());
		currentNamespace = "awk";
		includedSourceStack.clear();
		resetIncludedSourcePaths();
		pendingIndirectIdentifier = null;
		pendingColon = false;
		read();
		lexer();
		return SCRIPT();
	}

	/**
	 * Parse a single AWK expression and return the corresponding AST.
	 *
	 * @param expressionSource The expression to parse (not a statement or rule, just an expression)
	 * @return tuples representing the expression
	 * @throws IOException upon an IO error or parsing error
	 */
	public AstNode parseExpression(ScriptSource expressionSource) throws IOException {

		// Sanity check
		if (expressionSource == null) {
			throw new IOException("No source supplied");
		}

		// Reader of the expression
		this.scriptSources = Collections.singletonList(expressionSource);
		scriptSourcesCurrentIndex = 0;
		currentScriptSource = expressionSource;
		reader = new LineNumberReader(currentScriptSource.getReader());
		currentNamespace = "awk";
		includedSourceStack.clear();
		resetIncludedSourcePaths();
		pendingIndirectIdentifier = null;
		pendingColon = false;

		// Initialize the lexer
		read();
		lexer();

		// An expression is a TERNARY_EXPRESSION
		return EXPRESSION_TO_EVALUATE();
	}

	private void resetIncludedSourcePaths() throws IOException {
		includedSourcePaths.clear();
		topLevelSourcePaths.clear();
		for (ScriptSource source : scriptSources) {
			if (source instanceof ScriptFileSource) {
				String filePath = ((ScriptFileSource) source).getFilePath();
				Path sourcePath = Paths.get(filePath).toRealPath();
				includedSourcePaths.add(sourcePath);
				topLevelSourcePaths.add(sourcePath);
			}
		}
	}

	private LexerException lexerException(String msg) {
		return new LexerException(
				msg,
				currentScriptSource.getDescription(),
				reader.getLineNumber());
	}

	/**
	 * Returns the current 1-based source line number to stamp onto AST nodes that
	 * will later emit tuple line markers for runtime error reporting.
	 *
	 * @return current source line number using 1-based counting
	 */
	private int currentSourceLineNumber() {
		return reader.getLineNumber() + 1;
	}

	/**
	 * Reads the string and handle all escape codes.
	 *
	 * @throws IOException
	 */
	private void readString() throws IOException {
		string.setLength(0);

		while (token != Token.EOF && c > 0 && c != '"' && c != '\n') {
			if (c == '\\') {
				read();
				switch (c) {
				case 'n':
					string.append('\n');
					break;
				case 't':
					string.append('\t');
					break;
				case 'r':
					string.append('\r');
					break;
				case 'a':
					string.append('\007');
					break; // BEL 0x07
				case 'b':
					string.append('\010');
					break; // BS 0x08
				case 'f':
					string.append('\014');
					break; // FF 0x0C
				case 'v':
					string.append('\013');
					break; // VT 0x0B
				// Octal notation: \N \NN \NNN
				case '0':
				case '1':
				case '2':
				case '3':
				case '4':
				case '5':
				case '6':
				case '7': {
					int octalChar = c - '0';
					read();
					if (c >= '0' && c <= '7') {
						octalChar = (octalChar << 3) + c - '0';
						read();
						if (c >= '0' && c <= '7') {
							octalChar = (octalChar << 3) + c - '0';
							read();
						}
					}
					string.append((char) octalChar);
					continue;
				}
				// Hexadecimal notation: \xN \xNN
				case 'x': {
					int hexChar = 0;
					read();
					if (c >= '0' && c <= '9') {
						hexChar = c - '0';
					} else if (c >= 'A' && c <= 'F') {
						hexChar = c - 'A' + 10;
					} else if (c >= 'a' && c <= 'f') {
						hexChar = c - 'a' + 10;
					} else {
						string.append('x');
						continue;
					}
					read();
					if (c >= '0' && c <= '9') {
						hexChar = (hexChar << 4) + c - '0';
					} else if (c >= 'A' && c <= 'F') {
						hexChar = (hexChar << 4) + c - 'A' + 10;
					} else if (c >= 'a' && c <= 'f') {
						hexChar = (hexChar << 4) + c - 'a' + 10;
					} else {
						// Append what we already have, and continue directly, because we already have read the next char
						string.append((char) hexChar);
						continue;
					}
					string.append((char) hexChar);
					break;
				}
				default:
					string.append((char) c);
					break; // Remove the backslash
				}
			} else {
				string.append((char) c);
			}
			read();
		}
		if (token == Token.EOF || c == '\n' || c <= 0) {
			throw lexerException("Unterminated string: " + text);
		}
		read();
	}

	/**
	 * Reads the regular expression (between slashes '/') and handle '\/'.
	 * A slash within a bracket expression (e.g. {@code /[/]/}) does not
	 * terminate the regular expression, per POSIX ERE bracket semantics.
	 *
	 * @throws IOException
	 */
	private void readRegexp() throws IOException {
		regexp.setLength(0);

		boolean inBracket = false;
		while (token != Token.EOF && c > 0 && (c != '/' || inBracket) && c != '\n') {
			if (c == '\\') {
				read();
				if (c != '/') {
					regexp.append('\\');
				}
				regexp.append((char) c);
				read();
				continue;
			}
			if (!inBracket && c == '[') {
				inBracket = true;
				regexp.append((char) c);
				read();
				// a ']' right after '[' (or after '[^') is a literal ']'
				if (c == '^') {
					regexp.append((char) c);
					read();
				}
				if (c == ']') {
					regexp.append((char) c);
					read();
				}
				continue;
			}
			if (inBracket && c == '[') {
				regexp.append((char) c);
				read();
				// POSIX character class, collating element, or equivalence
				// class ([:alpha:], [.x.], [=e=]): its closing ']' does not
				// end the outer bracket expression.
				if (c == ':' || c == '.' || c == '=') {
					int delimiter = c;
					boolean closed = false;
					while (token != Token.EOF && c > 0 && c != '\n' && !closed) {
						int previous = c;
						regexp.append((char) c);
						read();
						if (previous == delimiter && c == ']') {
							regexp.append((char) c);
							read();
							closed = true;
						}
					}
				}
				continue;
			}
			if (inBracket && c == ']') {
				inBracket = false;
			}
			regexp.append((char) c);
			read();
		}
		if (token == Token.EOF || c == '\n' || c <= 0) {
			throw lexerException("Unterminated string: " + text);
		}
		read();
	}

	private Token lexer(Token expectedToken) throws IOException {
		if (token != expectedToken) {
			throw parserException(
					"Expecting " + expectedToken.name() + ". Found: " + token.name() + " (" + text + ")");
		}
		return lexer();
	}

	private Token lexer() throws IOException {
		// clear whitespace
		while (true) {
			advancePastEndOfSource();
			if (c < 0 || c != ' ' && c != '\t' && c != '#' && c != '\\') {
				break;
			}
			if (c == '\\') {
				read();
				if (c == '\n') {
					read();
				}
				continue;
			}
			if (c == '#') {
				// kill comment
				while (c >= 0 && c != '\n') {
					read();
				}
			} else {
				read();
			}
		}
		text.setLength(0);
		if (pendingColon) {
			pendingColon = false;
			token = Token.COLON;
			return token;
		}
		if (pendingIndirectIdentifier != null) {
			text.append(pendingIndirectIdentifier);
			pendingIndirectIdentifier = null;
			token = Token.ID;
			return token;
		}
		if (c < 0) {
			token = Token.EOF;
			return token;
		}
		if (c == ',') {
			read();
			skipWhitespaces();
			token = Token.COMMA;
			return token;
		}
		if (c == '(') {
			read();
			token = Token.OPEN_PAREN;
			return token;
		}
		if (c == ')') {
			read();
			token = Token.CLOSE_PAREN;
			return token;
		}
		if (c == '{') {
			read();
			skipWhitespaces();
			token = Token.OPEN_BRACE;
			return token;
		}
		if (c == '}') {
			read();
			token = Token.CLOSE_BRACE;
			return token;
		}
		if (c == '[') {
			read();
			token = Token.OPEN_BRACKET;
			return token;
		}
		if (c == ']') {
			read();
			token = Token.CLOSE_BRACKET;
			return token;
		}
		if (c == '$') {
			read();
			token = Token.DOLLAR;
			return token;
		}
		if (c == '@') {
			if (posix) {
				throw lexerException("gawk @ syntax is not supported in POSIX mode.");
			}
			read();
			if (c == '/') {
				read();
				readRegexp();
				token = Token.TYPED_REGEXP;
				return token;
			}
			if (Character.isJavaIdentifierStart(c)) {
				while (Character.isJavaIdentifierPart(c)) {
					read();
				}
				if (c == ':') {
					read();
					if (c != ':') {
						throw lexerException("Namespace separator must be two colons (::).");
					}
					read();
					if (!Character.isJavaIdentifierStart(c)) {
						throw lexerException("A namespace-qualified name requires an identifier after ::.");
					}
					read();
					while (Character.isJavaIdentifierPart(c)) {
						read();
					}
				}
				String atWord = text.toString();
				if ("@include".equals(atWord)) {
					token = Token.DIRECTIVE_INCLUDE;
					return token;
				}
				if ("@namespace".equals(atWord)) {
					token = Token.DIRECTIVE_NAMESPACE;
					return token;
				}
				if ("@load".equals(atWord)) {
					token = Token.DIRECTIVE_UNSUPPORTED;
					return token;
				}
				pendingIndirectIdentifier = atWord.substring(1);
				validateIndirectIdentifier(pendingIndirectIdentifier);
				token = Token.INDIRECT;
				return token;
			}
			token = Token.INDIRECT;
			return token;
		}
		if (c == '~') {
			read();
			token = Token.MATCHES;
			return token;
		}
		if (c == '?') {
			read();
			skipWhitespaces();
			token = Token.QUESTION_MARK;
			return token;
		}
		if (c == ':') {
			read();
			skipWhitespaces();
			token = Token.COLON;
			return token;
		}
		if (c == '&') {
			read();
			if (c == '&') {
				read();
				skipWhitespaces();
				token = Token.AND;
				return token;
			}
			throw lexerException("use && for logical and");
		}
		if (c == '|') {
			read();
			if (c == '|') {
				read();
				skipWhitespaces();
				token = Token.OR;
				return token;
			}
			token = Token.PIPE;
			return token;
		}
		if (c == '=') {
			read();
			if (c == '=') {
				read();
				token = Token.EQ;
				return token;
			}
			token = Token.EQUALS;
			return token;
		}
		if (c == '+') {
			read();
			if (c == '=') {
				read();
				token = Token.PLUS_EQ;
				return token;
			} else if (c == '+') {
				read();
				token = Token.INC;
				return token;
			}
			token = Token.PLUS;
			return token;
		}
		if (c == '-') {
			read();
			if (c == '=') {
				read();
				token = Token.MINUS_EQ;
				return token;
			} else if (c == '-') {
				read();
				token = Token.DEC;
				return token;
			}
			token = Token.MINUS;
			return token;
		}
		if (c == '*') {
			read();
			if (c == '=') {
				read();
				token = Token.MULT_EQ;
				return token;
			} else if (c == '*') {
				read();
				if (c == '=') {
					read();
					token = Token.POW_EQ;
					return token;
				}
				token = Token.POW;
				return token;
			}
			token = Token.MULT;
			return token;
		}
		if (c == '/') {
			read();
			if (c == '=') {
				read();
				token = Token.DIV_EQ;
				return token;
			}
			token = Token.DIVIDE;
			return token;
		}
		if (c == '%') {
			read();
			if (c == '=') {
				read();
				token = Token.MOD_EQ;
				return token;
			}
			token = Token.MOD;
			return token;
		}
		if (c == '^') {
			read();
			if (c == '=') {
				read();
				token = Token.POW_EQ;
				return token;
			}
			token = Token.POW;
			return token;
		}
 		if (c == '>') {
 			read();
 			if (c == '=') {
 				read();
 				token = Token.GE;
 				return token;
 			} else if (c == '>') {
 				read();
 				token = Token.APPEND;
 				return token;
 			}
 			token = Token.GT;
 			return token;
 		}
 		if (c == '<') {
 			read();
 			if (c == '=') {
 				read();
 				token = Token.LE;
 				return token;
 			}
 			token = Token.LT;
 			return token;
 		}
 		if (c == '!') {
 			read();
 			if (c == '=') {
 				read();
 				token = Token.NE;
 				return token;
 			} else if (c == '~') {
 				read();
 				token = Token.NOT_MATCHES;
 				return token;
 			}
 			token = Token.NOT;
 			return token;
 		}

 		if (c == '.') {
 			// double!
 			read();
 			boolean hit = false;
 			while (c > 0 && Character.isDigit(c)) {
 				hit = true;
 				read();
 			}
 			if (!hit) {
 				throw lexerException("Decimal point encountered with no values on either side.");
 			}
 			readExponent();
 			token = Token.DOUBLE;
 			return token;
 		}

 		if (Character.isDigit(c)) {
 			// integer or double.
 			read();
 			while (c > 0) {
 				if (c == '.') {
 					// double!
 					read();
 					while (c > 0 && Character.isDigit(c)) {
 						read();
 					}
 					readExponent();
 					token = Token.DOUBLE;
 					return token;
 				} else if (Character.isDigit(c)) {
 					// integer or double.
 					read();
 				} else {
 					break;
 				}
 			}
 			if (readExponent()) {
 				token = Token.DOUBLE;
 				return token;
 			}
 			// integer, only
 			token = Token.INTEGER;
 			return token;
 		}

 		if (Character.isJavaIdentifierStart(c)) {
 			read();
 			while (Character.isJavaIdentifierPart(c)) {
 				read();
 			}
 			if (c == ':') {
 				read();
 				if (c != ':') {
 					text.setLength(text.length() - 1);
 					pendingColon = true;
 				} else {
 					if (posix) {
 						throw lexerException("gawk namespace syntax is not supported in POSIX mode.");
 					}
 					read();
 					if (!Character.isJavaIdentifierStart(c)) {
 						throw lexerException("A namespace-qualified name requires an identifier after ::.");
 					}
 					read();
 					while (Character.isJavaIdentifierPart(c)) {
 						read();
 					}
 					if (c == ':') {
 						read();
 						if (c == ':') {
 							throw lexerException("A namespace-qualified name may contain only one :: separator.");
 						}
 						text.setLength(text.length() - 1);
 						pendingColon = true;
 					}
 				}
 			}
 			// check for certain keywords
 			// extensions override built-in stuff
 			String sourceIdentifier = text.toString();
 			int namespaceSeparator = sourceIdentifier.indexOf("::");
 			if (namespaceSeparator >= 0) {
 				String namespaceComponent = sourceIdentifier.substring(namespaceSeparator + 2);
 				if (KEYWORDS.containsKey(namespaceComponent)
 						|| BuiltinFunction.of(namespaceComponent) != null) {
 					throw lexerException(
 							"Reserved word cannot be used after a namespace separator: "
 									+ sourceIdentifier);
 				}
 			}
 			String lookupIdentifier = awkNamespaceComponent(sourceIdentifier);
 			boolean awkNamespaceIdentifier = isAwkNamespaceIdentifier(sourceIdentifier);
 			boolean unqualifiedIdentifier = namespaceSeparator < 0;
 			if (awkNamespaceIdentifier && extensions.get(lookupIdentifier) != null) {
 				text.setLength(0);
 				text.append(lookupIdentifier);
 				token = Token.EXTENSION;
 				return token;
 			}
 			Token kwToken = KEYWORDS.get(sourceIdentifier);
 			if (kwToken != null && !isDisabledKeyword(kwToken)) {
 				token = kwToken;
 				return token;
 			}
 			if ((unqualifiedIdentifier || awkNamespaceIdentifier)
 					&& BuiltinFunction.of(lookupIdentifier) != null) {
 				text.setLength(0);
 				text.append(lookupIdentifier);
 				token = Token.BUILTIN_FUNC_NAME;
 				return token;
 			}
 			if (c == '(' && !pendingColon) {
 				token = Token.FUNC_ID;
 				return token;
 			} else {
 				token = Token.ID;
 				return token;
 			}
 		}

 		if (c == ';') {
 			read();
 			while (c == ' ' || c == '\t' || c == '\n' || c == '#') {
 				if (c == '\n') {
 					break;
 				}
 				if (c == '#') {
 					while (c >= 0 && c != '\n') {
 						read();
 					}
 					if (c == '\n') {
 						read();
 					}
 				} else {
 					read();
 				}
 			}
 			token = Token.SEMICOLON;
 			return token;
 		}

 		if (c == '\n') {
 			read();
 			while (c == ' ' || c == '\t' || c == '#' || c == '\n') {
 				if (c == '#') {
 					while (c >= 0 && c != '\n') {
 						read();
 					}
 				}
 				read();
 			}
 			token = Token.NEWLINE;
 			return token;
 		}

 		if (c == '"') {
 			// string
 			read();
 			readString();
 			token = Token.STRING;
 			return token;
 		}

 		/*
 		 * if (c == '\\') {
 		 * c = reader.read();
 		 * // completely bypass \r's
 		 * while(c == '\r') c = reader.read();
 		 * if (c<0)
 		 * chr=0; // eof
 		 * else
 		 * chr=c;
 		 * }
 		 */

 		throw lexerException("Invalid character (" + c + "): " + ((char) c));
 	}

 	// SUPPORTING FUNCTIONS/METHODS
 	private void terminator() throws IOException {
 		// like optTerminator, except error if no terminator was found
 		if (!optTerminator()) {
 			throw parserException("Expecting statement terminator. Got " + token.name() + ": " + text);
 		}
 	}

 	private boolean optTerminator() throws IOException {
 		if (optNewline()) {
 			return true;
 		} else if (token == Token.EOF || token == Token.CLOSE_BRACE) {
 			return true; // do nothing
 		} else if (token == Token.SEMICOLON) {
 			lexer();
 			return true;
 		} else {
 			// no terminator consumed
 			return false;
 		}
 	}

 	private boolean optNewline() throws IOException {
 		if (token == Token.NEWLINE) {
 			lexer();
 			return true;
 		} else {
 			return false;
 		}
 	}

 	// RECURSIVE DECENT PARSER:
 	// CHECKSTYLE.OFF: MethodName
 	// SCRIPT : \n [RULE_LIST] Token.EOF
 	AST SCRIPT() throws IOException {
 		AST rl;
 		if (token != Token.EOF) {
 			rl = RULE_LIST();
 		} else {
 			rl = null;
 		}
 		lexer(Token.EOF);
 		return rl;
 	}

 	// EXPRESSION_TO_EVALUATE: [TERNARY_EXPRESSION] Token.EOF
 	// Used to parse simple expressions to evaluate instead of full scripts
 	AST EXPRESSION_TO_EVALUATE() throws IOException {
 		AST exprAst = token != Token.EOF ? TERNARY_EXPRESSION(null, true, false, true) : null;
 		lexer(Token.EOF);
 		return new ExpressionToEvaluateAst(exprAst);
 	}

 	// RULE_LIST : \n [ ( RULE | FUNCTION terminator ) optTerminator RULE_LIST ]
 	AST RULE_LIST() throws IOException {
 		optNewline();
 		AST ruleOrFunction = null;
 		if (token == Token.DIRECTIVE_INCLUDE) {
 			INCLUDE_DIRECTIVE();
 			return RULE_LIST();
 		} else if (token == Token.DIRECTIVE_NAMESPACE) {
 			NAMESPACE_DIRECTIVE();
 			return RULE_LIST();
 		} else if (token == Token.DIRECTIVE_UNSUPPORTED) {
 			throw parserException("Unsupported gawk directive: " + text);
 		} else if (token == Token.KW_FUNCTION) {
 			ruleOrFunction = FUNCTION();
 		} else if (token != Token.EOF) {
 			ruleOrFunction = RULE();
 		} else {
 			return null;
 		}
 		optTerminator(); // newline or ; (maybe)
 		return new RuleListAst(ruleOrFunction, RULE_LIST());
 	}

 	private void NAMESPACE_DIRECTIVE() throws IOException {
 		lexer();
 		if (token != Token.STRING) {
 			throw parserException("@namespace requires a quoted namespace name.");
 		}
 		String namespace = string.toString();
 		validateNamespace(namespace);
 		currentNamespace = namespace;
 		lexer();
 		terminator();
 	}

 	private void INCLUDE_DIRECTIVE() throws IOException {
 		lexer();
 		if (token != Token.STRING) {
 			throw parserException("@include requires a quoted file name.");
 		}
 		if (!sourceIncludeAllowed) {
 			throw new AwkSandboxException("@include is disabled in sandbox mode");
 		}
 		String includeName = string.toString();
 		boolean includeTerminatedByEndOfSource = validateIncludeTerminator();
 		Path includePath = resolveIncludePath(includeName);
 		if (topLevelSourcePaths.contains(includePath)) {
 			throw parserException(
 					"Cannot include a top-level program source: " + includeName);
 		}
 		if (!includedSourcePaths.add(includePath)) {
 			lexer();
 			if (!includeTerminatedByEndOfSource) {
 				terminator();
 			}
 			return;
 		}
 		includedSourceStack.push(new SourceState(currentScriptSource, reader, c, currentNamespace));
 		currentScriptSource = new ScriptSource(
 				includePath.toString(),
 				Files.newBufferedReader(includePath, StandardCharsets.UTF_8));
 		reader = new LineNumberReader(currentScriptSource.getReader());
 		currentNamespace = "awk";
 		c = reader.read();
 		while (c == '\r') {
 			c = reader.read();
 		}
 		advancePastEndOfSource();
 		lexer();
 	}

 	private boolean validateIncludeTerminator() throws IOException {
 		while (c == ' ' || c == '\t') {
 			read();
 		}
 		if (c == '#') {
 			while (c >= 0 && c != '\n') {
 				read();
 			}
 		}
 		if (c >= 0 && c != '\n' && c != ';') {
 			throw parserException("@include must be followed by a newline, semicolon, or end of file.");
 		}
 		return c < 0;
 	}

 	private void validateNamespace(String namespace) {
 		if (namespace == null
 				|| namespace.isEmpty()
 				|| !Character.isJavaIdentifierStart(namespace.charAt(0))) {
 			throw parserException("Invalid gawk namespace name: " + namespace);
 		}
 		for (int i = 1; i < namespace.length(); i++) {
 			if (!Character.isJavaIdentifierPart(namespace.charAt(i))) {
 				throw parserException("Invalid gawk namespace name: " + namespace);
 			}
 		}
 		if (KEYWORDS.containsKey(namespace)
 				|| BuiltinFunction.of(namespace) != null
 				|| extensions.containsKey(namespace)) {
 			throw parserException("Reserved identifier cannot be used as a gawk namespace: " + namespace);
 		}
 	}

 	private void validateIndirectIdentifier(String identifier) throws LexerException {
 		int separator = identifier.indexOf("::");
 		String namespace = separator < 0 ? "awk" : identifier.substring(0, separator);
 		String component = separator < 0 ? identifier : identifier.substring(separator + 2);
 		if (KEYWORDS.containsKey(component)
 				|| BuiltinFunction.of(component) != null
 				|| ("awk".equals(namespace) && extensions.containsKey(component))) {
 			throw lexerException("Reserved identifier cannot be used as an indirect-call selector: " + identifier);
 		}
 	}

 	private Path resolveIncludePath(String includeName) {
 		Path requested = Paths.get(includeName);
 		List<Path> candidates = new ArrayList<Path>();
 		if (requested.isAbsolute()) {
 			candidates.add(requested);
 		} else {
 			if (!ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT.equals(currentScriptSource.getDescription())) {
 				try {
 					Path sourcePath = Paths.get(currentScriptSource.getDescription());
 					Path parent = sourcePath.toAbsolutePath().normalize().getParent();
 					if (parent != null) {
 						candidates.add(parent.resolve(requested));
 					}
 				} catch (InvalidPathException ignored) {
 					// Reader-backed ScriptSource values may use a descriptive
 					// label rather than a file path.
 				}
 			}
 			String awkPath = System.getenv("AWKPATH");
 			if (awkPath != null) {
 				for (String entry : awkPath.split(java.util.regex.Pattern.quote(File.pathSeparator), -1)) {
 					candidates.add(Paths.get(entry.isEmpty() ? "." : entry).resolve(requested));
 				}
 			}
 			candidates.add(requested);
 		}
 		for (Path candidate : candidates) {
 			Path normalized = candidate.toAbsolutePath().normalize();
 			if (Files.isRegularFile(normalized)) {
 				try {
 					return normalized.toRealPath();
 				} catch (IOException ignored) {
 					// The candidate may have disappeared between the existence
 					// check and canonicalization; continue searching AWKPATH.
 				}
 			}
 		}
 		throw parserException("Cannot find @include file: " + includeName);
 	}

 	// FUNCTION: function functionName( [FORMAL_PARAM_LIST] ) STATEMENT_LIST
 	AST FUNCTION() throws IOException {
 		expectKeyword("function");
 		String functionName;
 		if (token == Token.FUNC_ID || token == Token.ID) {
 			functionName = qualifyGlobalIdentifier(text.toString());
 			lexer();
 		} else {
 			throw parserException("Expecting function name. Got " + token.name() + ": " + text);
 		}
 		symbolTable.setFunctionName(functionName);
 		lexer(Token.OPEN_PAREN);
 		AST formalParamList;
 		if (token == Token.CLOSE_PAREN) {
 			formalParamList = null;
 		} else {
 			formalParamList = FORMAL_PARAM_LIST(functionName);
 		}
 		lexer(Token.CLOSE_PAREN);
 		optNewline();

 		lexer(Token.OPEN_BRACE);
 		AST functionBlock = STATEMENT_LIST();
 		lexer(Token.CLOSE_BRACE);
 		symbolTable.clearFunctionName(functionName);
 		return symbolTable.addFunctionDef(functionName, formalParamList, functionBlock);
 	}

 	// FORMAT_PARAM_LIST:
 	AST FORMAL_PARAM_LIST(String functionName) throws IOException {
 		if (token == Token.ID) {
 			String id = text.toString();
 			symbolTable.addFunctionParameter(functionName, id);
 			lexer();
 			if (token == Token.COMMA) {
 				lexer();
 				optNewline();
 				AST rest = FORMAL_PARAM_LIST(functionName);
 				if (rest == null) {
 					throw parserException("Cannot terminate a formal parameter list with a comma.");
 				} else {
 					return new FunctionDefParamListAst(id, rest);
 				}
 			} else {
 				return new FunctionDefParamListAst(id, null);
 			}
 		} else {
 			return null;
 		}
 	}

 	// RULE : [ASSIGNMENT_EXPRESSION] [ { STATEMENT_LIST } ]
 	AST RULE() throws IOException {
 		AST optExpr;
 		AST optStmts;
 		if (token == Token.KW_BEGIN) {
 			lexer();
 			optExpr = symbolTable.addBEGIN();
 		} else if (token == Token.KW_END) {
 			lexer();
 			optExpr = symbolTable.addEND();
 		} else if (token == Token.KW_BEGINFILE) {
 			lexer();
 			optExpr = symbolTable.addBEGINFILE();
 		} else if (token == Token.KW_ENDFILE) {
 			lexer();
 			optExpr = symbolTable.addENDFILE();
 		} else if (token != Token.OPEN_BRACE && token != Token.SEMICOLON && token != Token.NEWLINE && token != Token.EOF) {
 			// true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions
 			optExpr = ASSIGNMENT_EXPRESSION(null, true, true, false);
 			// for ranges, like conditionStart, conditionEnd
 			if (token == Token.COMMA) {
 				lexer();
 				optNewline();
 				// true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions
 				optExpr = new ConditionPairAst(
 						optExpr,
 						ASSIGNMENT_EXPRESSION(null, true, true, false));
 			}
 		} else {
 			optExpr = null;
 		}
 		if (token == Token.OPEN_BRACE) {
 			lexer();
 			optStmts = STATEMENT_LIST();
 			lexer(Token.CLOSE_BRACE);
 		} else {
 			optStmts = null;
 		}
 		return new RuleAst(optExpr, optStmts);
 	}

 	// STATEMENT_LIST : [ STATEMENT_BLOCK|STATEMENT STATEMENT_LIST ]
 	private AST STATEMENT_LIST() throws IOException {
 		// statement lists can only live within curly brackets (braces)
 		optNewline();
 		if (token == Token.CLOSE_BRACE || token == Token.EOF) {
 			return null;
 		}
 		AST stmt;
 		if (token == Token.OPEN_BRACE) {
 			lexer();
 			stmt = STATEMENT_LIST();
 			lexer(Token.CLOSE_BRACE);
 		} else {
 			if (token == Token.SEMICOLON) {
 				// an empty statement (;)
 				// do not polute the syntax tree with nulls in this case
 				// just return the next statement (recursively)
 				lexer();
 				return STATEMENT_LIST();
 			} else {
 				stmt = STATEMENT();
 			}
 		}

 		AST rest = STATEMENT_LIST();
 		if (rest == null) {
 			return stmt;
 		} else if (stmt == null) {
 			return rest;
 		} else {
 			return new StatementListAst(stmt, rest);
 		}
 	}

 	/**
 	 * Parse a (possibly comma-separated) list of ASSIGNMENT_EXPRESSIONs.
 	 *
 	 * @param allowComparisons
 	 *        – true ⇒ treat ‘>’ and ‘<’ as comparison operators
 	 *        – false ⇒ treat ‘>’ and ‘<’ as redirection tokens (break out)
 	 * @param allowInKeyword
 	 *        – true ⇒ allow the “in” keyword inside expressions
 	 *        – false ⇒ disallow “in”
 	 */
 	AST EXPRESSION_LIST(boolean allowComparisons, boolean allowInKeyword) throws IOException {
 		// Captured before parsing: the list node's line is where its first
 		// expression starts, so a parenthesized group rebuilt into a
 		// multi-dimensional subscript reports errors there.
 		int lineNo = currentSourceLineNumber();

 		// 1) Parse exactly one assignment expression.
 		// Passing `allowComparisons` will decide if ‘>’/’<’ become comparisons or redirectors.
 		AST expr = ASSIGNMENT_EXPRESSION(null, allowComparisons, allowInKeyword, /* allowMultidim= */ false);

 		// 2) If the next token is a comma, consume it and build the rest of the list.
 		// This supports both regular function calls and print/printf argument lists.
 		if (token == Token.COMMA) {
 			lexer(); // consume ','
 			optNewline(); // allow newline after comma (AWK style)

 			AST rest = EXPRESSION_LIST(allowComparisons, allowInKeyword);
 			return new FunctionCallParamListAst(lineNo, expr, rest);
 		}

 		// 3) No comma ⇒ this single expression is a one‐element list.
 		return new FunctionCallParamListAst(lineNo, expr, null);
 	}

 	private AST ASSIGNMENT_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		// Captured before parsing: if the expression turns out to be the first
 		// component of a multi-dimensional subscript group, the group's line is
 		// where this component starts.
 		int startLineNo = currentSourceLineNumber();
 		AST ternaryExpression = TERNARY_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		AST result = ternaryExpression;
 		if (token == Token.EQUALS
 				|| token == Token.PLUS_EQ
 				|| token == Token.MINUS_EQ
 				|| token == Token.MULT_EQ
 				|| token == Token.DIV_EQ
 				|| token == Token.MOD_EQ
 				|| token == Token.POW_EQ) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			// An assignment RHS is a single expression: a following comma belongs to
 			// the enclosing grouping, as in ((y=1, 2) in a), where the group elements
 			// are (y=1) and (2)
 			AST assignmentExpression = ASSIGNMENT_EXPRESSION(
 					null,
 					allowComparison,
 					allowInKeyword,
 					false);
 			result = new AssignmentExpressionAst(ternaryExpression, op, txt, assignmentExpression);
 		}
 		// ASSIGNMENT_EXPRESSION [, ASSIGNMENT_EXPRESSION] !!!ONLY IF!!! allowMultidimIndices is true
 		// allowMultidimIndices is set to true when we need (1,2,3,4) expressions to collapse into an array index
 		// expression (converts 1,2,3,4 to 1 SUBSEP 2 SUBSEP 3 SUBSEP 4) after an open parenthesis (grouping)
 		// expression starter
 		if (allowMultidimIndices && token == Token.COMMA) {
 			lexer();
 			optNewline();
 			int restLineNo = currentSourceLineNumber();
 			AST rest = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, true);
 			if (rest instanceof ArrayIndexAst) {
 				return new ArrayIndexAst(startLineNo, result, rest);
 			}
 			return new ArrayIndexAst(startLineNo, result, new ArrayIndexAst(restLineNo, rest, null));
 		}
 		return result;
 	}

 	// TERNARY_EXPRESSION = LOGICAL_OR_EXPRESSION [ ? TERNARY_EXPRESSION : TERNARY_EXPRESSION ]
 	private AST TERNARY_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST condition = LOGICAL_OR_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		if (token == Token.QUESTION_MARK) {
 			lexer();
 			AST trueBlock = TERNARY_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			lexer(Token.COLON);
 			AST falseBlock = TERNARY_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			return new TernaryExpressionAst(condition, trueBlock, falseBlock);
 		}
 		return condition;
 	}

 	// LOGICAL_OR_EXPRESSION = LOGICAL_AND_EXPRESSION [ || LOGICAL_OR_EXPRESSION ]
 	private AST LOGICAL_OR_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = LOGICAL_AND_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		while (token == Token.OR) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST rhs = LOGICAL_OR_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			result = new LogicalExpressionAst(result, op, txt, rhs);
 		}
 		return result;
 	}

 	// LOGICAL_AND_EXPRESSION = IN_EXPRESSION [ && LOGICAL_AND_EXPRESSION ]
 	private AST LOGICAL_AND_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = IN_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		while (token == Token.AND) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST rhs = LOGICAL_AND_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			result = new LogicalExpressionAst(result, op, txt, rhs);
 		}
 		return result;
 	}

 	// IN_EXPRESSION = MATCHING_EXPRESSION [ IN_EXPRESSION ]
 	// allowInKeyword is set false while parsing the first expression within
 	// a for() statement (because it could be "for (key in arr)", and this
 	// production will consume and the for statement will never have a chance
 	// of processing it
 	// all other times, it is true
 	private AST IN_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = MATCHING_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		if (allowInKeyword && token == Token.KW_IN) {
 			lexer();
 			result = new InExpressionAst(
 					result,
 					IN_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices));
 		}
 		return result;
 	}

 	// MATCHING_EXPRESSION = COMPARISON_EXPRESSION [ (~,!~) MATCHING_EXPRESSION ]
 	private AST MATCHING_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = COMPARISON_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		while (token == Token.MATCHES || token == Token.NOT_MATCHES) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST rhs = MATCHING_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			result = new ComparisonExpressionAst(result, op, txt, rhs);
 		}
 		return result;
 	}

 	// COMPARISON_EXPRESSION = CONCAT_EXPRESSION [ (==,>,>=,<,<=,!=,|) COMPARISON_EXPRESSION ]
 	// allowComparison is set false when within a print/printf statement;
 	// all other times it is set true
 	private AST COMPARISON_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = CONCAT_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		if (token == Token.EQ
 				|| token == Token.GE
 				|| token == Token.LT
 				|| token == Token.LE
 				|| token == Token.NE
 				|| (token == Token.GT && allowComparison)) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST rhs = COMPARISON_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			return new ComparisonExpressionAst(result, op, txt, rhs);
 		} else if (allowComparison && token == Token.PIPE) {
 			lexer();
 			return GETLINE_EXPRESSION(result, allowComparison, allowInKeyword);
 		}

 		return result;
 	}

 	// CONCAT_EXPRESSION = EXPRESSION [ CONCAT_EXPRESSION ]
 	private AST CONCAT_EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		if (token == Token.INTEGER
 				|| token == Token.DOUBLE
 				|| token == Token.OPEN_PAREN
 				|| token == Token.FUNC_ID
 				|| token == Token.INC
 				|| token == Token.DEC
 				|| token == Token.ID
 				|| token == Token.STRING
 				|| token == Token.DOLLAR
 				|| token == Token.BUILTIN_FUNC_NAME
 				|| token == Token.EXTENSION) {
 			return new ConcatExpressionAst(
 					result,
 					CONCAT_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices));
 		}
 		return result;
 	}

 	// EXPRESSION : TERM [ (+|-) EXPRESSION ]
 	private AST EXPRESSION(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = TERM(left, allowComparison, allowInKeyword, allowMultidimIndices);
 		while (token == Token.PLUS || token == Token.MINUS) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST nextTerm = TERM(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			result = new BinaryExpressionAst(result, op, txt, nextTerm);
 		}
 		return result;
 	}

 	// TERM : UNARY_FACTOR [ (*|/|%) TERM ]
 	private AST TERM(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = (left == null) ? UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices) : left;
 		while (token == Token.MULT || token == Token.DIVIDE || token == Token.MOD) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST nextUnaryFactor = UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices);
 			result = new BinaryExpressionAst(result, op, txt, nextUnaryFactor);
 		}
 		return result;
 	}

 	// UNARY_FACTOR : [ ! | - | + ] POWER_FACTOR
 	AST UNARY_FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
 			throws IOException {
 		if (token == Token.NOT) {
 			lexer();
 			return new NotExpressionAst(POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
 		} else if (token == Token.MINUS) {
 			lexer();
 			return new NegativeExpressionAst(
 					POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
 		} else if (token == Token.PLUS) {
 			lexer();
 			return new UnaryPlusExpressionAst(
 					POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
 		} else {
 			return POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices);
 		}
 	}

 	// POWER_FACTOR : FACTOR_FOR_INCDEC [ ^ POWER_FACTOR ]
 	private AST POWER_FACTOR(
 			AST left,
 			boolean allowComparison,
 			boolean allowInKeyword,
 			boolean allowMultidimIndices)
 			throws IOException {
 		AST result = (left == null) ? FACTOR_FOR_INCDEC(allowComparison, allowInKeyword, allowMultidimIndices) : left;
 		if (token == Token.POW) {
 			Token op = token;
 			String txt = text.toString();
 			lexer();
 			AST rhs = POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices);
 			return new BinaryExpressionAst(result, op, txt, rhs);
 		}
 		return result;
 	}

 	// according to the spec, pre/post inc can occur
 	// only on lvalues, which are NAMES (IDs), array,
 	// or field references
 	private boolean isLvalue(AST ast) {
 		return (ast instanceof IDAst) || (ast instanceof ArrayReferenceAst) || (ast instanceof DollarExpressionAst);
 	}

 	AST FACTOR_FOR_INCDEC(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
 			throws IOException {
 		boolean preInc = false;
 		boolean preDec = false;
 		boolean postInc = false;
 		boolean postDec = false;
 		if (token == Token.INC) {
 			preInc = true;
 			lexer();
 		} else if (token == Token.DEC) {
 			preDec = true;
 			lexer();
 		}

 		AST factorAst = FACTOR(allowComparison, allowInKeyword, allowMultidimIndices);

 		if ((preInc || preDec) && !isLvalue(factorAst)) {
 			throw parserException("Cannot pre inc/dec a non-lvalue");
 		}

 		// only do post ops if:
 		// - factorAst is an lvalue
 		// - pre ops were not encountered
 		if (isLvalue(factorAst) && !preInc && !preDec) {
 			if (token == Token.INC) {
 				postInc = true;
 				lexer();
 			} else if (token == Token.DEC) {
 				postDec = true;
 				lexer();
 			}
 		}

 		if ((preInc || preDec) && (postInc || postDec)) {
 			throw parserException("Cannot do pre inc/dec Token.AND post inc/dec.");
 		}

 		if (preInc) {
 			return new PreIncAst(factorAst);
 		} else if (preDec) {
 			return new PreDecAst(factorAst);
 		} else if (postInc) {
 			return new PostIncAst(factorAst);
 		} else if (postDec) {
 			return new PostDecAst(factorAst);
 		} else {
 			return factorAst;
 		}
 	}

 	// FACTOR : '(' ASSIGNMENT_EXPRESSION ')' | Token.INTEGER | Token.DOUBLE | Token.STRING | GETLINE
 	// [Token.ID-or-array-or-$val] | /[=].../
 	// | [++|--] SYMBOL [++|--]
 	// AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allow_post_incdec_operators)
 	AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices) throws IOException {
 		if (token == Token.OPEN_PAREN) {
 			lexer();
 			// true = allow multi-dimensional array indices (i.e., commas for 1,2,3,4)
 			AST assignmentExpression = ASSIGNMENT_EXPRESSION(null, true, allowInKeyword, true);
 			lexer(Token.CLOSE_PAREN);
 			if (assignmentExpression instanceof ArrayIndexAst && !(allowInKeyword && token == Token.KW_IN)) {
 				// (expr, expr, ...) is a grouping, not an expression: it is only valid
 				// immediately before "in", as in ((i, j) in array). This also rejects
 				// nested groupings such as (1, (2, 3)), whose inner group is followed
 				// by ')' rather than "in".
 				throw parserException("A parenthesized expression list is only valid before 'in'.");
 			}
 			return assignmentExpression;
 		} else if (token == Token.INTEGER) {
 			AST integer = symbolTable.addINTEGER(text.toString());
 			lexer();
 			return integer;
 		} else if (token == Token.DOUBLE) {
 			AST dbl = symbolTable.addDOUBLE(text.toString());
 			lexer();
 			return dbl;
 		} else if (token == Token.STRING) {
 			AST str = symbolTable.addSTRING(string.toString());
 			lexer();
 			return str;
 		} else if (token == Token.INDIRECT) {
 			return INDIRECT_FUNCTION_CALL(allowInKeyword);
 		} else if (token == Token.TYPED_REGEXP) {
 			AST regexpAst = symbolTable.addTYPED_REGEXP(regexp.toString());
 			lexer();
 			return regexpAst;
 		} else if (token == Token.KW_GETLINE) {
 			return GETLINE_EXPRESSION(null, allowComparison, allowInKeyword);
 		} else if (token == Token.DIVIDE || token == Token.DIV_EQ) {
 			readRegexp();
 			if (token == Token.DIV_EQ) {
 				regexp.insert(0, '=');
 			}
 			AST regexpAst = symbolTable.addREGEXP(regexp.toString());
 			lexer();
 			return regexpAst;
 		} else {
 			if (token == Token.DOLLAR) {
 				lexer();
 				if (token == Token.INC || token == Token.DEC) {
 					return new DollarExpressionAst(
 							FACTOR_FOR_INCDEC(allowComparison, allowInKeyword, allowMultidimIndices));
 				}
 				if (token == Token.NOT || token == Token.MINUS || token == Token.PLUS) {
 					return new DollarExpressionAst(UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices));
 				}
 				return new DollarExpressionAst(FACTOR(allowComparison, allowInKeyword, allowMultidimIndices));
 			}
 			return SYMBOL(allowComparison, allowInKeyword);
 		}
 	}

 	private AST INDIRECT_FUNCTION_CALL(boolean allowInKeyword) throws IOException {
 		lexer();
 		if (token != Token.ID) {
 			throw parserException("An indirect function call requires a variable name after @.");
 		}
 		AST functionNameAst = symbolTable.getID(text.toString());
 		lexer();
 		lexer(Token.OPEN_PAREN);
 		AST params = token == Token.CLOSE_PAREN ? null : EXPRESSION_LIST(true, allowInKeyword);
 		lexer(Token.CLOSE_PAREN);
 		return new IndirectFunctionCallAst(functionNameAst, params);
 	}

 	// SYMBOL : Token.ID [ '(' params ')' | '[' ASSIGNMENT_EXPRESSION ']' ]
 	AST SYMBOL(boolean allowComparison, boolean allowInKeyword) throws IOException {
 		if (token != Token.ID && token != Token.FUNC_ID && token != Token.BUILTIN_FUNC_NAME && token != Token.EXTENSION) {
 			throw parserException("Expecting an Token.ID. Got " + token.name() + ": " + text);
 		}
 		Token idToken = token;
 		String id = text.toString();
 		lexer();

 		if (idToken == Token.EXTENSION) {
 			String extensionKeyword = id;
 			ExtensionFunction function = extensions.get(extensionKeyword);
 			if (function == null) {
 				throw parserException("Unknown extension keyword: " + extensionKeyword);
 			}
 			AST params;

 			/*
 			 * if (extension.requiresParen()) {
 			 * lexer(Token.OPEN_PAREN);
 			 * if (token == Token.CLOSE_PAREN)
 			 * params = null;
 			 * else
 			 * params = EXPRESSION_LIST(allowComparison, allowInKeyword);
 			 * lexer(Token.CLOSE_PAREN);
 			 * } else {
 			 * boolean parens = c == '(';
 			 * //expectKeyword("delete");
 			 * if (parens) {
 			 * assert token == Token.OPEN_PAREN;
 			 * lexer();
 			 * }
 			 * //AST symbolAst = SYMBOL(true,true); // allow comparators
 			 * params = EXPRESSION_LIST(allowComparison, allowInKeyword);
 			 * if (parens)
 			 * lexer(Token.CLOSE_PAREN);
 			 * }
 			 */

 			// like the built-in functions (and gawk's own builtins, which is
 			// what these keywords stand in for), an extension call accepts
 			// whitespace between the keyword and its argument list
 			if (token == Token.OPEN_PAREN) {
 				lexer();
 				if (token == Token.CLOSE_PAREN) {
 					params = null;
 				} else { // comparators allowed, allow “in” inside the extension call
 					params = EXPRESSION_LIST(true, allowInKeyword);
 				}
 				lexer(Token.CLOSE_PAREN);
 			} else {
 				/*
 				 * if (token == Token.NEWLINE || token == Token.SEMICOLON || token == Token.CLOSE_BRACE || token ==
 				 * Token.CLOSE_PAREN
 				 * || (token == Token.GT || token == Token.APPEND || token == Token.PIPE) )
 				 * params = null;
 				 * else
 				 * params = EXPRESSION_LIST(false,true);
 				 */
 				params = null;
 			}

 			return new ExtensionAst(function, params, extensionCallLineNumber(params));
 		} else if (idToken == Token.FUNC_ID || idToken == Token.BUILTIN_FUNC_NAME) {
 			AST params;
 			// length can take on the special form of no parens
 			if (id.equals("length")) {
 				if (token == Token.OPEN_PAREN) {
 					lexer();
 					if (token == Token.CLOSE_PAREN) {
 						params = null;
 					} else {
 						params = EXPRESSION_LIST(true, allowInKeyword);
 					}
 					lexer(Token.CLOSE_PAREN);
 				} else {
 					params = null;
 				}
 			} else {
 				lexer(Token.OPEN_PAREN);
 				if (token == Token.CLOSE_PAREN) {
 					params = null;
 				} else {
 					params = EXPRESSION_LIST(true, allowInKeyword);
 				}
 				lexer(Token.CLOSE_PAREN);
 			}
 			if (idToken == Token.BUILTIN_FUNC_NAME) {
 				return new BuiltinFunctionCallAst(id, params);
 			} else {
 				return symbolTable.addFunctionCall(id, params);
 			}
 		}
 		if (token == Token.OPEN_BRACKET) {
 			int arrayReferenceLineNo = currentSourceLineNumber();
 			lexer();
 			AST idxAst = ARRAY_INDEX(true, allowInKeyword);
 			lexer(Token.CLOSE_BRACKET);
 			AST arrayReference = symbolTable.addArrayReference(id, idxAst, arrayReferenceLineNo);
 			if (posix && token == Token.OPEN_BRACKET) {
 				throw parserException("Use [a,b,c,...] instead of [a][b][c]... for multi-dimensional arrays.");
 			}
 			while (!posix && token == Token.OPEN_BRACKET) {
 				int nestedArrayReferenceLineNo = currentSourceLineNumber();
 				lexer();
 				idxAst = ARRAY_INDEX(true, allowInKeyword);
 				lexer(Token.CLOSE_BRACKET);
 				arrayReference = new ArrayReferenceAst(nestedArrayReferenceLineNo, arrayReference, idxAst);
 			}
 			return arrayReference;
 		}
 		return symbolTable.addID(id);
 	}

 	// ARRAY_INDEX : ASSIGNMENT_EXPRESSION [, ARRAY_INDEX]
 	AST ARRAY_INDEX(boolean allowComparison, boolean allowInKeyword) throws IOException {
 		// Capture the line before parsing the expression: the expression AST can
 		// be a shared identifier node carrying its first occurrence's line, and
 		// by the end of the expression the lexer may have read past the line.
 		int lineNo = currentSourceLineNumber();
 		AST exprAst = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, false);
 		if (token == Token.COMMA) {
 			optNewline();
 			lexer();
 			return new ArrayIndexAst(lineNo, exprAst, ARRAY_INDEX(allowComparison, allowInKeyword));
 		} else {
 			return new ArrayIndexAst(lineNo, exprAst, null);
 		}
 	}

 	// STATEMENT :
 	// IF_STATEMENT
 	// | WHILE_STATEMENT
 	// | FOR_STATEMENT
 	// | DO_STATEMENT
 	// | RETURN_STATEMENT
 	// | ASSIGNMENT_EXPRESSION
 	AST STATEMENT() throws IOException {
 		if (token == Token.OPEN_BRACE) {
 			lexer();
 			AST lst = STATEMENT_LIST();
 			lexer(Token.CLOSE_BRACE);
 			return lst;
 		}
 		AST stmt;
 		if (token == Token.KW_IF) {
 			stmt = IF_STATEMENT();
 		} else if (token == Token.KW_WHILE) {
 			stmt = WHILE_STATEMENT();
 		} else if (token == Token.KW_FOR) {
 			stmt = FOR_STATEMENT();
 		} else {
 			if (token == Token.KW_DO) {
 				stmt = DO_STATEMENT();
 			} else if (token == Token.KW_RETURN) {
 				stmt = RETURN_STATEMENT();
 			} else if (token == Token.KW_EXIT) {
 				stmt = EXIT_STATEMENT();
 			} else if (token == Token.KW_DELETE) {
 				stmt = DELETE_STATEMENT();
 			} else if (token == Token.KW_PRINT) {
 				stmt = PRINT_STATEMENT();
 			} else if (token == Token.KW_PRINTF) {
 				stmt = PRINTF_STATEMENT();
 			} else if (token == Token.KW_NEXT) {
 				stmt = NEXT_STATEMENT();
 			} else if (token == Token.KW_NEXTFILE) {
 				stmt = NEXTFILE_STATEMENT();
 			} else if (token == Token.KW_CONTINUE) {
 				stmt = CONTINUE_STATEMENT();
 			} else if (token == Token.KW_BREAK) {
 				stmt = BREAK_STATEMENT();
 			} else {
 				stmt = EXPRESSION_STATEMENT(true);
 			}
 			terminator();
 			return stmt;
 		}
 		// NO TERMINATOR FOR IF, WHILE, Token.AND FOR
 		// (leave it for absorption by the callee)
 		return stmt;
 	}

 	AST EXPRESSION_STATEMENT(boolean allowInKeyword) throws IOException {
 		// true = allow comparators
 		// false = do Token.NOT allow multi-dimensional array indices
 		// return new ExpressionStatementAst(ASSIGNMENT_EXPRESSION(true, allowInKeyword, false));

 		AST exprAst = ASSIGNMENT_EXPRESSION(null, true, allowInKeyword, false);
 		return new ExpressionStatementAst(exprAst);
 	}

 	AST IF_STATEMENT() throws IOException {
 		expectKeyword("if");
 		lexer(Token.OPEN_PAREN);
 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow in keyword, do Token.NOT
 																																// allow
 		// multidim
 		// indices expressions
 		lexer(Token.CLOSE_PAREN);

 		//// Was:
 		//// AST b1 = BLOCK_OR_STMT();
 		//// But it didn't handle
 		//// if ; else ...
 		//// properly
 		optNewline();
 		AST b1;
 		if (token == Token.SEMICOLON) {
 			lexer();
 			// consume the newline after the semicolon
 			optNewline();
 			b1 = null;
 		} else {
 			b1 = BLOCK_OR_STMT();
 		}

 		// The OPT_NEWLINE() above causes issues with the following form:
 		// if (...) {
 		// }
 		// else { ... }
 		// The \n before the else disassociates subsequent statements
 		// if an "else" does not immediately follow.
 		// To accommodate, the ifStatement will continue to manage
 		// statements, causing the original OPT_STATEMENT_LIST to relinquish
 		// processing statements to this OPT_STATEMENT_LIST.

 		optNewline();
 		if (token == Token.KW_ELSE) {
 			lexer();
 			optNewline();
 			AST b2 = BLOCK_OR_STMT();
 			return new IfStatementAst(expr, b1, b2);
 		} else {
 			AST ifAst = new IfStatementAst(expr, b1, null);
 			return ifAst;
 		}
 	}

 	AST BREAK_STATEMENT() throws IOException {
 		expectKeyword("break");
 		return new BreakStatementAst();
 	}

 	AST BLOCK_OR_STMT() throws IOException {
 		// default case, does Token.NOT consume (require) a terminator
 		return BLOCK_OR_STMT(false);
 	}

 	AST BLOCK_OR_STMT(boolean requireTerminator) throws IOException {
 		optNewline();
 		AST block;
 		// HIJACK BRACES HERE SINCE WE MAY Token.NOT HAVE A TERMINATOR AFTER THE CLOSING BRACE
 		if (token == Token.OPEN_BRACE) {
 			lexer();
 			block = STATEMENT_LIST();
 			lexer(Token.CLOSE_BRACE);
 			return block;
 		} else if (token == Token.SEMICOLON) {
 			block = null;
 		} else {
 			block = STATEMENT();
 			// NO TERMINATOR HERE!
 		}
 		if (requireTerminator) {
 			terminator();
 		}
 		return block;
 	}

 	AST WHILE_STATEMENT() throws IOException {
 		expectKeyword("while");
 		lexer(Token.OPEN_PAREN);
 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow IN keyword, do Token.NOT
 																																// allow
 		// multidim
 		// indices expressions
 		lexer(Token.CLOSE_PAREN);
 		AST block = BLOCK_OR_STMT();
 		return new WhileStatementAst(expr, block);
 	}

 	AST FOR_STATEMENT() throws IOException {
 		expectKeyword("for");
 		AST expr1 = null;
 		AST expr2 = null;
 		AST expr3 = null;
 		lexer(Token.OPEN_PAREN);
 		expr1 = OPT_SIMPLE_STATEMENT(false); // false = "no in keyword allowed"

 		// branch here if we expect a for(... in ...) statement
 		if (token == Token.KW_IN) {
 			if (expr1.ast1 == null || expr1.ast2 != null) {
 				throw parserException("Invalid expression prior to 'in' statement. Got : " + expr1);
 			}
 			expr1 = expr1.ast1;
 			// analyze expr1 to make sure it's a singleton IDAst
 			if (!(expr1 instanceof IDAst)) {
 				throw parserException("Expecting an Token.ID for 'in' statement. Got : " + expr1);
 			}
 			// in
 			lexer();
 			if (token != Token.ID) {
 				throw parserException(
 						"Expecting an array or subarray for 'in' statement. Got " + token.name() + ": " + text);
 			}
 			AST arrayAst = SYMBOL(true, true);
 			// close paren ...
 			lexer(Token.CLOSE_PAREN);
 			AST block = BLOCK_OR_STMT();
 			return new ForInStatementAst(expr1, arrayAst, block);
 		}

 		if (token == Token.SEMICOLON) {
 			lexer();
 			optNewline();
 		} else {
 			throw parserException("Expecting ;. Got " + token.name() + ": " + text);
 		}
 		if (token != Token.SEMICOLON) {
 			expr2 = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow IN keyword, do Token.NOT allow
 			// multidim
 			// indices expressions
 		}
 		if (token == Token.SEMICOLON) {
 			lexer();
 			optNewline();
 		} else {
 			throw parserException("Expecting ;. Got " + token.name() + ": " + text);
 		}
 		if (token != Token.CLOSE_PAREN) {
 			expr3 = OPT_SIMPLE_STATEMENT(true); // true = "allow the in keyword"
 		}
 		lexer(Token.CLOSE_PAREN);
 		AST block = BLOCK_OR_STMT();
 		return new ForStatementAst(expr1, expr2, expr3, block);
 	}

 	AST OPT_SIMPLE_STATEMENT(boolean allowInKeyword) throws IOException {
 		if (token == Token.SEMICOLON) {
 			return null;
 		} else if (token == Token.KW_DELETE) {
 			return DELETE_STATEMENT();
 		} else if (token == Token.KW_PRINT) {
 			return PRINT_STATEMENT();
 		} else if (token == Token.KW_PRINTF) {
 			return PRINTF_STATEMENT();
 		} else {
 			// allow non-statement ASTs
 			return EXPRESSION_STATEMENT(allowInKeyword);
 		}
 	}

 	AST DELETE_STATEMENT() throws IOException {
 		boolean parens = c == '(';
 		expectKeyword("delete");
 		if (parens) {
 			lexer();
 		}
 		AST symbolAst = SYMBOL(true, true); // allow comparators
 		if (parens) {
 			lexer(Token.CLOSE_PAREN);
 		}

 		return new DeleteStatementAst(symbolAst);
 	}

 	private static final class ParsedPrintStatement {

 		private final AST funcParams;
 		private final Token outputToken;
 		private final AST outputExpr;
 		private final boolean parenthesized;

 		ParsedPrintStatement(AST funcParams, Token outputToken, AST outputExpr, boolean parenthesized) {
 			this.funcParams = funcParams;
 			this.outputToken = outputToken;
 			this.outputExpr = outputExpr;
 			this.parenthesized = parenthesized;
 		}

 		public AST getFuncParams() {
 			return funcParams;
 		}

 		public Token getOutputToken() {
 			return outputToken;
 		}

 		public AST getOutputExpr() {
 			return outputExpr;
 		}

 		public boolean isParenthesized() {
 			return parenthesized;
 		}
 	}

 	private ParsedPrintStatement parsePrintStatement() throws IOException {
 		AST funcParams;
 		Token outputToken;
 		AST outputExpr;
 		boolean parenthesized = false;

 		if (token == Token.OPEN_PAREN) {
 			parenthesized = true;
 			funcParams = parseParenthesizedPrintArguments();
 		} else if (endsPrintArgumentList(token)) {
 			funcParams = null;
 		} else {
 			funcParams = EXPRESSION_LIST(false, true); // no comparisons allowed, but allow “in”
 		}

 		if (token == Token.GT || token == Token.APPEND || token == Token.PIPE) {
 			outputToken = token;
 			lexer();
 			outputExpr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparisons, “in”; no multidim indices
 		} else {
 			outputToken = null;
 			outputExpr = null;
 		}

 		return new ParsedPrintStatement(funcParams, outputToken, outputExpr, parenthesized);
 	}

 	private boolean endsPrintArgumentList(Token candidate) {
 		return candidate == Token.NEWLINE
 				|| candidate == Token.SEMICOLON
 				|| candidate == Token.CLOSE_BRACE
 				|| candidate == Token.CLOSE_PAREN
 				|| candidate == Token.GT
 				|| candidate == Token.APPEND
 				|| candidate == Token.PIPE
 				|| candidate == Token.EOF;
 	}

 	private AST parseParenthesizedPrintArguments() throws IOException {
 		lexer(); // consume '('
 		if (token == Token.CLOSE_PAREN) {
 			lexer(); // consume ')'
 			return null;
 		}

 		AST params = EXPRESSION_LIST(true, true); // allow comparisons and “in” within parentheses
 		lexer(Token.CLOSE_PAREN);

 		if (params instanceof FunctionCallParamListAst) {
 			FunctionCallParamListAst paramList = (FunctionCallParamListAst) params;
 			boolean singleExpression = paramList.getAst2() == null;
 			// A parenthesized group followed by "in" is a membership test whose key is
 			// the whole group, e.g.: print (1,2) in a
 			boolean membershipKey = !singleExpression && token == Token.KW_IN;
 			if ((singleExpression || membershipKey) && !endsPrintArgumentList(token)) {
 				AST continuedExpression = ASSIGNMENT_EXPRESSION(
 						singleExpression ? paramList.getAst1() : toMultidimIndex(paramList),
 						false,
 						true,
 						false);
 				if (token == Token.COMMA) {
 					// A single parenthesized expression followed by a comma continues the
 					// output expression list, e.g.: print (i==0), (i=="")
 					lexer(); // consume ','
 					optNewline(); // allow newline after comma (AWK style)
 					return new FunctionCallParamListAst(continuedExpression, EXPRESSION_LIST(false, true));
 				}
 				return new FunctionCallParamListAst(continuedExpression, null);
 			}
 		}

 		return params;
 	}

 	// Converts a print argument list back into the multi-dimensional array index
 	// it turned out to be, when the parenthesized group is followed by "in"
 	private AST toMultidimIndex(FunctionCallParamListAst list) {
 		AST rest = list.getAst2() == null ? null : toMultidimIndex((FunctionCallParamListAst) list.getAst2());
 		return new ArrayIndexAst(list.getLineNo(), list.getAst1(), rest);
 	}

 	AST PRINT_STATEMENT() throws IOException {
 		expectKeyword("print");
 		ParsedPrintStatement parsedPrintStatement = parsePrintStatement();

 		AST params = parsedPrintStatement.getFuncParams();
 		if (parsedPrintStatement.isParenthesized()
 				&& token == Token.QUESTION_MARK
 				&& params instanceof FunctionCallParamListAst
 				&& ((FunctionCallParamListAst) params).getAst2() == null) {
 			AST condExpr = ((FunctionCallParamListAst) params).getAst1();
 			lexer();
 			AST trueBlock = TERNARY_EXPRESSION(null, true, true, true);
 			lexer(Token.COLON);
 			AST falseBlock = TERNARY_EXPRESSION(null, true, true, true);
 			params = new FunctionCallParamListAst(
 					new TernaryExpressionAst(condExpr, trueBlock, falseBlock),
 					null);
 		}

 		return new PrintAst(
 				params,
 				parsedPrintStatement.getOutputToken(),
 				parsedPrintStatement.getOutputExpr(),
 				parsedPrintStatement.isParenthesized());
 	}

 	AST PRINTF_STATEMENT() throws IOException {
 		expectKeyword("printf");
 		ParsedPrintStatement parsedPrintStatement = parsePrintStatement();

 		AST params = parsedPrintStatement.getFuncParams();
 		if (parsedPrintStatement.isParenthesized()
 				&& token == Token.QUESTION_MARK
 				&& params instanceof FunctionCallParamListAst
 				&& ((FunctionCallParamListAst) params).getAst2() == null) {
 			AST condExpr = ((FunctionCallParamListAst) params).getAst1();
 			lexer();
 			AST trueBlock = TERNARY_EXPRESSION(null, true, true, true);
 			lexer(Token.COLON);
 			AST falseBlock = TERNARY_EXPRESSION(null, true, true, true);
 			params = new FunctionCallParamListAst(
 					new TernaryExpressionAst(condExpr, trueBlock, falseBlock),
 					null);
 		}

 		return new PrintfAst(
 				params,
 				parsedPrintStatement.getOutputToken(),
 				parsedPrintStatement.getOutputExpr());
 	}

 	AST GETLINE_EXPRESSION(AST pipeExpr, boolean allowComparison, boolean allowInKeyword) throws IOException {
 		expectKeyword("getline");
 		AST lvalue = LVALUE(allowComparison, allowInKeyword);
 		if (token == Token.LT) {
 			lexer();
 			AST assignmentExpr = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, false); // do Token.NOT allow
 																																																// multidim
 			// indices expressions
 			return pipeExpr == null ?
 					new GetlineAst(null, lvalue, assignmentExpr) : new GetlineAst(pipeExpr, lvalue, assignmentExpr);
 		} else {
 			return pipeExpr == null ? new GetlineAst(null, lvalue, null) : new GetlineAst(pipeExpr, lvalue, null);
 		}
 	}

 	AST LVALUE(boolean allowComparison, boolean allowInKeyword) throws IOException {
 		// false = do Token.NOT allow multi dimension indices expressions
 		if (token == Token.DOLLAR) {
 			return FACTOR(allowComparison, allowInKeyword, false);
 		}
 		if (token == Token.ID) {
 			return FACTOR(allowComparison, allowInKeyword, false);
 		}
 		return null;
 	}

 	AST DO_STATEMENT() throws IOException {
 		expectKeyword("do");
 		optNewline();
 		AST block = BLOCK_OR_STMT();
 		if (token == Token.SEMICOLON) {
 			lexer();
 		}
 		optNewline();
 		expectKeyword("while");
 		lexer(Token.OPEN_PAREN);
 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // true = allow comparators, allow IN keyword, do
 																																// Token.NOT
 		// allow
 		// multidim indices expressions
 		lexer(Token.CLOSE_PAREN);
 		return new DoStatementAst(block, expr);
 	}

 	AST RETURN_STATEMENT() throws IOException {
 		expectKeyword("return");
 		if (token == Token.SEMICOLON || token == Token.NEWLINE || token == Token.CLOSE_BRACE) {
 			return new ReturnStatementAst(null);
 		} else {
 			return new ReturnStatementAst(ASSIGNMENT_EXPRESSION(null, true, true, false)); // true = allow comparators, allow
 																																											// IN
 			// keyword, do Token.NOT allow multidim
 			// indices expressions
 		}
 	}

 	AST EXIT_STATEMENT() throws IOException {
 		expectKeyword("exit");
 		if (token == Token.SEMICOLON || token == Token.NEWLINE || token == Token.CLOSE_BRACE) {
 			return new ExitStatementAst(null);
 		} else {
 			return new ExitStatementAst(ASSIGNMENT_EXPRESSION(null, true, true, false)); // true = allow comparators, allow IN
 			// keyword, do Token.NOT allow multidim
 			// indices
 			// expressions
 		}
 	}

 	AST NEXT_STATEMENT() throws IOException {
 		expectKeyword("next");
 		return new NextStatementAst();
 	}

 	AST NEXTFILE_STATEMENT() throws IOException {
 		expectKeyword("nextfile");
 		nextfileEncountered = true;
 		return new NextfileStatementAst();
 	}

 	AST CONTINUE_STATEMENT() throws IOException {
 		expectKeyword("continue");
 		return new ContinueStatementAst();
 	}

 	// CHECKSTYLE.ON MethodName

 	private void expectKeyword(String keyword) throws IOException {
 		if (token == KEYWORDS.get(keyword)) {
 			lexer();
 		} else {
 			throw parserException("Expecting " + keyword + ". Got " + token.name() + ": " + text);
 		}
 	}

 	private void populateArrayOperandTuples(
 			AST arrayAst,
 			AwkTuples tuples,
 			boolean createIfMissing,
 			String errorMessage) {
 		if (arrayAst instanceof IDAst) {
 			IDAst idAst = (IDAst) arrayAst;
 			idAst.setArray(true);
 			if (isJrtManagedSpecialName(idAst.id)) {
 				idAst.populateTuples(tuples);
 			} else {
 				tuples.dereference(idAst.offset, true, idAst.isGlobal);
 			}
 			return;
 		}
 		if (arrayAst instanceof ArrayReferenceAst) {
 			if (posix) {
 				arrayAst.throwSemanticException(errorMessage);
 			}
 			((ArrayReferenceAst) arrayAst).populateArrayValueTuples(tuples, createIfMissing);
 			return;
 		}
 		arrayAst.throwSemanticException(errorMessage);
 	}

 	private int populateActualParameters(
 			AwkTuples tuples,
 			FunctionCallParamListAst params,
 			Set<Integer> arrayParameterIndexes,
 			Set<Integer> rawValueParameterIndexes,
 			Set<Integer> literalRegexpIndexes,
 			int parameterIndex) {
 		if (params == null) {
 			return 0;
 		}
 		if (arrayParameterIndexes.contains(Integer.valueOf(parameterIndex))) {
 			populateArrayOperandTuples(
 					params.getAst1(),
 					tuples,
 					true,
 					"Parameter position " + (parameterIndex + 1) + " must be an array or subarray.");
 		} else if (literalRegexpIndexes.contains(Integer.valueOf(parameterIndex))) {
 			populateRawRegexpParameterTuples(params.getAst1(), tuples);
 		} else if (rawValueParameterIndexes.contains(Integer.valueOf(parameterIndex))) {
 			populateRawValueTuples(params.getAst1(), tuples);
 		} else {
 			params.getAst1().populateTuples(tuples);
 		}
 		if (params.getAst2() == null) {
 			return 1;
 		}
 		return 1 + populateActualParameters(
 				tuples,
 				(FunctionCallParamListAst) params.getAst2(),
 				arrayParameterIndexes,
 				rawValueParameterIndexes,
 				literalRegexpIndexes,
 				parameterIndex + 1);
 	}

 	private int populateActualParameters(
 			AwkTuples tuples,
 			FunctionCallParamListAst params,
 			int... literalRegexpIndexesParam) {
 		Set<Integer> literalRegexpIndexes = new HashSet<Integer>();
 		for (int idx : literalRegexpIndexesParam) {
 			literalRegexpIndexes.add(Integer.valueOf(idx));
 		}
 		return populateActualParameters(
 				tuples,
 				params,
 				Collections.<Integer>emptySet(),
 				Collections.<Integer>emptySet(),
 				literalRegexpIndexes,
 				0);
 	}

 	private int populateActualParametersUpTo(
 			AwkTuples tuples,
 			FunctionCallParamListAst params,
 			int parameterIndex,
 			int maxParameterCount) {
 		/*
 		 * Gawk accepts extra user-function arguments with a runtime warning: the
 		 * callee has no local slots for them, but their expressions are still
 		 * evaluated for their side effects. Extra arguments are therefore emitted
 		 * followed by a POP, and only the formal parameter prefix is counted.
 		 */
 		if (params == null) {
 			return 0;
 		}
 		if (parameterIndex >= maxParameterCount) {
 			// Raw-value evaluation runs the expression's side effects but merely
 			// peeks at bare variables, so an untyped variable passed as an extra
 			// argument is not autovivified into an assigned scalar.
 			populateRawValueTuples(params.getAst1(), tuples);
 			tuples.pop();
 			populateActualParametersUpTo(
 					tuples,
 					(FunctionCallParamListAst) params.getAst2(),
 					parameterIndex + 1,
 					maxParameterCount);
 			return 0;
 		}
 		AST argument = params.getAst1();
 		if (argument instanceof IDAst
 				&& !isJrtManagedSpecialName(((IDAst) argument).id)) {
 			IDAst idAst = (IDAst) argument;
 			tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
 		} else if (argument instanceof ArrayReferenceAst) {
 			((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples);
 			tuples.pushIndirectArrayArgument();
 		} else {
 			argument.populateTuples(tuples);
 		}
 		if (params.getAst2() == null) {
 			return 1;
 		}
 		return 1 + populateActualParametersUpTo(
 				tuples,
 				(FunctionCallParamListAst) params.getAst2(),
 				parameterIndex + 1,
 				maxParameterCount);
 	}

 	private void populateRawValueTuples(AST valueAst, AwkTuples tuples) {
 		/*
 		 * typeof() and isarray() need to inspect an lvalue's current state. A
 		 * normal scalar dereference autoconverts an untyped variable into AWK's
 		 * assigned blank scalar, which would erase the distinction gawk exposes,
 		 * so scalar variables use a non-assigning peek instead.
 		 * Array elements need no special opcode: a normal element read already
 		 * returns the raw untyped marker for a missing element, and, as in gawk,
 		 * brings that element into existence so a later `in`, delete, or for-in
 		 * observes it.
 		 */
 		if (valueAst instanceof IDAst) {
 			IDAst idAst = (IDAst) valueAst;
 			if (isJrtManagedSpecialName(idAst.id)) {
 				idAst.populateTuples(tuples);
 			} else {
 				tuples.peekDereference(idAst.offset, idAst.isGlobal);
 			}
 			return;
 		}
 		valueAst.populateTuples(tuples);
 	}

 	private void populateRawRegexpParameterTuples(AST valueAst, AwkTuples tuples) {
 		if (valueAst instanceof RegexpAst) {
 			((RegexpAst) valueAst).populateRawRegexpTuples(tuples);
 			return;
 		}
 		valueAst.populateTuples(tuples);
 	}

 	private int populateIndirectActualParameters(
 			AwkTuples tuples,
 			FunctionCallParamListAst params) {
 		if (params == null) {
 			return 0;
 		}
 		AST argument = params.getAst1();
 		if (argument instanceof IDAst
 				&& !isJrtManagedSpecialName(((IDAst) argument).id)) {
 			IDAst idAst = (IDAst) argument;
 			tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
 		} else if (argument instanceof ArrayReferenceAst) {
 			((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples);
 			tuples.pushIndirectArrayArgument();
 		} else {
 			argument.populateTuples(tuples);
 		}
 		return 1 + populateIndirectActualParameters(
 				tuples,
 				(FunctionCallParamListAst) params.getAst2());
 	}

 	private Set<Integer> collectArrayParameterIndexes(FunctionDefAst functionDefAst) {
 		Set<Integer> arrayIndexes = new HashSet<Integer>();
 		FunctionDefParamListAst fPtr = (FunctionDefParamListAst) functionDefAst.getAst1();
 		int index = 0;
 		while (fPtr != null) {
 			IDAst fparam = symbolTable.getFunctionParameterIDAST(functionDefAst.id, fPtr.id);
 			if (fparam.isArray()) {
 				arrayIndexes.add(Integer.valueOf(index));
 			}
 			fPtr = (FunctionDefParamListAst) fPtr.getAst1();
 			index++;
 		}
 		return arrayIndexes;
 	}

 	// parser
 	// ===============================================================================
 	// AST class defs
 	private abstract class AST extends AstNode {

 		private final String sourceDescription = currentScriptSource.getDescription();
 		// PositionTracker consumes these tuple-emitted source lines at runtime, but
 		// AST nodes have to capture them here during parsing before tuples exist.
 		private final int lineNo;
 		private AST parent;
 		private AST ast1, ast2, ast3, ast4;
 		private final EnumSet<AstFlag> flags = EnumSet.noneOf(AstFlag.class);

 		protected final void addFlag(AstFlag flag) {
 			flags.add(flag);
 		}

 		protected final boolean hasFlag(AstFlag flag) {
 			return flags.contains(flag);
 		}

 		protected Address breakAddress() {
 			return null;
 		}

 		protected Address continueAddress() {
 			return null;
 		}

 		protected Address nextAddress() {
 			return null;
 		}

 		protected Address returnAddress() {
 			return null;
 		}

 		protected final AST getParent() {
 			return parent;
 		}

 		@SuppressWarnings("unused")
 		protected final void setParent(AST p) {
 			parent = p;
 		}

 		protected final AST getAst1() {
 			return ast1;
 		}

 		@SuppressWarnings("unused")
 		protected final void setAst1(AST a1) {
 			ast1 = a1;
 		}

 		protected final AST getAst2() {
 			return ast2;
 		}

 		@SuppressWarnings("unused")
 		protected final void setAst2(AST a2) {
 			ast2 = a2;
 		}

 		protected final AST getAst3() {
 			return ast3;
 		}

 		@SuppressWarnings("unused")
 		protected final void setAst3(AST a3) {
 			ast3 = a3;
 		}

 		protected final AST getAst4() {
 			return ast4;
 		}

 		@SuppressWarnings("unused")
 		protected final void setAst4(AST a4) {
 			ast4 = a4;
 		}

 		protected final AST searchFor(AstFlag flag) {
 			AST ptr = this;
 			while (ptr != null) {
 				if (ptr.hasFlag(flag)) {
 					return ptr;
 				}
 				ptr = ptr.parent;
 			}
 			return null;
 		}

 		protected AST() {
 			this(currentSourceLineNumber());
 		}

 		protected AST(int lineNo) {
 			this.lineNo = lineNo;
 		}

 		protected int getLineNo() {
 			return lineNo;
 		}

 		protected String getSourceDescription() {
 			return sourceDescription;
 		}

 		protected String sourceBasename() {
 			// File.getName is a pure string operation and, unlike java.nio.Path,
 			// never rejects non-path descriptions such as <command-line-supplied-script>
 			return new File(getSourceDescription()).getName();
 		}

 		protected AST(AST ast1) {
 			this(currentSourceLineNumber(), ast1);
 		}

 		protected AST(int lineNo, AST ast1) {
 			this(lineNo);
 			this.ast1 = ast1;

 			if (ast1 != null) {
 				ast1.parent = this;
 			}
 		}

 		protected AST(AST ast1, AST ast2) {
 			this(currentSourceLineNumber(), ast1, ast2);
 		}

 		protected AST(int lineNo, AST ast1, AST ast2) {
 			this(lineNo);
 			this.ast1 = ast1;
 			this.ast2 = ast2;

 			if (ast1 != null) {
 				ast1.parent = this;
 			}
 			if (ast2 != null) {
 				ast2.parent = this;
 			}
 		}

 		protected AST(AST ast1, AST ast2, AST ast3) {
 			this(currentSourceLineNumber(), ast1, ast2, ast3);
 		}

 		protected AST(int lineNo, AST ast1, AST ast2, AST ast3) {
 			this(lineNo);
 			this.ast1 = ast1;
 			this.ast2 = ast2;
 			this.ast3 = ast3;

 			if (ast1 != null) {
 				ast1.parent = this;
 			}
 			if (ast2 != null) {
 				ast2.parent = this;
 			}
 			if (ast3 != null) {
 				ast3.parent = this;
 			}
 		}

 		protected AST(AST ast1, AST ast2, AST ast3, AST ast4) {
 			this(currentSourceLineNumber(), ast1, ast2, ast3, ast4);
 		}

 		protected AST(int lineNo, AST ast1, AST ast2, AST ast3, AST ast4) {
 			this(lineNo);
 			this.ast1 = ast1;
 			this.ast2 = ast2;
 			this.ast3 = ast3;
 			this.ast4 = ast4;

 			if (ast1 != null) {
 				ast1.parent = this;
 			}
 			if (ast2 != null) {
 				ast2.parent = this;
 			}
 			if (ast3 != null) {
 				ast3.parent = this;
 			}
 			if (ast4 != null) {
 				ast4.parent = this;
 			}
 		}

 		/**
 		 * Dump a meaningful text representation of this
 		 * abstract syntax tree node to the output (print)
 		 * stream. Either it is called directly by the
 		 * application program, or it is called by the
 		 * parent node of this tree node.
 		 *
 		 * @param ps The print stream to dump the text
 		 *        representation.
 		 */
 		@Override
 		public void dump(PrintStream ps) {
 			dump(ps, 0);
 		}

 		private void dump(PrintStream ps, int lvl) {
 			StringBuffer spaces = new StringBuffer();
 			for (int i = 0; i < lvl; i++) {
 				spaces.append(' ');
 			}
 			ps.println(spaces + toString());
 			if (ast1 != null) {
 				ast1.dump(ps, lvl + 1);
 			}
 			if (ast2 != null) {
 				ast2.dump(ps, lvl + 1);
 			}
 			if (ast3 != null) {
 				ast3.dump(ps, lvl + 1);
 			}
 			if (ast4 != null) {
 				ast4.dump(ps, lvl + 1);
 			}
 		}

 		/**
 		 * Apply semantic checks to this node. The default
 		 * implementation is to simply call semanticAnalysis()
 		 * on all the children of this abstract syntax tree node.
 		 * Therefore, this method must be overridden to provide
 		 * meaningful semantic analysis / checks.
 		 *
 		 * @throws SemanticException upon a semantic error.
 		 */
 		@Override
 		public void semanticAnalysis() {
 			if (ast1 != null) {
 				ast1.semanticAnalysis();
 			}
 			if (ast2 != null) {
 				ast2.semanticAnalysis();
 			}
 			if (ast3 != null) {
 				ast3.semanticAnalysis();
 			}
 			if (ast4 != null) {
 				ast4.semanticAnalysis();
 			}
 		}

 		/**
 		 * Appends tuples to the AwkTuples list
 		 * for this abstract syntax tree node. Subclasses
 		 * must implement this method.
 		 * <p>
 		 * This is called either by the main program to generate a full
 		 * list of tuples for the abstract syntax tree, or it is called
 		 * by other abstract syntax tree nodes in response to their
 		 * attempt at populating tuples.
 		 *
 		 * @param tuples The tuples to populate.
 		 * @return The number of items left on the stack after
 		 *         these tuples have executed.
 		 */
 		@Override
 		public abstract int populateTuples(AwkTuples tuples);

 		protected final void pushSourceLineNumber(AwkTuples tuples) {
 			tuples.pushSourceLineNumber(lineNo);
 		}

 		protected final void popSourceLineNumber(AwkTuples tuples) {
 			tuples.popSourceLineNumber(lineNo);
 		}

 		private boolean isBegin = isBegin();

 		@SuppressWarnings("unused")
 		protected final boolean isBeginFlag() {
 			return isBegin;
 		}

 		protected final void setBeginFlag(boolean flag) {
 			isBegin = flag;
 		}

 		private boolean isBegin() {
 			boolean result = isBegin;
 			if (!result && ast1 != null) {
 				result = ast1.isBegin();
 			}
 			if (!result && ast2 != null) {
 				result = ast2.isBegin();
 			}
 			if (!result && ast3 != null) {
 				result = ast3.isBegin();
 			}
 			if (!result && ast4 != null) {
 				result = ast4.isBegin();
 			}
 			return result;
 		}

 		private boolean isEnd = isEnd();

 		@SuppressWarnings("unused")
 		protected final boolean isEndFlag() {
 			return isEnd;
 		}

 		protected final void setEndFlag(boolean flag) {
 			isEnd = flag;
 		}

 		private boolean isEnd() {
 			boolean result = isEnd;
 			if (!result && ast1 != null) {
 				result = ast1.isEnd();
 			}
 			if (!result && ast2 != null) {
 				result = ast2.isEnd();
 			}
 			if (!result && ast3 != null) {
 				result = ast3.isEnd();
 			}
 			if (!result && getAst4() != null) {
 				result = getAst4().isEnd();
 			}
 			return result;
 		}

 		private boolean isBeginFile = isBeginFile();

 		protected final void setBeginFileFlag(boolean flag) {
 			isBeginFile = flag;
 		}

 		private boolean isBeginFile() {
 			boolean result = isBeginFile;
 			if (!result && ast1 != null) {
 				result = ast1.isBeginFile();
 			}
 			if (!result && ast2 != null) {
 				result = ast2.isBeginFile();
 			}
 			if (!result && ast3 != null) {
 				result = ast3.isBeginFile();
 			}
 			if (!result && ast4 != null) {
 				result = ast4.isBeginFile();
 			}
 			return result;
 		}

 		private boolean isEndFile = isEndFile();

 		protected final void setEndFileFlag(boolean flag) {
 			isEndFile = flag;
 		}

 		private boolean isEndFile() {
 			boolean result = isEndFile;
 			if (!result && ast1 != null) {
 				result = ast1.isEndFile();
 			}
 			if (!result && ast2 != null) {
 				result = ast2.isEndFile();
 			}
 			if (!result && ast3 != null) {
 				result = ast3.isEndFile();
 			}
 			if (!result && ast4 != null) {
 				result = ast4.isEndFile();
 			}
 			return result;
 		}

 		private boolean isFunction = isFunction();

 		@SuppressWarnings("unused")
 		protected final boolean isFunctionFlag() {
 			return isFunction;
 		}

 		protected final void setFunctionFlag(boolean flag) {
 			isFunction = flag;
 		}

 		private boolean isFunction() {
 			boolean result = isFunction;
 			if (!result && getAst1() != null) {
 				result = getAst1().isFunction();
 			}
 			if (!result && getAst2() != null) {
 				result = getAst2().isFunction();
 			}
 			if (!result && getAst3() != null) {
 				result = getAst3().isFunction();
 			}
 			if (!result && getAst4() != null) {
 				result = getAst4().isFunction();
 			}
 			return result;
 		}

 		public boolean isArray() {
 			return false;
 		}

 		public boolean isScalar() {
 			return false;
 		}

 		/**
 		 * Made protected so that subclasses can access it.
 		 * Package-level access was not necessary.
 		 */
 		protected class SemanticException extends RuntimeException {

 			private static final long serialVersionUID = 1L;

 			SemanticException(String msg) {
 				super(msg + " (" + sourceDescription + ":" + lineNo + ")");
 			}
 		}

 		protected final void throwSemanticException(String msg) {
 			throw new SemanticException(msg);
 		}

 		@Override
 		public String toString() {
 			return getClass().getName().replaceFirst(".*[$.]", "");
 		}
 	}

 	private abstract class ScalarExpressionAst extends AST {

 		protected ScalarExpressionAst() {
 			super();
 		}

 		protected ScalarExpressionAst(int lineNo) {
 			super(lineNo);
 		}

 		protected ScalarExpressionAst(AST a1) {
 			super(a1);
 		}

 		protected ScalarExpressionAst(int lineNo, AST a1) {
 			super(lineNo, a1);
 		}

 		protected ScalarExpressionAst(AST a1, AST a2) {
 			super(a1, a2);
 		}

 		protected ScalarExpressionAst(int lineNo, AST a1, AST a2) {
 			super(lineNo, a1, a2);
 		}

 		protected ScalarExpressionAst(AST a1, AST a2, AST a3) {
 			super(a1, a2, a3);
 		}

 		protected ScalarExpressionAst(int lineNo, AST a1, AST a2, AST a3) {
 			super(lineNo, a1, a2, a3);
 		}

 		@Override
 		public boolean isArray() {
 			return false;
 		}

 		@Override
 		public boolean isScalar() {
 			return true;
 		}
 	}

 	private static boolean isRule(AST ast) {
 		return ast != null
 				&& !ast.isBegin()
 				&& !ast.isEnd()
 				&& !ast.isBeginFile()
 				&& !ast.isEndFile()
 				&& !ast.isFunction();
 	}

 	/**
 	 * Inspects the action rule condition whether it contains
 	 * extensions. It does a superficial check of
 	 * the abstract syntax tree of the action rule.
 	 * In other words, it will not examine whether user-defined
 	 * functions within the action rule contain extensions.
 	 *
 	 * @param ast The action rule expression to examine.
 	 * @return true if the action rule condition contains
 	 *         an extension; false otherwise.
 	 */
 	@SuppressWarnings("unused")
 	private static boolean isExtensionConditionRule(AST ast) {
 		if (!isRule(ast)) {
 			return false;
 		}
 		if (ast.getAst1() == null) {
 			return false;
 		}

 		if (!containsASTType(ast.getAst1(), ExtensionAst.class)) {
 			return false;
 		}

 		if (containsASTType(ast.getAst1(), new Class[] { FunctionCallAst.class, DollarExpressionAst.class })) {
 			return false;
 		}

 		return true;
 	}

 	private static boolean containsASTType(AST ast, Class<?> cls) {
 		return containsASTType(ast, new Class[] { cls });
 	}

 	private static boolean containsASTType(AST ast, Class<?>[] clsArray) {
 		if (ast == null) {
 			return false;
 		}
 		for (Class<?> cls : clsArray) {
 			if (cls.isInstance(ast)) {
 				return true;
 			}
 		}
 		// prettier-ignore
 		return containsASTType(ast.getAst1(), clsArray)
 				|| containsASTType(ast.getAst2(), clsArray)
 				|| containsASTType(ast.getAst3(), clsArray)
 				|| containsASTType(ast.getAst4(), clsArray);
 	}

 	private Address nextAddress;

 	/**
 	 * Whether the program contains at least one {@code nextfile} statement,
 	 * anywhere (rules or user-defined functions). When set, the main input
 	 * loop is compiled with per-file stepping so the runtime can jump to the
 	 * ENDFILE section and advance to the next input file.
 	 */
 	private boolean nextfileEncountered;

 	private final class RuleListAst extends AST {

 		private RuleListAst(AST rule, AST rest) {
 			super(rule, rest);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {

 			pushSourceLineNumber(tuples);

 			nextAddress = tuples.createAddress("nextAddress");

 			// goto start address
 			Address startAddress = tuples.createAddress("start address");
 			tuples.gotoAddress(startAddress);

 			AST ptr;

 			// compile functions
 			ptr = this;
 			while (ptr != null) {
 				if (ptr.getAst1() != null && ptr.getAst1().isFunction()) {
 					ptr.getAst1().populateTuples(tuples);
 				}

 				ptr = ptr.getAst2();
 			}

 			// START OF MAIN BLOCK
 			tuples.address(startAddress);

 			// initialize runtime-managed special variables via JRT defaults in AVM
 			symbolTable.getID("NR");
 			symbolTable.getID("FNR");
 			symbolTable.getID("NF");
 			symbolTable.getID("FS");
 			symbolTable.getID("RS");
 			symbolTable.getID("OFS");
 			symbolTable.getID("ORS");
 			symbolTable.getID("RSTART");
 			symbolTable.getID("RLENGTH");
 			symbolTable.getID("FILENAME");
 			symbolTable.getID("SUBSEP");
 			symbolTable.getID("CONVFMT");
 			symbolTable.getID("OFMT");
 			IDAst environAst = symbolTable.getID("ENVIRON");
 			IDAst argcAst = symbolTable.getID("ARGC");
 			IDAst argvAst = symbolTable.getID("ARGV");

 			// MUST BE DONE AFTER FUNCTIONS ARE COMPILED,
 			// and after special variables are made known to the symbol table
 			// (see above)!
 			tuples.setNumGlobals(symbolTable.numGlobals());

 			// Only ENVIRON/ARGC/ARGV remain regular globals. ENVIRON and ARGV
 			// are materialized only when the script references them, or when
 			// SYMTAB is active (its snapshot exposes them); unreferenced ones
 			// are answered by the synthetic accessors. ARGC is always
 			// materialized (a single cheap assignment): its slot must stay
 			// authoritative so ARGC=n command-line operand assignments affect
 			// input traversal.
 			boolean symtabActive = !posix && symbolTable.isGlobalReferenced("SYMTAB");
 			if (environAst.isReferenced() || symtabActive) {
 				tuples.environOffset(environAst.offset);
 			}
 			tuples.argcOffset(argcAst.offset);
 			if (argvAst.isReferenced() || symtabActive) {
 				tuples.argvOffset(argvAst.offset);
 			}
 			// SYMTAB and FUNCTAB are gawk extensions, not POSIX
 			if (symtabActive) {
 				tuples.updateSymtab(symbolTable.getID("SYMTAB").offset);
 			}
 			if (!posix && symbolTable.isGlobalReferenced("FUNCTAB")) {
 				tuples.updateFunctab(symbolTable.getID("FUNCTAB").offset);
 			}
 			tuples.beforeStartHooks();

 			Address exitAddr = tuples.createAddress("end blocks start address");
 			tuples.setExitAddress(exitAddr);

 			// Does the program use BEGINFILE/ENDFILE rules or nextfile? If so,
 			// the main input loop must step through the input one file at a
 			// time (per-file scaffolding) instead of streaming across files.
 			boolean hasBeginFileRules = false;
 			boolean hasEndFileRules = false;
 			ptr = this;
 			while (ptr != null) {
 				if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) {
 					hasBeginFileRules = true;
 				}
 				if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) {
 					hasEndFileRules = true;
 				}
 				ptr = ptr.getAst2();
 			}

 			// Do we have rules? (apart from BEGIN)
 			// If we have rules, END, BEGINFILE, or ENDFILE, we need to parse
 			// the input
 			boolean reqInput = hasBeginFileRules || hasEndFileRules;

 			// Check for "normal" rules
 			ptr = this;
 			while (!reqInput && (ptr != null)) {
 				if (isRule(ptr.getAst1())) {
 					reqInput = true;
 				}
 				ptr = ptr.getAst2();
 			}

 			// Now check for "END" rules
 			ptr = this;
 			while (!reqInput && (ptr != null)) {
 				if (ptr.getAst1() != null && ptr.getAst1().isEnd()) {
 					reqInput = true;
 				}
 				ptr = ptr.getAst2();
 			}

 			boolean perFileScaffolding = reqInput
 					&& (hasBeginFileRules || hasEndFileRules || nextfileEncountered);

 			// The per-file addresses are carried as properties of the tuple
 			// stream, so a nextfile executed from a user-defined function can
 			// resolve them at runtime. The begin_file address labels the
 			// NEXT_FILE tuple that opens each input file.
 			Address beginFileAddress = null;
 			Address endFileAddress = null;
 			if (perFileScaffolding) {
 				beginFileAddress = tuples.createAddress("begin_file");
 				endFileAddress = tuples.createAddress("end_file");
 				// The ENDFILE address is registered only when BEGINFILE or
 				// ENDFILE rules exist: its presence also confines a
 				// non-redirected getline to the current input file, which
 				// only matters when there are per-file hooks to protect.
 				if (hasBeginFileRules || hasEndFileRules) {
 					tuples.setEndFileAddress(endFileAddress);
 				}
 				tuples.setNextFileAddress(beginFileAddress);
 			}

 			// grab all BEGINs
 			ptr = this;
 			// ptr.getAst1() == blank rule condition (i.e.: { print })
 			while (ptr != null) {
 				if (ptr.getAst1() != null && ptr.getAst1().isBegin()) {
 					ptr.getAst1().populateTuples(tuples);
 				}

 				ptr = ptr.getAst2();
 			}

 			if (reqInput) {
 				Address inputLoopAddress = tuples.createAddress("input_loop_address");
 				Address noMoreInput = tuples.createAddress("no_more_input");

 				if (perFileScaffolding) {
 					// Advance to the next input file so the BEGINFILE rules
 					// observe its FILENAME, FNR, ARGIND, and ERRNO. The
 					// ENDFILE section loops back here for the following files.
 					tuples.address(beginFileAddress);
 					tuples.nextFile(noMoreInput);

 					// BEGINFILE rules, in the order they were read
 					ptr = this;
 					while (ptr != null) {
 						if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) {
 							ptr.getAst1().populateTuples(tuples);
 						}
 						ptr = ptr.getAst2();
 					}

 					// per-file input loop: at end of the current file, run the
 					// ENDFILE rules instead of silently opening the next file
 					tuples.address(inputLoopAddress);
 					tuples.consumeFileInput(endFileAddress);
 				} else {
 					tuples.address(inputLoopAddress);
 					tuples.consumeInput(noMoreInput);
 				}

 				// grab all INPUT RULES
 				ptr = this;
 				while (ptr != null) {
 					// the first one of these is an input rule
 					if (isRule(ptr.getAst1())) {
 						ptr.getAst1().populateTuples(tuples);
 					}
 					ptr = ptr.getAst2();
 				}
 				// The main-loop resume point of a runtime next executed from
 				// a user-defined function, carried as a property of the tuple
 				// stream like the per-file addresses.
 				tuples.setNextAddress(nextAddress);
 				tuples.address(nextAddress);

 				tuples.gotoAddress(inputLoopAddress);

 				if (perFileScaffolding) {
 					// ENDFILE rules, in the order they were read
 					tuples.address(endFileAddress);
 					ptr = this;
 					while (ptr != null) {
 						if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) {
 							ptr.getAst1().populateTuples(tuples);
 						}
 						ptr = ptr.getAst2();
 					}

 					// then loop back to the NEXT_FILE tuple at begin_file,
 					// which falls through to the END rules once the input is
 					// exhausted
 					tuples.gotoAddress(beginFileAddress);
 				}

 				tuples.address(noMoreInput);
 				// compiler has issue with missing nop here
 				tuples.nop();
 			}

 			// indicate where the first end block resides
 			// in the event of an exit statement
 			tuples.address(exitAddr);
 			tuples.setWithinEndBlocks(true);

 			// grab all ENDs
 			ptr = this;
 			while (ptr != null) {
 				if (ptr.getAst1() != null && ptr.getAst1().isEnd()) {
 					ptr.getAst1().populateTuples(tuples);
 				}
 				ptr = ptr.getAst2();
 			}

 			// force a nop here to resolve any addresses that haven't been resolved yet
 			// (i.e., no_more_input wouldn't be resolved if there are no END{} blocks)
 			tuples.nop();

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class ExpressionToEvaluateAst extends AST {

 		private ExpressionToEvaluateAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {

 			pushSourceLineNumber(tuples);

 			// initialize runtime-managed special variables via JRT defaults in AVM
 			symbolTable.getID("NR");
 			symbolTable.getID("FNR");
 			symbolTable.getID("NF");
 			symbolTable.getID("FS");
 			symbolTable.getID("RS");
 			symbolTable.getID("SUBSEP");
 			symbolTable.getID("CONVFMT");
 			IDAst environAst = symbolTable.getID("ENVIRON");

 			// MUST BE DONE AFTER FUNCTIONS ARE COMPILED,
 			// and after special variables are made known to the symbol table
 			// (see above)!
 			tuples.markEvalTupleStream();
 			tuples.setNumGlobals(symbolTable.numGlobals());

 			boolean evalSymtabActive = !posix && symbolTable.isGlobalReferenced("SYMTAB");
 			if (environAst.isReferenced() || evalSymtabActive) {
 				tuples.environOffset(environAst.offset);
 			}
 			if (evalSymtabActive) {
 				tuples.updateSymtab(symbolTable.getID("SYMTAB").offset);
 			}
 			if (!posix && symbolTable.isGlobalReferenced("FUNCTAB")) {
 				tuples.updateFunctab(symbolTable.getID("FUNCTAB").offset);
 			}
 			tuples.beforeStartHooks();

 			if (getAst1() != null) {
 				getAst1().populateTuples(tuples);
 			}
 			// Some expression forms (for example ternaries) still need a concrete
 			// terminal tuple to resolve branch targets during post-processing.
 			tuples.nop();

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	// made non-static to access the "nextAddress" field of the frontend
 	private final class RuleAst extends AST {

 		private RuleAst(AST optExpression, AST optRule) {
 			super(optExpression, optRule);
 			addFlag(AstFlag.NEXTABLE);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			boolean unconditionalRule = getAst1() == null
 					|| getAst1().isBegin()
 					|| getAst1().isEnd()
 					|| getAst1().isBeginFile()
 					|| getAst1().isEndFile();
 			if (!unconditionalRule) {
 				getAst1().populateTuples(tuples);
 				// result of whether to execute or not is on the stack
 				Address bypassRule = tuples.createAddress("bypassRule");
 				tuples.ifFalse(bypassRule);
 				populateRuleBody(tuples);
 				tuples.address(bypassRule).nop();
 			} else {
 				populateRuleBody(tuples);
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}

 		private void populateRuleBody(AwkTuples tuples) {
 			// execute the optRule here!
 			if (getAst2() == null) {
 				if (isRule(this)) {
 					// display $0
 					tuples.print(0);
 				}
 				// else, don't populate it with anything
 				// (i.e., blank BEGIN/END/BEGINFILE/ENDFILE rule)
 			} else {
 				// execute it, and leave nothing on the stack
 				getAst2().populateTuples(tuples);
 			}
 		}

 		@Override
 		public Address nextAddress() {
 			if (!isRule(this)) {
 				throw new SemanticException(
 						"`next' cannot be called from a `" + specialRuleName() + "' rule.");
 			}
 			if (nextAddress == null) {
 				throw new SemanticException("Cannot call next here.");
 			}
 			return nextAddress;
 		}

 		/**
 		 * Names the special (non-input) rule this AST represents, for
 		 * gawk-compatible diagnostics.
 		 */
 		private String specialRuleName() {
 			AST pattern = getAst1();
 			if (pattern != null && pattern.isBegin()) {
 				return "BEGIN";
 			}
 			if (pattern != null && pattern.isEnd()) {
 				return "END";
 			}
 			if (pattern != null && pattern.isBeginFile()) {
 				return "BEGINFILE";
 			}
 			if (pattern != null && pattern.isEndFile()) {
 				return "ENDFILE";
 			}
 			return "special";
 		}
 	}

 	private final class IfStatementAst extends AST {

 		private IfStatementAst(AST expr, AST b1, AST b2) {
 			super(expr, b1, b2);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			Address elseblock = tuples.createAddress("elseblock");

 			getAst1().populateTuples(tuples);
 			tuples.ifFalse(elseblock);
 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}
 			if (getAst3() == null) {
 				tuples.address(elseblock);
 			} else {
 				Address end = tuples.createAddress("end");
 				tuples.gotoAddress(end);
 				tuples.address(elseblock);
 				getAst3().populateTuples(tuples);
 				tuples.address(end);
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class TernaryExpressionAst extends ScalarExpressionAst {

 		private TernaryExpressionAst(AST a1, AST a2, AST a3) {
 			super(a1, a2, a3);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			Address elseexpr = tuples.createAddress("elseexpr");
 			Address endTertiary = tuples.createAddress("endTertiary");

 			getAst1().populateTuples(tuples);
 			tuples.ifFalse(elseexpr);
 			getAst2().populateTuples(tuples);
 			tuples.gotoAddress(endTertiary);

 			tuples.address(elseexpr);
 			getAst3().populateTuples(tuples);
 			tuples.address(endTertiary);

 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class WhileStatementAst extends AST {

 		private Address breakAddress;
 		private Address continueAddress;

 		private WhileStatementAst(AST expr, AST block) {
 			super(expr, block);
 			addFlag(AstFlag.BREAKABLE);
 			addFlag(AstFlag.CONTINUEABLE);
 		}

 		@Override
 		public Address breakAddress() {
 			return breakAddress;
 		}

 		@Override
 		public Address continueAddress() {
 			return continueAddress;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			breakAddress = tuples.createAddress("breakAddress");

 			// LOOP
 			Address loop = tuples.createAddress("loop");
 			tuples.address(loop);

 			// for while statements, the start-of-loop is the continue jump address
 			continueAddress = loop;

 			// condition
 			getAst1().populateTuples(tuples);
 			tuples.ifFalse(breakAddress);

 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}

 			tuples.gotoAddress(loop);

 			tuples.address(breakAddress);

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class DoStatementAst extends AST {

 		private Address breakAddress;
 		private Address continueAddress;

 		private DoStatementAst(AST block, AST expr) {
 			super(block, expr);
 			addFlag(AstFlag.BREAKABLE);
 			addFlag(AstFlag.CONTINUEABLE);
 		}

 		@Override
 		public Address breakAddress() {
 			return breakAddress;
 		}

 		@Override
 		public Address continueAddress() {
 			return continueAddress;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			breakAddress = tuples.createAddress("breakAddress");
 			continueAddress = tuples.createAddress("continueAddress");

 			// LOOP
 			Address loop = tuples.createAddress("loop");
 			tuples.address(loop);

 			if (getAst1() != null) {
 				getAst1().populateTuples(tuples);
 			}

 			// for do-while statements, the continue jump address is the loop condition
 			tuples.address(continueAddress);

 			// condition
 			getAst2().populateTuples(tuples);
 			tuples.ifTrue(loop);

 			// tuples.gotoAddress(loop);

 			tuples.address(breakAddress);

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class ForStatementAst extends AST {

 		private Address breakAddress;
 		private Address continueAddress;

 		private ForStatementAst(AST expr1, AST expr2, AST expr3, AST block) {
 			super(expr1, expr2, expr3, block);
 			addFlag(AstFlag.BREAKABLE);
 			addFlag(AstFlag.CONTINUEABLE);
 		}

 		@Override
 		public Address breakAddress() {
 			return breakAddress;
 		}

 		@Override
 		public Address continueAddress() {
 			return continueAddress;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			breakAddress = tuples.createAddress("breakAddress");
 			continueAddress = tuples.createAddress("continueAddress");

 			// initial actions
 			if (getAst1() != null) {
 				int ast1Result = getAst1().populateTuples(tuples);
 				for (int i = 0; i < ast1Result; i++) {
 					tuples.pop();
 				}
 			}
 			// LOOP
 			Address loop = tuples.createAddress("loop");
 			tuples.address(loop);

 			if (getAst2() != null) {
 				// condition
 				// assert(getAst2() != null);
 				getAst2().populateTuples(tuples);
 				tuples.ifFalse(breakAddress);
 			}

 			if (getAst4() != null) {
 				// post loop action
 				getAst4().populateTuples(tuples);
 			}

 			// for for-loops, the continue jump address is the post-loop-action
 			tuples.address(continueAddress);

 			// post-loop action
 			if (getAst3() != null) {
 				int ast3Result = getAst3().populateTuples(tuples);
 				for (int i = 0; i < ast3Result; i++) {
 					tuples.pop();
 				}
 			}

 			tuples.gotoAddress(loop);

 			tuples.address(breakAddress);

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class ForInStatementAst extends AST {

 		private Address breakAddress;
 		private Address continueAddress;

 		private ForInStatementAst(AST keyIdAst, AST arrayIdAst, AST block) {
 			super(keyIdAst, arrayIdAst, block);
 			addFlag(AstFlag.BREAKABLE);
 			addFlag(AstFlag.CONTINUEABLE);
 		}

 		@Override
 		public Address breakAddress() {
 			return breakAddress;
 		}

 		@Override
 		public Address continueAddress() {
 			return continueAddress;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			breakAddress = tuples.createAddress("breakAddress");

 			populateArrayOperandTuples(getAst2(), tuples, false, getAst2() + " is not an array");
 			// pops the array and pushes the keyset
 			tuples.keylist();

 			// stack now contains:
 			// keylist

 			// LOOP
 			Address loop = tuples.createAddress("loop");
 			tuples.address(loop);

 			// for for-in loops, the continue jump address is the start-of-loop address
 			continueAddress = loop;

 			// condition
 			tuples.dup();
 			tuples.isEmptyList(breakAddress);

 			// take an element off the set
 			tuples.dup();
 			tuples.getFirstAndRemoveFromList();
 			// assign it to the id
 			tuples.assign(((IDAst) getAst1()).offset, ((IDAst) getAst1()).isGlobal);
 			tuples.pop(); // remove the assignment result

 			if (getAst3() != null) {
 				// execute the block
 				getAst3().populateTuples(tuples);
 			}
 			// otherwise, there is no block to execute

 			tuples.gotoAddress(loop);

 			tuples.address(breakAddress);
 			tuples.pop(); // keylist

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	@SuppressWarnings("unused")
 	private final class EmptyStatementAst extends AST {

 		private EmptyStatementAst() {
 			super();
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			// nothing to populate!
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	/**
 	 * The AST for an expression used as a statement.
 	 * If the expression returns a value, the value is popped
 	 * off the stack and discarded.
 	 */
 	private final class ExpressionStatementAst extends AST {

 		private ExpressionStatementAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			int exprCount = getAst1().populateTuples(tuples);
 			if (exprCount == 1) {
 				tuples.popScalar();
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class AssignmentExpressionAst extends ScalarExpressionAst {

 		/** operand / operator */
 		private Token op;
 		private String text;

 		private AssignmentExpressionAst(AST lhs, Token op, String text, AST rhs) {
 			super(lhs, rhs);
 			this.op = op;
 			this.text = text;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + op + "/" + text + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst2().populateTuples(tuples); // here, stack contains one value
 			if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				idAst.setScalar(true);
 				boolean isSpecial = isJrtManagedSpecialName(idAst.id);
 				if (isSpecial) {
 					// value is on stack from RHS
 					switch (op) {
 					case EQUALS:
 						assignSpecial(tuples, idAst.id);
 						break;
 					case PLUS_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.add();
 						assignSpecial(tuples, idAst.id);
 						break;
 					case MINUS_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.subtract();
 						assignSpecial(tuples, idAst.id);
 						break;
 					case MULT_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.multiply();
 						assignSpecial(tuples, idAst.id);
 						break;
 					case DIV_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.divide();
 						assignSpecial(tuples, idAst.id);
 						break;
 					case MOD_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.mod();
 						assignSpecial(tuples, idAst.id);
 						break;
 					case POW_EQ:
 						pushSpecialThenSwap(tuples, idAst.id);
 						tuples.pow();
 						assignSpecial(tuples, idAst.id);
 						break;
 					default:
 						throw new Error("Unhandled op: " + op + " / " + text);
 					}
 					if ("RS".equals(idAst.id)) {
 						tuples.applyRS();
 					}
 				} else {
 					if (op == Token.EQUALS) {
 						// Expected side effect:
 						// Upon assignment, if the var is RS, reapply RS to input streams.
 						tuples.assign(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.PLUS_EQ) {
 						tuples.plusEq(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.MINUS_EQ) {
 						tuples.minusEq(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.MULT_EQ) {
 						tuples.multEq(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.DIV_EQ) {
 						tuples.divEq(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.MOD_EQ) {
 						tuples.modEq(idAst.offset, idAst.isGlobal);
 					} else if (op == Token.POW_EQ) {
 						tuples.powEq(idAst.offset, idAst.isGlobal);
 					} else {
 						throw new Error("Unhandled op: " + op + " / " + text);
 					}
 					if (idAst.id.equals("RS")) {
 						tuples.applyRS();
 					}
 				}
 			} else if (getAst1() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arr = (ArrayReferenceAst) getAst1();
 				if (arr.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arr.getAst1();
 					idAst.setArray(true);
 				}
 				arr.populateTargetReferenceTuples(tuples);
 				if (op == Token.EQUALS) {
 					tuples.assignMapElement();
 				} else if (op == Token.PLUS_EQ) {
 					tuples.plusEqMapElement();
 				} else if (op == Token.MINUS_EQ) {
 					tuples.minusEqMapElement();
 				} else if (op == Token.MULT_EQ) {
 					tuples.multEqMapElement();
 				} else if (op == Token.DIV_EQ) {
 					tuples.divEqMapElement();
 				} else if (op == Token.MOD_EQ) {
 					tuples.modEqMapElement();
 				} else if (op == Token.POW_EQ) {
 					tuples.powEqMapElement();
 				} else {
 					throw new NotImplementedError("Unhandled op: " + op + " / " + text + " for arrays.");
 				}
 			} else if (getAst1() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
 				dollarExpr.getAst1().populateTuples(tuples); // stack contains eval of dollar arg

 				if (op == Token.EQUALS) {
 					tuples.assignAsInputField();
 				} else if (op == Token.PLUS_EQ) {
 					tuples.plusEqInputField();
 				} else if (op == Token.MINUS_EQ) {
 					tuples.minusEqInputField();
 				} else if (op == Token.MULT_EQ) {
 					tuples.multEqInputField();
 				} else if (op == Token.DIV_EQ) {
 					tuples.divEqInputField();
 				} else if (op == Token.MOD_EQ) {
 					tuples.modEqInputField();
 				} else if (op == Token.POW_EQ) {
 					tuples.powEqInputField();
 				} else {
 					throw new NotImplementedError("Unhandled op: " + op + " / " + text + " for dollar expressions.");
 				}
 			} else {
 				throw new SemanticException("Cannot perform an assignment on: " + getAst1());
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		private void pushSpecialThenSwap(AwkTuples tuples, String id) {
 			pushSpecialVariable(tuples, id);
 			tuples.swap();
 		}

 		private void assignSpecial(AwkTuples tuples, String id) {
 			assignSpecialVariable(tuples, id);
 		}
 	}

 	private final class InExpressionAst extends ScalarExpressionAst {

 		private InExpressionAst(AST arg, AST arr) {
 			super(arg, arr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (!(getAst2() instanceof IDAst) && !(getAst2() instanceof ArrayReferenceAst)) {
 				throw new SemanticException("Expecting an array for rhs of IN. Got an expression.");
 			}

 			// The key is joined and converted with the SUBSEP and CONVFMT in
 			// effect when the operator executes, i.e. after the array operand
 			// has been evaluated (it may have side effects), just like gawk.
 			AST keyAst = getAst1();
 			if (keyAst instanceof ArrayIndexAst) {
 				ArrayIndexAst indexAst = (ArrayIndexAst) keyAst;
 				int count = indexAst.populateComponentTuples(tuples);
 				populateArrayOperandTuples(getAst2(), tuples, false, "Expecting an array for rhs of IN. Got a scalar.");
 				// Errors while converting the key report where the subscript
 				// starts.
 				indexAst.pushSourceLineNumber(tuples);
 				tuples.applySubsepUnderTop(count);
 				indexAst.popSourceLineNumber(tuples);
 			} else {
 				keyAst.populateTuples(tuples);
 				populateArrayOperandTuples(getAst2(), tuples, false, "Expecting an array for rhs of IN. Got a scalar.");
 				tuples.applySubsepUnderTop(1);
 			}
 			tuples.isIn();

 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class ComparisonExpressionAst extends ScalarExpressionAst {

 		/**
 		 * operand / operator
 		 */
 		private Token op;
 		private String text;

 		private ComparisonExpressionAst(AST lhs, Token op, String text, AST rhs) {
 			super(lhs, rhs);
 			this.op = op;
 			this.text = text;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + op + "/" + text + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			getAst1().populateTuples(tuples);
 			if (op == Token.MATCHES || op == Token.NOT_MATCHES) {
 				populateRawRegexpParameterTuples(getAst2(), tuples);
 			} else {
 				getAst2().populateTuples(tuples);
 			}
 			// 2 values on the stack

 			if (op == Token.EQ) {
 				tuples.cmpEq();
 			} else if (op == Token.NE) {
 				tuples.cmpEq();
 				tuples.not();
 			} else if (op == Token.LT) {
 				tuples.cmpLt();
 			} else if (op == Token.GT) {
 				tuples.cmpGt();
 			} else if (op == Token.LE) {
 				tuples.cmpGt();
 				tuples.not();
 			} else if (op == Token.GE) {
 				tuples.cmpLt();
 				tuples.not();
 			} else if (op == Token.MATCHES) {
 				tuples.matches();
 			} else if (op == Token.NOT_MATCHES) {
 				tuples.matches();
 				tuples.not();
 			} else {
 				throw new Error("Unhandled op: " + op + " / " + text);
 			}

 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class LogicalExpressionAst extends ScalarExpressionAst {

 		/**
 		 * operand / operator
 		 */
 		private Token op;
 		private String text;

 		private LogicalExpressionAst(AST lhs, Token op, String text, AST rhs) {
 			super(lhs, rhs);
 			this.op = op;
 			this.text = text;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + op + "/" + text + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			// exhibit short-circuit behavior
 			Address end = tuples.createAddress("end");
 			getAst1().populateTuples(tuples);
 			tuples.dup();
 			if (op == Token.OR) {
 				// shortCircuit when op is Token.OR and 1st arg is true
 				tuples.ifTrue(end);
 			} else if (op == Token.AND) {
 				tuples.ifFalse(end);
 			}
 			tuples.pop();
 			getAst2().populateTuples(tuples);
 			tuples.address(end);

 			// turn the result into boolean one or zero
 			tuples.toNumber();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class BinaryExpressionAst extends ScalarExpressionAst {

 		/**
 		 * operand / operator
 		 */
 		private Token op;
 		private String text;

 		private BinaryExpressionAst(AST lhs, Token op, String text, AST rhs) {
 			super(lhs, rhs);
 			this.op = op;
 			this.text = text;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + op + "/" + text + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			getAst2().populateTuples(tuples);
 			if (op == Token.PLUS) {
 				tuples.add();
 			} else if (op == Token.MINUS) {
 				tuples.subtract();
 			} else if (op == Token.MULT) {
 				tuples.multiply();
 			} else if (op == Token.DIVIDE) {
 				tuples.divide();
 			} else if (op == Token.MOD) {
 				tuples.mod();
 			} else if (op == Token.POW) {
 				tuples.pow();
 			} else {
 				throw new Error("Unhandled op: " + op + " / " + this);
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class ConcatExpressionAst extends ScalarExpressionAst {

 		private ConcatExpressionAst(AST lhs, AST rhs) {
 			super(lhs, rhs);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			getAst2().populateTuples(tuples);
 			tuples.concat();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class NegativeExpressionAst extends ScalarExpressionAst {

 		private NegativeExpressionAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			tuples.negate();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class UnaryPlusExpressionAst extends ScalarExpressionAst {

 		private UnaryPlusExpressionAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			tuples.unaryPlus();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class NotExpressionAst extends ScalarExpressionAst {

 		private NotExpressionAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			tuples.not();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class DollarExpressionAst extends ScalarExpressionAst {

 		private DollarExpressionAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			tuples.getInputField();
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class ArrayIndexAst extends ScalarExpressionAst {

 		private ArrayIndexAst(int lineNo, AST exprAst, AST next) {
 			// The line is captured where the subscript expression starts, so
 			// the APPLY_SUBSEP tuple reports errors there.
 			super(lineNo, exprAst, next);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			int cnt = populateComponentTuples(tuples);
 			// Convert the subscript to its string form now, with the CONVFMT in
 			// effect at this point of the execution: a later CONVFMT change must
 			// not retroactively alter existing keys.
 			tuples.applySubsep(cnt);
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		/**
 		 * Evaluates the raw subscript components without joining or converting
 		 * them, and returns how many were pushed. The "in" operator uses this
 		 * to delay the conversion until its array operand has been evaluated.
 		 */
 		private int populateComponentTuples(AwkTuples tuples) {
 			AST ptr = this;
 			int cnt = 0;
 			while (ptr != null) {
 				ptr.getAst1().populateTuples(tuples);
 				++cnt;
 				ptr = ptr.getAst2();
 			}
 			return cnt;
 		}
 	}

 	// made classname all capitals to stand out in a syntax tree dump
 	private final class StatementListAst extends AST {

 		private StatementListAst(AST statementAst, AST rest) {
 			super(statementAst, rest);
 		}

 		/**
 		 * Recursively process statements within this statement list.
 		 * <p>
 		 * It originally was done linearly. However, quirks in the grammar required
 		 * a more general, recursive approach to processing this "list".
 		 * <p>
 		 * Note: this should be reevaluated periodically in case the grammar
 		 * becomes linear again.
 		 */
 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			// typical recursive processing of a list
 			getAst1().populateTuples(tuples);
 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " <" + getAst1() + ">";
 		}
 	}

 	// made non-static to access the symbol table
 	private final class FunctionDefAst extends AST {

 		private String id;
 		private Address functionAddress;
 		private Address returnAddress;

 		@Override
 		public Address returnAddress() {
 			return returnAddress;
 		}

 		private FunctionDefAst(String id, AST params, AST funcBody) {
 			super(params, funcBody);
 			this.id = id;
 			setFunctionFlag(true);
 			addFlag(AstFlag.RETURNABLE);
 		}

 		public Address getAddress() {
 			return functionAddress;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			functionAddress = tuples.createAddress("function: " + id);
 			returnAddress = tuples.createAddress("returnAddress for " + id);

 			// annotate the tuple list
 			// (useful for compilation,
 			// not necessary for interpretation)
 			tuples.function(id, paramCount());

 			// functionAddress refers to first function body statement
 			// rather than to function def opcode because during
 			// interpretation, the function definition is a nop,
 			// and for compilation, the next match of the function
 			// name can be used
 			tuples.address(functionAddress);

 			// the stack contains the parameters to the function call (in rev order, which is good)

 			// execute the body
 			// (function body could be empty [no statements])
 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}

 			tuples.address(returnAddress);

 			tuples.returnFromFunction();

 			/////////////////////////////////////////////

 			popSourceLineNumber(tuples);
 			return 0;
 		}

 		int paramCount() {
 			AST ptr = getAst1();
 			int count = 0;
 			while (ptr != null) {
 				++count;
 				ptr = ptr.getAst1();
 			}
 			return count;
 		}

 	}

 	private final class FunctionCallAst extends ScalarExpressionAst {

 		private FunctionProxy functionProxy;

 		private FunctionCallAst(FunctionProxy functionProxy, AST params) {
 			super(params);
 			this.functionProxy = functionProxy;
 		}

 		/**
 		 * Applies several semantic checks with respect
 		 * to user-defined-function calls.
 		 * <p>
 		 * The checks performed are:
 		 * <ul>
 		 * <li>Make sure the function is defined.
 		 * </ul>
 		 * A failure of any one of these checks
 		 * results in a SemanticException.
 		 *
 		 * @throws SemanticException upon a failure of
 		 *         any of the semantic checks specified above.
 		 */
 		@Override
 		public void semanticAnalysis() throws SemanticException {
 			if (!functionProxy.isDefined()) {
 				throw new SemanticException("function " + functionProxy + " not defined");
 			}
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (!functionProxy.isDefined()) {
 				throw new SemanticException("function " + functionProxy + " not defined");
 			}
 			tuples.scriptThis();
 			int actualParamCountLocal;
 			if (getAst1() == null) {
 				actualParamCountLocal = 0;
 			} else {
 				actualParamCountLocal = populateActualParametersUpTo(
 						tuples,
 						(FunctionCallParamListAst) getAst1(),
 						0,
 						functionProxy.getFunctionParamCount());
 			}
 			int formalParamCount = functionProxy.getFunctionParamCount();

 			if (actualParamCount() > formalParamCount) {
 				// gawk accepts the call but reports it each time it runs
 				tuples.warning(extraArgumentWarning());
 			}
 			tuples
 					.callFunction(
 							functionProxy,
 							functionProxy.getFunctionName(),
 							formalParamCount,
 							actualParamCountLocal);
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		private int actualParamCount() {
 			int cnt = 0;
 			AST ptr = getAst1();
 			while (ptr != null) {
 				++cnt;
 				ptr = ptr.getAst2();
 			}
 			return cnt;
 		}

 		private String extraArgumentWarning() {
 			return String
 					.format(
 							"gawk: %s:%d: warning: function `%s' called with more arguments than declared",
 							sourceBasename(),
 							warningLineNo(),
 							functionProxy.getFunctionName());
 		}

 		private int warningLineNo() {
 			if (getAst1() instanceof FunctionCallParamListAst) {
 				AST firstParam = getAst1().getAst1();
 				if (firstParam != null) {
 					return firstParam.getLineNo();
 				}
 			}
 			return getLineNo();
 		}

 	}

 	private final class IndirectFunctionCallAst extends ScalarExpressionAst {

 		private IndirectFunctionCallAst(AST functionNameAst, AST params) {
 			super(functionNameAst, params);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			getAst1().populateTuples(tuples);
 			int actualParamCount = populateIndirectActualParameters(
 					tuples,
 					(FunctionCallParamListAst) getAst2());
 			tuples
 					.indirectCall(
 							symbolTable.indirectFunctionTargets(),
 							extensions,
 							actualParamCount,
 							sourceBasename(),
 							getLineNo());
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class BuiltinFunctionCallAst extends ScalarExpressionAst {

 		private final String id;
 		private final BuiltinFunction builtin;

 		private BuiltinFunctionCallAst(String id, AST params) {
 			super(params);
 			this.id = id;
 			this.builtin = BuiltinFunction.of(id);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			switch (builtin) {
 			case SPRINTF:
 				populateSprintfTuples(tuples);
 				break;
 			case CLOSE:
 				populateOneArgumentTuples(tuples, "close");
 				tuples.close();
 				break;
 			case LENGTH:
 				populateLengthTuples(tuples);
 				break;
 			case SRAND:
 				populateSrandTuples(tuples);
 				break;
 			case RAND:
 				if (getAst1() != null) {
 					throw new SemanticException("rand does not take arguments");
 				}
 				tuples.rand();
 				break;
 			case SQRT:
 				populateArgumentsTuples(tuples, 1, "sqrt requires only 1 argument");
 				tuples.sqrt();
 				break;
 			case INT:
 				populateArgumentsTuples(tuples, 1, "int requires only 1 argument");
 				tuples.intFunc();
 				break;
 			case LOG:
 				populateArgumentsTuples(tuples, 1, "log requires only 1 argument");
 				tuples.log();
 				break;
 			case EXP:
 				populateArgumentsTuples(tuples, 1, "exp requires only 1 argument");
 				tuples.exp();
 				break;
 			case SIN:
 				populateArgumentsTuples(tuples, 1, "sin requires only 1 argument");
 				tuples.sin();
 				break;
 			case COS:
 				populateArgumentsTuples(tuples, 1, "cos requires only 1 argument");
 				tuples.cos();
 				break;
 			case ATAN2:
 				populateArgumentsTuples(tuples, 2, "atan2 requires 2 arguments");
 				tuples.atan2();
 				break;
 			case MATCH:
 				populateMatchArgumentsTuples(tuples);
 				tuples.match();
 				break;
 			case INDEX:
 				populateArgumentsTuples(tuples, 2, "index requires 2 arguments");
 				tuples.index();
 				break;
 			case SUB:
 			case GSUB:
 				populateSubTuples(tuples, builtin == BuiltinFunction.GSUB);
 				break;
 			case SPLIT:
 				populateSplitTuples(tuples);
 				break;
 			case SUBSTR:
 				populateSubstrTuples(tuples);
 				break;
 			case TOLOWER:
 				populateOneArgumentTuples(tuples, "tolower");
 				tuples.tolower();
 				break;
 			case TOUPPER:
 				populateOneArgumentTuples(tuples, "toupper");
 				tuples.toupper();
 				break;
 			case SYSTEM:
 				populateOneArgumentTuples(tuples, "system");
 				tuples.system();
 				break;
 			default:
 				throw new NotImplementedError("builtin: " + id);
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		/**
 		 * Populates the tuples of the argument list, validating that
 		 * it produces exactly the expected number of values on the stack.
 		 *
 		 * @param tuples the tuples to populate
 		 * @param expectedCount the exact number of arguments required
 		 * @param errorMessage the message of the {@link SemanticException}
 		 *        thrown when the argument count does not match
 		 */
 		private void populateArgumentsTuples(AwkTuples tuples, int expectedCount, String errorMessage) {
 			int ast1Result = getAst1().populateTuples(tuples);
 			if (ast1Result != expectedCount) {
 				throw new SemanticException(errorMessage);
 			}
 		}

 		/**
 		 * Populates the tuples of the argument list for a built-in function
 		 * taking exactly one argument, rejecting missing or extra arguments.
 		 *
 		 * @param tuples the tuples to populate
 		 * @param functionName the name of the built-in function, used in
 		 *        error messages
 		 */
 		private void populateOneArgumentTuples(AwkTuples tuples, String functionName) {
 			if (getAst1() == null) {
 				throw new SemanticException(functionName + " requires 1 argument");
 			}
 			int ast1Result = getAst1().populateTuples(tuples);
 			if (ast1Result != 1) {
 				throw new SemanticException(functionName + " requires only 1 argument");
 			}
 		}

 		/**
 		 * Populates the tuples of the argument list for the <code>match</code>
 		 * built-in function, treating its 2nd argument as a literal regular
 		 * expression.
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateMatchArgumentsTuples(AwkTuples tuples) {
 			int ast1Result = populateActualParameters(tuples, (FunctionCallParamListAst) getAst1(), 1);
 			if (ast1Result != 2) {
 				throw new SemanticException("match requires 2 arguments");
 			}
 		}

 		/**
 		 * Populates the tuples for the <code>sprintf</code> built-in function.
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateSprintfTuples(AwkTuples tuples) {
 			if (getAst1() == null) {
 				throw new SemanticException("sprintf requires at least 1 argument");
 			}
 			int ast1Result = getAst1().populateTuples(tuples);
 			if (ast1Result == 0) {
 				throw new SemanticException("sprintf requires at minimum 1 argument");
 			}
 			tuples.sprintf(ast1Result);
 		}

 		/**
 		 * Populates the tuples for the <code>length</code> built-in function,
 		 * which takes either no argument (implying $0) or one argument.
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateLengthTuples(AwkTuples tuples) {
 			if (getAst1() == null) {
 				tuples.length(0);
 			} else {
 				AST params = getAst1();
 				AST argument = params instanceof FunctionCallParamListAst
 						&& params.getAst2() == null ?
 								params.getAst1() : params;
 				int ast1Result;
 				if (argument instanceof IDAst
 						&& !isJrtManagedSpecialName(((IDAst) argument).id)) {
 					IDAst idAst = (IDAst) argument;
 					tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
 					ast1Result = 1;
 				} else {
 					ast1Result = argument.populateTuples(tuples);
 				}
 				if (ast1Result != 1) {
 					throw new SemanticException("length requires at least one argument");
 				}
 				tuples.length(1);
 			}
 		}

 		/**
 		 * Populates the tuples for the <code>srand</code> built-in function,
 		 * which takes either no argument or one argument (the seed).
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateSrandTuples(AwkTuples tuples) {
 			if (getAst1() == null) {
 				tuples.srand(0);
 			} else {
 				int ast1Result = getAst1().populateTuples(tuples);
 				if (ast1Result != 1) {
 					throw new SemanticException("srand takes either 0 or one argument, not " + ast1Result);
 				}
 				tuples.srand(1);
 			}
 		}

 		/**
 		 * Populates the tuples for the <code>sub</code> and <code>gsub</code>
 		 * built-in functions, which take 2 or 3 arguments; the optional 3rd
 		 * argument is the substitution target (a variable, an array element,
 		 * or an input field reference), defaulting to $0.
 		 *
 		 * @param tuples the tuples to populate
 		 * @param isGsub <code>true</code> for <code>gsub</code> (global
 		 *        substitution), <code>false</code> for <code>sub</code>
 		 */
 		private void populateSubTuples(AwkTuples tuples, boolean isGsub) {
 			if (getAst1() == null || getAst1().getAst2() == null || getAst1().getAst2().getAst1() == null) {
 				throw new SemanticException("sub needs at least 2 arguments");
 			}
 			int numargs = 0;
 			for (AST paramPtr = getAst1(); paramPtr != null; paramPtr = paramPtr.getAst2()) {
 				numargs++;
 			}
 			if (numargs != 2 && numargs != 3) {
 				throw new SemanticException("sub requires 2 or 3 arguments, not " + numargs);
 			}

 			populateRawRegexpParameterTuples(getAst1().getAst1(), tuples);
 			getAst1().getAst2().getAst1().populateTuples(tuples);
 			if (numargs == 3) {
 				AST targetAst = getAst1().getAst2().getAst2().getAst1();
 				if (targetAst instanceof ArrayReferenceAst) {
 					((ArrayReferenceAst) targetAst).populateTargetValueTuples(tuples);
 				} else {
 					targetAst.populateTuples(tuples);
 				}
 			}

 			// stack contains arg1,arg2[,arg3] - in that pop() order

 			if (numargs == 2) {
 				tuples.subForDollar0(isGsub);
 			} else if (numargs == 3) {
 				AST ptr = getAst1().getAst2().getAst2().getAst1();
 				if (ptr instanceof IDAst) {
 					IDAst idAst = (IDAst) ptr;
 					idAst.setScalar(true);
 					tuples.subForVariable(idAst.offset, idAst.isGlobal, isGsub);
 				} else if (ptr instanceof ArrayReferenceAst) {
 					ArrayReferenceAst arrAst = (ArrayReferenceAst) ptr;
 					if (arrAst.getAst1() instanceof IDAst) {
 						IDAst idAst = (IDAst) arrAst.getAst1();
 						idAst.setArray(true);
 					}
 					arrAst.populateTargetReferenceTuples(tuples);
 					tuples.subForMapReference(isGsub);
 				} else if (ptr instanceof DollarExpressionAst) {
 					// push the field ref
 					DollarExpressionAst dollarExpr = (DollarExpressionAst) ptr;
 					dollarExpr.getAst1().populateTuples(tuples);
 					tuples.subForDollarReference(isGsub);
 				} else {
 					throw new SemanticException(
 							"sub's 3rd argument must be either an id, an array reference, or an input field reference");
 				}
 			}
 		}

 		/**
 		 * Populates the tuples for the <code>split</code> built-in function.
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateSplitTuples(AwkTuples tuples) {
 			// split can take 2 or 3 args:
 			// split (string, array [,fs])
 			// the 2nd argument is pass by reference, which is ok (?)

 			// funccallparamlist.funccallparamlist.idAst
 			if (getAst1() == null || getAst1().getAst2() == null || getAst1().getAst2().getAst1() == null) {
 				throw new SemanticException("split needs at least 2 arguments");
 			}
 			AST ptr = getAst1().getAst2().getAst1();
 			if (!(ptr instanceof IDAst) && !(ptr instanceof ArrayReferenceAst)) {
 				throw new SemanticException("split needs an array or subarray reference as its 2nd argument");
 			}
 			if (ptr instanceof IDAst) {
 				IDAst arrAst = (IDAst) ptr;
 				if (arrAst.isScalar()) {
 					throw new SemanticException("split's 2nd arg cannot be a scalar");
 				}
 				arrAst.setArray(true);
 			}

 			int ast1Result = 0;
 			for (AST paramPtr = getAst1(); paramPtr != null; paramPtr = paramPtr.getAst2()) {
 				ast1Result++;
 			}
 			if (ast1Result != 2 && ast1Result != 3) {
 				throw new SemanticException("split requires 2 or 3 arguments, not " + ast1Result);
 			}

 			getAst1().getAst1().populateTuples(tuples);
 			populateArrayOperandTuples(
 					ptr,
 					tuples,
 					true,
 					"split's 2nd arg must be an array or subarray reference");
 			if (ast1Result == 3) {
 				populateRawRegexpParameterTuples(getAst1().getAst2().getAst2().getAst1(), tuples);
 			}
 			tuples.split(ast1Result);
 		}

 		/**
 		 * Populates the tuples for the <code>substr</code> built-in function,
 		 * which takes 2 or 3 arguments.
 		 *
 		 * @param tuples the tuples to populate
 		 */
 		private void populateSubstrTuples(AwkTuples tuples) {
 			if (getAst1() == null) {
 				throw new SemanticException("substr requires at least 2 arguments");
 			}
 			int ast1Result = getAst1().populateTuples(tuples);
 			if (ast1Result != 2 && ast1Result != 3) {
 				throw new SemanticException("substr requires 2 or 3 arguments, not " + ast1Result);
 			}
 			tuples.substr(ast1Result);
 		}
 	}

 	private final class FunctionCallParamListAst extends AST {

 		private FunctionCallParamListAst(AST expr, AST rest) {
 			super(expr, rest);
 		}

 		private FunctionCallParamListAst(int lineNo, AST expr, AST rest) {
 			super(lineNo, expr, rest);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			int retval;
 			if (getAst2() == null) {
 				retval = getAst1().populateTuples(tuples);
 			} else {
 				retval = getAst1().populateTuples(tuples) + getAst2().populateTuples(tuples);
 			}
 			popSourceLineNumber(tuples);
 			return retval;
 		}
 	}

 	private final class FunctionDefParamListAst extends AST {

 		private String id;

 		private FunctionDefParamListAst(String id, AST rest) {
 			super(rest);
 			this.id = id;
 		}

 		public int populateTuples(AwkTuples tuples) {
 			throw new Error("Cannot 'execute' function definition parameter list (formal parameters) in this manner.");
 		}

 		/**
 		 * According to the spec
 		 * (http://www.opengroup.org/onlinepubs/007908799/xcu/awk.html)
 		 * formal function parameters cannot be special variables,
 		 * such as NF, NR, etc).
 		 *
 		 * @throws SemanticException upon a semantic error.
 		 */
 		@Override
 		public void semanticAnalysis() throws SemanticException {
 			// could do it recursively, but not necessary
 			// since all getAst1()'s are FunctionDefParamList's
 			// and, thus, terminals (no need to do further
 			// semantic analysis)

 			FunctionDefParamListAst ptr = this;
 			while (ptr != null) {
 				if (isSpecialVariableName(ptr.id)) {
 					throw new SemanticException("Special variable " + ptr.id + " cannot be used as a formal parameter");
 				}
 				ptr = (FunctionDefParamListAst) ptr.getAst1();
 			}
 		}
 	}

 	/**
 	 * Flag for non-statement expressions.
 	 * Unknown for certain, but I think this is done
 	 * to avoid partial variable assignment mistakes.
 	 * For example, instead of a=3, the programmer
 	 * inadvertently places the a on the line. If IDAsts
 	 * were not tagged with AstFlag.NON_STATEMENT, then the
 	 * incomplete assignment would parse properly, and
 	 * the developer might remain unaware of this issue.
 	 */

 	private final class IDAst extends AST {

 		private String id;
 		private int offset = AVM.NULL_OFFSET;
 		private boolean isGlobal;
 		private boolean referenced;

 		private IDAst(String id, boolean isGlobal) {
 			this.id = id;
 			this.isGlobal = isGlobal;
 			addFlag(AstFlag.NON_STATEMENT);
 		}

 		private boolean isArray = false;
 		private boolean isScalar = false;

 		@Override
 		public String toString() {
 			return super.toString() + " (" + id + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (isJrtManagedSpecialName(id)) {
 				// Use JRT-managed reads for specials
 				pushSpecialVariable(tuples, id);
 			} else {
 				// Bare identifiers are scalar uses. Array-only contexts emit a
 				// typed dereference through populateArrayOperandTuples().
 				tuples.dereference(offset, false, isGlobal);
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		@Override
 		public boolean isArray() {
 			return isArray;
 		}

 		@Override
 		public boolean isScalar() {
 			return isScalar;
 		}

 		private boolean isReferenced() {
 			return referenced;
 		}

 		private void markReferenced() {
 			referenced = true;
 		}

 		private void setArray(boolean b) {
 			isArray = b;
 		}

 		private void setScalar(boolean b) {
 			isScalar = b;
 		}
 	}

 	private final class ArrayReferenceAst extends ScalarExpressionAst {

 		private ArrayReferenceAst(AST idAst, AST idxAst) {
 			super(idAst, idxAst);
 		}

 		private ArrayReferenceAst(int lineNo, AST idAst, AST idxAst) {
 			super(lineNo, idAst, idxAst);
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + getAst1() + " [...])";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			// get the containing array, autovivifying missing parent subarrays
 			populateContainerTuples(tuples);
 			// get the index
 			getAst2().populateTuples(tuples);
 			tuples.dereferenceArray();
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		private void populateTargetReferenceTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			populateContainerTuples(tuples);
 			getAst2().populateTuples(tuples);
 			popSourceLineNumber(tuples);
 		}

 		private void populateArrayValueTuples(AwkTuples tuples, boolean createIfMissing) {
 			pushSourceLineNumber(tuples);
 			populateContainerTuples(tuples);
 			getAst2().populateTuples(tuples);
 			if (createIfMissing) {
 				tuples.ensureArrayElement();
 			} else {
 				tuples.peekArrayElement();
 			}
 			popSourceLineNumber(tuples);
 		}

 		private void populateTargetValueTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			populateContainerTuples(tuples);
 			getAst2().populateTuples(tuples);
 			tuples.dereferenceArray();
 			popSourceLineNumber(tuples);
 		}

 		private void populateContainerTuples(AwkTuples tuples) {
 			if (getAst1() instanceof ArrayReferenceAst) {
 				((ArrayReferenceAst) getAst1()).populateArrayValueTuples(tuples, true);
 			} else if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				if (isJrtManagedSpecialName(idAst.id)) {
 					idAst.populateTuples(tuples);
 				} else {
 					tuples.dereference(idAst.offset, true, idAst.isGlobal);
 				}
 			} else {
 				getAst1().populateTuples(tuples);
 			}
 		}
 	}

 	private final class IntegerAst extends ScalarExpressionAst {

 		private Long value;

 		private IntegerAst(Long value) {
 			this.value = value;
 			addFlag(AstFlag.NON_STATEMENT);
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + value + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(value);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	/**
 	 * Can either assume the role of a double or an integer
 	 * by aggressively normalizing the value to an int if possible.
 	 */
 	private final class DoubleAst extends ScalarExpressionAst {

 		private Object value;

 		private DoubleAst(Double val) {
 			double d = val.doubleValue();
 			if (d == (int) d) {
 				this.value = (int) d;
 			} else {
 				this.value = d;
 			}
 			addFlag(AstFlag.NON_STATEMENT);
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + value + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(value);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	/**
 	 * A string is a string; Awk doesn't attempt to normalize
 	 * it until it is used in an arithmetic operation!
 	 */
 	private final class StringAst extends ScalarExpressionAst {

 		private String value;

 		private StringAst(String str) {
 			this.value = str;
 			addFlag(AstFlag.NON_STATEMENT);
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + value + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(value);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class RegexpAst extends ScalarExpressionAst {

 		private String regexpStr;
 		private boolean typed;

 		private RegexpAst(String regexpStr, boolean typedParam) {
 			this.regexpStr = regexpStr;
 			this.typed = typedParam;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + regexpStr + ")";
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (typed) {
 				tuples.regexp(regexpStr);
 			} else {
 				tuples.getInputField(0);
 				tuples.regexp(regexpStr);
 				tuples.matches();
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}

 		private int populateRawRegexpTuples(AwkTuples tuples) {
 			tuples.regexp(regexpStr);
 			return 1;
 		}
 	}

 	private final class ConditionPairAst extends ScalarExpressionAst {

 		private final long conditionPairId;

 		private ConditionPairAst(AST booleanAst1, AST booleanAst2) {
 			super(booleanAst1, booleanAst2);
 			conditionPairId = ++conditionPairCount;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			// The start condition is evaluated only while outside the range, and the
 			// end condition only once the range has started (including on the very
 			// record that starts it, so a range can begin and end on the same record)
 			pushSourceLineNumber(tuples);

 			Address enterRange = tuples.createAddress("conditionPairEnterRange");
 			Address testEnd = tuples.createAddress("conditionPairTestEnd");
 			Address withinRange = tuples.createAddress("conditionPairWithinRange");
 			Address end = tuples.createAddress("conditionPairEnd");

 			tuples.conditionPairInRange(conditionPairId);
 			tuples.ifTrue(testEnd);
 			getAst1().populateTuples(tuples);
 			tuples.ifTrue(enterRange);
 			tuples.push(0);
 			tuples.gotoAddress(end);

 			tuples.address(enterRange);
 			tuples.conditionPairEnter(conditionPairId);

 			tuples.address(testEnd);
 			getAst2().populateTuples(tuples);
 			tuples.ifFalse(withinRange);
 			tuples.conditionPairLeave(conditionPairId);

 			tuples.address(withinRange);
 			tuples.push(1);
 			tuples.address(end);

 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class BeginAst extends AST {

 		private BeginAst() {
 			super();
 			setBeginFlag(true);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(1);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class EndAst extends AST {

 		private EndAst() {
 			super();
 			setEndFlag(true);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(1);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class BeginFileAst extends AST {

 		private BeginFileAst() {
 			super();
 			setBeginFileFlag(true);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(1);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class EndFileAst extends AST {

 		private EndFileAst() {
 			super();
 			setEndFileFlag(true);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			tuples.push(1);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class PreIncAst extends ScalarExpressionAst {

 		private PreIncAst(AST symbolAst) {
 			super(symbolAst);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
 				// the sequence already leaves the new value on the stack
 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, true, false);
 				popSourceLineNumber(tuples);
 				return 1;
 			} else if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				tuples.inc(idAst.offset, idAst.isGlobal);
 			} else if (getAst1() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
 				if (arrAst.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arrAst.getAst1();
 					if (idAst.isScalar()) {
 						throw new SemanticException("Cannot use " + idAst + " as an array.");
 					}
 					idAst.setArray(true);
 				}
 				arrAst.populateTargetReferenceTuples(tuples);
 				tuples.incMapRef();
 			} else if (getAst1() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
 				dollarExpr.getAst1().populateTuples(tuples); // OPTIMIATION: duplicate the x in $x here
 				// so that it is not evaluated again
 				tuples.dup();
 				// stack contains eval of dollar arg
 				// tuples.assignAsInputField();
 				tuples.incDollarRef();
 				// OPTIMIATION continued: now evaluate
 				// the dollar expression with x (for $x)
 				// instead of evaluating the expression again
 				tuples.getInputField();
 				popSourceLineNumber(tuples);
 				return 1; // NOTE, short-circuit return here!
 			} else {
 				throw new NotImplementedError("unhandled preinc for " + getAst1());
 			}
 			// else
 			// assert false : "cannot refer for preInc to "+getAst1();
 			getAst1().populateTuples(tuples);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class PreDecAst extends ScalarExpressionAst {

 		private PreDecAst(AST symbolAst) {
 			super(symbolAst);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
 				// the sequence already leaves the new value on the stack
 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, false, false);
 				popSourceLineNumber(tuples);
 				return 1;
 			} else if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				tuples.dec(idAst.offset, idAst.isGlobal);
 			} else if (getAst1() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
 				if (arrAst.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arrAst.getAst1();
 					if (idAst.isScalar()) {
 						throw new SemanticException("Cannot use " + idAst + " as an array.");
 					}
 					idAst.setArray(true);
 				}
 				arrAst.populateTargetReferenceTuples(tuples);
 				tuples.decMapRef();
 			} else if (getAst1() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
 				dollarExpr.getAst1().populateTuples(tuples); // OPTIMIATION: duplicate the x in $x here
 				// so that it is not evaluated again
 				tuples.dup();
 				// stack contains eval of dollar arg
 				// tuples.assignAsInputField();
 				tuples.decDollarRef();
 				// OPTIMIATION continued: now evaluate
 				// the dollar expression with x (for $x)
 				// instead of evaluating the expression again
 				tuples.getInputField();
 				popSourceLineNumber(tuples);
 				return 1; // NOTE, short-circuit return here!
 			} else {
 				throw new NotImplementedError("unhandled predec for " + getAst1());
 			}
 			getAst1().populateTuples(tuples);
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class PostIncAst extends ScalarExpressionAst {

 		private PostIncAst(AST symbolAst) {
 			super(symbolAst);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (getAst1() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
 				dollarExpr.getAst1().populateTuples(tuples);
 				tuples.incDollarRef();
 			} else if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, true, true);
 			} else {
 				if (getAst1() instanceof ArrayReferenceAst) {
 					((ArrayReferenceAst) getAst1()).populateTargetValueTuples(tuples);
 					tuples.unaryPlus();
 				} else {
 					getAst1().populateTuples(tuples);
 				}
 				if (getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) getAst1();
 					tuples.postInc(idAst.offset, idAst.isGlobal);
 				} else if (getAst1() instanceof ArrayReferenceAst) {
 					ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
 					if (arrAst.getAst1() instanceof IDAst) {
 						IDAst idAst = (IDAst) arrAst.getAst1();
 						if (idAst.isScalar()) {
 							throw new SemanticException("Cannot use " + idAst + " as an array.");
 						}
 						idAst.setArray(true);
 					}
 					arrAst.populateTargetReferenceTuples(tuples);
 					tuples.incMapRef();
 				} else {
 					throw new NotImplementedError("unhandled postinc for " + getAst1());
 				}
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class PostDecAst extends ScalarExpressionAst {

 		private PostDecAst(AST symbolAst) {
 			super(symbolAst);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, false, true);
 				popSourceLineNumber(tuples);
 				return 1;
 			}
 			if (getAst1() instanceof ArrayReferenceAst) {
 				((ArrayReferenceAst) getAst1()).populateTargetValueTuples(tuples);
 				tuples.unaryPlus();
 			} else {
 				getAst1().populateTuples(tuples);
 			}
 			if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				tuples.postDec(idAst.offset, idAst.isGlobal);
 			} else if (getAst1() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
 				if (arrAst.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arrAst.getAst1();
 					if (idAst.isScalar()) {
 						throw new SemanticException("Cannot use " + idAst + " as an array.");
 					}
 					idAst.setArray(true);
 				}
 				arrAst.populateTargetReferenceTuples(tuples);
 				tuples.decMapRef();
 			} else if (getAst1() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
 				dollarExpr.getAst1().populateTuples(tuples);
 				tuples.decDollarRef();
 			} else {
 				throw new NotImplementedError("unhandled postinc for " + getAst1());
 			}
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class PrintAst extends ScalarExpressionAst {

 		private Token outputToken;
 		private boolean parenthesized;

 		private PrintAst(AST exprList, Token outToken, AST outputExpr, boolean parenthesized) {
 			super(exprList, outputExpr);
 			this.outputToken = outToken;
 			this.parenthesized = parenthesized;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			int paramCount;
 			if (getAst1() == null) {
 				if (parenthesized) {
 					throw new SemanticException("print() requires at least 1 argument");
 				}
 				paramCount = 0;
 			} else {
 				paramCount = getAst1().populateTuples(tuples);
 				if (paramCount == 0) {
 					throw new SemanticException("Cannot print the result. The expression doesn't return anything.");
 				}
 			}

 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}

 			if (outputToken == Token.GT) {
 				tuples.printToFile(paramCount, false); // false = no append
 			} else if (outputToken == Token.APPEND) {
 				tuples.printToFile(paramCount, true); // false = no append
 			} else if (outputToken == Token.PIPE) {
 				tuples.printToPipe(paramCount);
 			} else {
 				tuples.print(paramCount);
 			}

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	// we don't know if it is a scalar
 	/**
 	 * Returns whether the identifier names a JRT-managed special variable,
 	 * read and written through dedicated opcodes instead of a global slot.
 	 * ENVIRON and ARGV are excluded: they are special names but plain
 	 * slot-backed arrays, materialized by the preamble. SYMTAB and FUNCTAB
 	 * need no exclusion because they are not special names at all: they are
 	 * ordinary globals that the runtime populates when the script references
 	 * them.
 	 */
 	private boolean isJrtManagedSpecialName(String id) {
 		return isSpecialVariableName(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id);
 	}

 	/**
 	 * Returns whether the identifier names a special variable in the current
 	 * compile-time mode. Most special names (NR, FS, FILENAME, ...) always
 	 * are. The gawk-only ERRNO and ARGIND are special outside POSIX mode
 	 * only: with {@code --posix} they are plain identifiers — usable as
 	 * function parameters and compiled as ordinary globals — exactly like
 	 * {@code gawk --posix} treats them.
 	 *
 	 * @param id the identifier to inspect
 	 * @return {@code true} when the name is special in the current mode
 	 */
 	private boolean isSpecialVariableName(String id) {
 		if (!SPECIAL_VAR_NAMES.containsKey(id)) {
 			return false;
 		}
 		if (JRT.isGawkOnlySpecialVariable(id)) {
 			return !posix;
 		}
 		return true;
 	}

 	/** Emits the tuple pushing the value of a JRT-managed special variable. */
 	private void pushSpecialVariable(AwkTuples tuples, String id) {
 		switch (id) {
 		case "NF":
 			tuples.pushNF();
 			break;
 		case "NR":
 			tuples.pushNR();
 			break;
 		case "FNR":
 			tuples.pushFNR();
 			break;
 		case "FS":
 			tuples.pushFS();
 			break;
 		case "RS":
 			tuples.pushRS();
 			break;
 		case "OFS":
 			tuples.pushOFS();
 			break;
 		case "ORS":
 			tuples.pushORS();
 			break;
 		case "RSTART":
 			tuples.pushRSTART();
 			break;
 		case "RLENGTH":
 			tuples.pushRLENGTH();
 			break;
 		case "IGNORECASE":
 			tuples.pushIGNORECASE();
 			break;
 		case "FILENAME":
 			tuples.pushFILENAME();
 			break;
 		case "SUBSEP":
 			tuples.pushSUBSEP();
 			break;
 		case "CONVFMT":
 			tuples.pushCONVFMT();
 			break;
 		case "OFMT":
 			tuples.pushOFMT();
 			break;
 		case "ARGC":
 			tuples.pushARGC();
 			break;
 		case "ERRNO":
 			tuples.pushERRNO();
 			break;
 		case "ARGIND":
 			tuples.pushARGIND();
 			break;
 		default:
 			throw new Error("Unhandled special var: " + id);
 		}
 	}

 	/** Emits the tuple assigning the top of the stack to a JRT-managed special variable. */
 	private void assignSpecialVariable(AwkTuples tuples, String id) {
 		switch (id) {
 		case "NF":
 			tuples.assignNF();
 			break;
 		case "NR":
 			tuples.assignNR();
 			break;
 		case "FNR":
 			tuples.assignFNR();
 			break;
 		case "FS":
 			tuples.assignFS();
 			break;
 		case "RS":
 			tuples.assignRS();
 			break;
 		case "OFS":
 			tuples.assignOFS();
 			break;
 		case "ORS":
 			tuples.assignORS();
 			break;
 		case "RSTART":
 			tuples.assignRSTART();
 			break;
 		case "RLENGTH":
 			tuples.assignRLENGTH();
 			break;
 		case "IGNORECASE":
 			tuples.assignIGNORECASE();
 			break;
 		case "FILENAME":
 			tuples.assignFILENAME();
 			break;
 		case "SUBSEP":
 			tuples.assignSUBSEP();
 			break;
 		case "CONVFMT":
 			tuples.assignCONVFMT();
 			break;
 		case "OFMT":
 			tuples.assignOFMT();
 			break;
 		case "ARGC":
 			tuples.assignARGC();
 			break;
 		case "ERRNO":
 			tuples.assignERRNO();
 			break;
 		case "ARGIND":
 			tuples.assignARGIND();
 			break;
 		default:
 			throw new Error("Unhandled special var: " + id);
 		}
 	}

 	/*
 	 * Increments or decrements a JRT-managed special variable. Specials live in
 	 * the JRT rather than in a global slot, so the slot-based INC/DEC opcodes
 	 * cannot be used; the sequence below reads, adjusts, and assigns through
 	 * the special-variable opcodes, leaving the expression value (old value for
 	 * postfix, new value for prefix) on the stack.
 	 */
 	private void populateSpecialIncDec(AwkTuples tuples, String id, boolean increment, boolean postfix) {
 		pushSpecialVariable(tuples, id);
 		if (postfix) {
 			tuples.dup();
 		}
 		tuples.push(Long.valueOf(1L));
 		if (increment) {
 			tuples.add();
 		} else {
 			tuples.subtract();
 		}
 		assignSpecialVariable(tuples, id);
 		if (postfix) {
 			tuples.pop();
 		}
 	}

 	/*
 	 * A call-expression node is normally stamped when it is reduced, after the
 	 * lexer may already have consumed the line terminator; the first argument
 	 * carries the line of the call site itself, which is what gawk-style
 	 * diagnostics report.
 	 */
 	private int extensionCallLineNumber(AST params) {
 		if (params != null && params.getAst1() != null) {
 			return params.getAst1().getLineNo();
 		}
 		return currentSourceLineNumber();
 	}

 	private final class ExtensionAst extends AST {

 		private final ExtensionFunction function;

 		private ExtensionAst(ExtensionFunction functionParam, AST paramAst, int lineNoParam) {
 			super(lineNoParam, paramAst);
 			this.function = functionParam;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			int argCount;
 			if (getAst1() == null) {
 				argCount = 0;
 			} else {
 				argCount = countParams((FunctionCallParamListAst) getAst1());
 			}

 			int[] reqArrayIdxs = function.collectAssocArrayIndexes(argCount);
 			int[] rawValueIdxs = function.collectRawValueIndexes(argCount);

 			int paramCount;
 			if (getAst1() == null) {
 				paramCount = 0;
 			} else {
 				Set<Integer> arrayIndexes = new HashSet<Integer>();
 				Set<Integer> rawValueIndexes = new HashSet<Integer>();
 				Set<Integer> literalRegexpIndexes = new HashSet<Integer>();
 				for (int idx : rawValueIdxs) {
 					rawValueIndexes.add(Integer.valueOf(idx));
 				}
 				for (int idx : reqArrayIdxs) {
 					AST paramAst = getParamAst((FunctionCallParamListAst) getAst1(), idx).getAst1();
 					if (paramAst instanceof IDAst) {
 						IDAst idAst = (IDAst) paramAst;
 						if (idAst.isScalar()) {
 							throw new SemanticException(
 									"Extension '"
 											+ function.getKeyword()
 											+ "' requires parameter position "
 											+ idx
 											+ " be an associative array, not a scalar.");
 						}
 						idAst.setArray(true);
 						arrayIndexes.add(Integer.valueOf(idx));
 					} else if (paramAst instanceof ArrayReferenceAst) {
 						arrayIndexes.add(Integer.valueOf(idx));
 					}
 				}
 				for (int idx : function.collectRegexpIndexes(argCount)) {
 					literalRegexpIndexes.add(Integer.valueOf(idx));
 				}

 				paramCount = populateActualParameters(
 						tuples,
 						(FunctionCallParamListAst) getAst1(),
 						arrayIndexes,
 						rawValueIndexes,
 						literalRegexpIndexes,
 						0);
 			}
 			tuples.extension(function, paramCount);
 			popSourceLineNumber(tuples);
 			// an extension always returns a value, even if it is blank/null
 			return 1;
 		}

 		private AST getParamAst(FunctionCallParamListAst pAst, int pos) {
 			for (int i = 0; i < pos; ++i) {
 				pAst = (FunctionCallParamListAst) pAst.getAst2();
 				if (pAst == null) {
 					throw new SemanticException("More arguments required for assoc array parameter position specification.");
 				}
 			}
 			return pAst;
 		}

 		private int countParams(FunctionCallParamListAst pAst) {
 			int cnt = 0;
 			while (pAst != null) {
 				pAst = (FunctionCallParamListAst) pAst.getAst2();
 				++cnt;
 			}
 			return cnt;
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + function.getKeyword() + ")";
 		}
 	}

 	private final class PrintfAst extends ScalarExpressionAst {

 		private Token outputToken;

 		private PrintfAst(AST exprList, Token outToken, AST outputExpr) {
 			super(exprList, outputExpr);
 			this.outputToken = outToken;
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			int paramCount;
 			if (getAst1() == null) {
 				throw new SemanticException("printf requires at least 1 argument");
 			} else {
 				paramCount = getAst1().populateTuples(tuples);
 				if (paramCount == 0) {
 					throw new SemanticException("Cannot printf the result. The expression doesn't return anything.");
 				}
 			}

 			if (getAst2() != null) {
 				getAst2().populateTuples(tuples);
 			}

 			if (outputToken == Token.GT) {
 				tuples.printfToFile(paramCount, false); // false = no append
 			} else if (outputToken == Token.APPEND) {
 				tuples.printfToFile(paramCount, true); // false = no append
 			} else if (outputToken == Token.PIPE) {
 				tuples.printfToPipe(paramCount);
 			} else {
 				tuples.printf(paramCount);
 			}

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class GetlineAst extends ScalarExpressionAst {

 		private GetlineAst(AST pipeExpr, AST lvalueAst, AST inRedirect) {
 			super(pipeExpr, lvalueAst, inRedirect);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			// gawk restriction: only redirected forms of getline may be used
 			// inside BEGINFILE/ENDFILE rules. Direct uses are rejected here at
 			// compile time; uses reached through user-defined functions are
 			// caught at runtime by the interpreter, which knows the current
 			// rule.
 			if (getAst1() == null && getAst3() == null) {
 				AST enclosingRule = searchFor(AstFlag.NEXTABLE);
 				AST pattern = enclosingRule == null ? null : enclosingRule.getAst1();
 				if (pattern != null && (pattern.isBeginFile() || pattern.isEndFile())) {
 					throw new SemanticException(
 							"non-redirected `getline' invalid inside `"
 									+ (pattern.isBeginFile() ? "BEGINFILE" : "ENDFILE")
 									+ "' rule");
 				}
 			}
 			if (getAst1() == null && getAst3() == null && getAst2() == null) {
 				tuples.getlineInput();
 				popSourceLineNumber(tuples);
 				return 1;
 			}
 			// When no record is read (EOF or, for the redirected forms, an I/O
 			// error), the read opcode pushes only the return code and jumps
 			// here, past the assignment, so the target keeps its previous
 			// value, as POSIX and gawk require.
 			Address noRecord = tuples.createAddress("getline_no_record");
 			if (getAst1() != null) {
 				getAst1().populateTuples(tuples);// stack has getAst1() (i.e., "command")
 				tuples.useAsCommandInput(noRecord);
 			} else if (getAst3() != null) {
 // getline ... < getAst3()
 				getAst3().populateTuples(tuples); // stack has getAst3() (i.e., "filename")
 				tuples.useAsFileInput(noRecord);
 			} else {
 				tuples.getlineInputToTarget(noRecord);
 			}
 			// A record was read: 2 resultant values on the stack!
 			// 2nd - 1 for success
 			// 1st(top) - the input
 			if (getAst2() == null) {
 				tuples.assignAsInput();
 				// stack still has the input, to be popped below...
 				// (all assignment results are placed on the stack)
 			} else if (getAst2() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst2();
 				tuples.assign(idAst.offset, idAst.isGlobal);
 				if (idAst.id.equals("RS")) {
 					tuples.applyRS();
 				}
 			} else if (getAst2() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arr = (ArrayReferenceAst) getAst2();
 				if (arr.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arr.getAst1();
 					if (idAst.isScalar()) {
 						throw new SemanticException("Cannot use " + idAst + " as an array.");
 					}
 					idAst.setArray(true);
 				}
 				arr.populateTargetReferenceTuples(tuples);
 				tuples.assignMapElement();
 			} else if (getAst2() instanceof DollarExpressionAst) {
 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst2();
 				// stack gets the eval of the dollar arg, kept in ast1
 				dollarExpr.getAst1().populateTuples(tuples);
 				tuples.assignAsInputField();
 			} else {
 				throw new SemanticException("Cannot getline into a " + getAst2());
 			}
 			// get rid of value left by the assignment
 			tuples.pop();
 			if (getAst2() instanceof ArrayReferenceAst || getAst2() instanceof DollarExpressionAst) {
 				// gawk evaluates the subscript of an array or field target even
 				// when nothing was read — and the mere reference creates the
 				// array element — a dark corner the getline5 gawk test pins.
 				// Replay that evaluation on the no-record path.
 				Address end = tuples.createAddress("getline_end");
 				tuples.gotoAddress(end);
 				tuples.address(noRecord);
 				// Evaluating the target as a value creates an array element and
 				// rejects an invalid field number, exactly like the assignment
 				// would, without storing anything.
 				getAst2().populateTuples(tuples);
 				tuples.pop();
 				tuples.address(end);
 			} else {
 				tuples.address(noRecord);
 			}
 			// one value (the return code) is left on the stack
 			popSourceLineNumber(tuples);
 			return 1;
 		}
 	}

 	private final class ReturnStatementAst extends AST {

 		private ReturnStatementAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			AST returnable = searchFor(AstFlag.RETURNABLE);
 			if (returnable == null) {
 				throw new SemanticException("Cannot use return here.");
 			}
 			if (getAst1() != null) {
 				getAst1().populateTuples(tuples);
 				tuples.setReturnResult();
 			}
 			tuples.gotoAddress(returnable.returnAddress());
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class ExitStatementAst extends AST {

 		private ExitStatementAst(AST expr) {
 			super(expr);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			if (getAst1() != null) {
 				getAst1().populateTuples(tuples);
 				tuples.exitWithCode();
 			} else {
 				tuples.exitWithoutCode();
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class DeleteStatementAst extends AST {

 		private DeleteStatementAst(AST symbolAst) {
 			super(symbolAst);
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);

 			if (getAst1() instanceof ArrayReferenceAst) {
 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
 				if (arrAst.getAst1() instanceof IDAst) {
 					IDAst idAst = (IDAst) arrAst.getAst1();
 					idAst.setArray(true);
 				}
 				arrAst.populateTargetReferenceTuples(tuples);
 				tuples.deleteMapElement();
 			} else if (getAst1() instanceof IDAst) {
 				IDAst idAst = (IDAst) getAst1();
 				idAst.setArray(true);
 				tuples.deleteArray(idAst.offset, idAst.isGlobal);
 			} else {
 				throw new Error("Should never reach here : delete for " + getAst1());
 			}

 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private class BreakStatementAst extends AST {

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			AST breakable = searchFor(AstFlag.BREAKABLE);
 			if (breakable == null) {
 				throw new SemanticException("cannot break; not within a loop");
 			}
 			tuples.gotoAddress(breakable.breakAddress());
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private class NextStatementAst extends AST {

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			AST nextable = searchFor(AstFlag.NEXTABLE);
 			if (nextable == null) {
 				// Inside a user-defined function: the calling rule cannot be
 				// known statically (the same function may be called from both
 				// an input rule and a special rule), so emit the runtime form,
 				// which unwinds the function calls and resumes the main input
 				// loop, or reports a fatal error for BEGIN, END, BEGINFILE,
 				// and ENDFILE callers.
 				tuples.execNext();
 			} else {
 				tuples.gotoAddress(nextable.nextAddress());
 			}
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private class NextfileStatementAst extends AST {

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			AST nextable = searchFor(AstFlag.NEXTABLE);
 			if (nextable != null) {
 				// Direct use inside a rule: BEGIN, END, and ENDFILE reject
 				// nextfile at compile time, mirroring gawk's fatal errors.
 				// (Uses within user-defined functions are checked at runtime.)
 				AST pattern = nextable.getAst1();
 				if (pattern != null && (pattern.isBegin() || pattern.isEnd() || pattern.isEndFile())) {
 					String ruleName = pattern.isBegin() ? "BEGIN" : pattern.isEnd() ? "END" : "ENDFILE";
 					throw new SemanticException(
 							"`nextfile' cannot be called from a `" + ruleName + "' rule.");
 				}
 			}
 			tuples.execNextfile();
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	private final class ContinueStatementAst extends AST {

 		private ContinueStatementAst() {
 			super();
 		}

 		@Override
 		public int populateTuples(AwkTuples tuples) {
 			pushSourceLineNumber(tuples);
 			AST continueable = searchFor(AstFlag.CONTINUEABLE);
 			if (continueable == null) {
 				throw new SemanticException("cannot issue a continue; not within any loops");
 			}
 			tuples.gotoAddress(continueable.continueAddress());
 			popSourceLineNumber(tuples);
 			return 0;
 		}
 	}

 	// this was static...
 	// made non-static to throw a meaningful ParserException when necessary
 	private final class FunctionProxy implements Supplier<Address> {

 		private FunctionDefAst functionDefAst;
 		private String id;

 		private FunctionProxy(String id) {
 			this.id = id;
 		}

 		private void setFunctionDefinition(FunctionDefAst functionDef) {
 			if (functionDefAst != null) {
 				throw parserException("function " + functionDef + " already defined");
 			} else {
 				functionDefAst = functionDef;
 			}
 		}

 		private boolean isDefined() {
 			return functionDefAst != null;
 		}

 		@Override
 		public Address get() {
 			return functionDefAst.getAddress();
 		}

 		private String getFunctionName() {
 			return id;
 		}

 		private int getFunctionParamCount() {
 			return functionDefAst.paramCount();
 		}

 		@Override
 		public String toString() {
 			return super.toString() + " (" + id + ")";
 		}

 	}

 	/**
 	 * Adds {varName -&gt; offset} mappings to the tuples so that global variables
 	 * can be set by the interpreter while processing filename and name=value
 	 * entries from the command-line.
 	 * Also sends function names to the tuples, to provide the back end
 	 * with names to invalidate if name=value assignments are passed
 	 * in via the -v or ARGV arguments.
 	 *
 	 * @param tuples The tuples to add the mapping to.
 	 */
 	public void populateGlobalVariableNameToOffsetMappings(AwkTuples tuples) {
 		for (String varname : symbolTable.globalIds.keySet()) {
 			IDAst idAst = symbolTable.globalIds.get(varname);
 			// The last arg originally was ", idAst.isScalar", but this is not set true
 			// if the variable use is ambiguous. Therefore, assume it is a scalar
 			// if it's Token.NOT used as an array.
 			tuples.addGlobalVariableNameToOffsetMapping(varname, idAst.offset, idAst.isArray);
 		}
 		tuples.setFunctionNameSet(symbolTable.functionProxies.keySet());
 	}

 	private class AwkSymbolTableImpl {

 		int numGlobals() {
 			return globalIds.size();
 		}

 		// "constants"
 		private BeginAst beginAst = null;
 		private EndAst endAst = null;
 		private BeginFileAst beginFileAst = null;
 		private EndFileAst endFileAst = null;

 		// functions (proxies)
 		private Map<String, FunctionProxy> functionProxies = new HashMap<String, FunctionProxy>();
 		private Map<String, Tuple.IndirectFunctionTarget> cachedIndirectFunctionTargets;

 		// variable management
 		private Map<String, IDAst> globalIds = new HashMap<String, IDAst>();
 		private Map<String, Map<String, IDAst>> localIds = new HashMap<String, Map<String, IDAst>>();
 		private Map<String, Set<String>> functionParameters = new HashMap<String, Set<String>>();
 		private Set<String> ids = new HashSet<String>();

 		// current function definition for symbols
 		private String currentFunctionName = null;

 		// using set/clear rather than push/pop, it is impossible to define functions within functions
 		void setFunctionName(String functionName) {
 			this.currentFunctionName = functionName;
 		}

 		void clearFunctionName(String functionName) {
 			this.currentFunctionName = null;
 		}

 		AST addBEGIN() {
 			if (beginAst == null) {
 				beginAst = new BeginAst();
 			}
 			return beginAst;
 		}

 		AST addEND() {
 			if (endAst == null) {
 				endAst = new EndAst();
 			}
 			return endAst;
 		}

 		AST addBEGINFILE() {
 			if (beginFileAst == null) {
 				beginFileAst = new BeginFileAst();
 			}
 			return beginFileAst;
 		}

 		AST addENDFILE() {
 			if (endFileAst == null) {
 				endFileAst = new EndFileAst();
 			}
 			return endFileAst;
 		}

 		/**
 		 * Returns whether the script references the named global variable,
 		 * without creating a symbol for it.
 		 */
 		private boolean isGlobalReferenced(String id) {
 			IDAst idAst = globalIds.get(id);
 			return idAst != null && idAst.isReferenced();
 		}

 		private IDAst getID(String id) {
 			id = resolveVariableIdentifier(id);

 			Map<String, IDAst> map;
 			if (currentFunctionName == null) {
 				map = globalIds;
 			} else {
 				Set<String> set = functionParameters.get(currentFunctionName);
 				// we need "set != null && ..." here because if function
 				// is defined with no args (i.e., function f() ...),
 				// then set is null
 				if (set != null && set.contains(id)) {
 					map = localIds.get(currentFunctionName);
 					if (map == null) {
 						map = new HashMap<String, IDAst>();
 						localIds.put(currentFunctionName, map);
 					}
 				} else {
 					map = globalIds;
 				}
 			}
 			if (map == globalIds) {
 				// Only global variables share the namespace with function names.
 				// Formal parameters may legitimately have the same name as an
 				// unrelated function.
 				if (functionProxies.get(id) != null) {
 					throw parserException("cannot use " + id + " as a variable; it is a function");
 				}
 				ids.add(id);
 			}
 			IDAst idAst = map.get(id);
 			if (idAst == null) {
 				idAst = new IDAst(id, map == globalIds);
 				idAst.offset = map.size();
 				if (map == globalIds && !posix && ("SYMTAB".equals(id) || "FUNCTAB".equals(id))) {
 					// the runtime-provided meta tables are array-only, as in
 					// gawk: using them as scalars must fail like any array
 					idAst.setArray(true);
 				}
 				map.put(id, idAst);
 			}
 			return idAst;
 		}

 		private String resolveVariableIdentifier(String id) {
 			if (currentFunctionName != null) {
 				Set<String> parameters = functionParameters.get(currentFunctionName);
 				if (parameters != null && parameters.contains(id)) {
 					return id;
 				}
 			}
 			return qualifyGlobalIdentifier(id);
 		}

 		AST addID(String id) throws ParserException {
 			IDAst retVal = getID(id);
 			retVal.markReferenced();
 			/// ***
                         /// We really don't know if the evaluation is for an array or for a scalar
                         /// here, because we can use an array as a function parameter (passed by reference).
 			/// ***
 			// if (retVal.isArray)
 			// throw parserException("Cannot use "+retVal+" as a scalar.");
 			// retVal.isScalar = true;
 			return retVal;
 		}

 		int addFunctionParameter(String functionName, String id) {
 			int namespaceSeparator = functionName.indexOf("::");
 			String unqualifiedFunctionName = namespaceSeparator < 0 ?
 					functionName : functionName.substring(namespaceSeparator + 2);
 			if (unqualifiedFunctionName.equals(id)) {
 				throw parserException("cannot use " + id + " as a parameter; it is the function name");
 			}
 			Set<String> set = functionParameters.get(functionName);
 			if (set == null) {
 				set = new HashSet<String>();
 				functionParameters.put(functionName, set);
 			}
 			if (set.contains(id)) {
 				throw parserException("multiply defined parameter " + id + " in function " + functionName);
 			}
 			int retval = set.size();
 			set.add(id);
 			Map<String, IDAst> map = localIds.get(functionName);
 			if (map == null) {
 				map = new HashMap<String, IDAst>();
 				localIds.put(functionName, map);
 			}
 			IDAst idAst = map.get(id);
 			if (idAst == null) {
 				idAst = new IDAst(id, map == globalIds);
 				idAst.offset = map.size();
 				map.put(id, idAst);
 			}

 			return retval;
 		}

 		IDAst getFunctionParameterIDAST(String functionName, String fIdString) {
 			return localIds.get(functionName).get(fIdString);
 		}

 		AST addArrayID(String id) throws ParserException {
 			IDAst retVal = getID(id);
 			retVal.markReferenced();
 			retVal.setArray(true);
 			return retVal;
 		}

 		AST addFunctionDef(String functionName, AST paramList, AST block) {
 			if (ids.contains(functionName)) {
 				throw parserException("cannot use " + functionName + " as a function; it is a variable");
 			}
 			FunctionProxy functionProxy = functionProxies.get(functionName);
 			if (functionProxy == null) {
 				functionProxy = new FunctionProxy(functionName);
 				functionProxies.put(functionName, functionProxy);
 			}
 			FunctionDefAst functionDef = new FunctionDefAst(functionName, paramList, block);
 			functionProxy.setFunctionDefinition(functionDef);
 			return functionDef;
 		}

 		AST addFunctionCall(String id, AST paramList) {
 			id = qualifyGlobalIdentifier(id);
 			FunctionProxy functionProxy = functionProxies.get(id);
 			if (functionProxy == null) {
 				functionProxy = new FunctionProxy(id);
 				functionProxies.put(id, functionProxy);
 			}
 			return new FunctionCallAst(functionProxy, paramList);
 		}

 		Map<String, Tuple.IndirectFunctionTarget> indirectFunctionTargets() {
 			if (cachedIndirectFunctionTargets != null) {
 				return cachedIndirectFunctionTargets;
 			}
 			Map<String, Tuple.IndirectFunctionTarget> targets = new HashMap<String, Tuple.IndirectFunctionTarget>();
 			for (Map.Entry<String, FunctionProxy> entry : functionProxies.entrySet()) {
 				FunctionProxy proxy = entry.getValue();
 				if (proxy.isDefined()) {
 					targets
 							.put(
 									entry.getKey(),
 									new Tuple.IndirectFunctionTarget(
 											proxy,
 											proxy.getFunctionParamCount(),
 											collectArrayParameterIndexes(proxy.functionDefAst)));
 				}
 			}
 			cachedIndirectFunctionTargets = Collections.unmodifiableMap(targets);
 			return cachedIndirectFunctionTargets;
 		}

 		AST addArrayReference(String id, AST idxAst, int lineNo) throws ParserException {
 			return new ArrayReferenceAst(lineNo, addArrayID(id), idxAst);
 		}

 		// constants are no longer cached/hashed so that individual ASTs
 		// can report accurate line numbers upon errors

 		AST addINTEGER(String integer) {
 			try {
 				return new IntegerAst(Long.parseLong(integer));
 			} catch (NumberFormatException beyondLongRange) {
 				// A literal too large for a 64-bit integer is a floating-point
 				// constant, as in gawk, rather than a parse error.
 				return new DoubleAst(Double.parseDouble(integer));
 			}
 		}

 		AST addDOUBLE(String dbl) {
 			return new DoubleAst(Double.valueOf(dbl));
 		}

 		AST addSTRING(String str) {
 			return new StringAst(str);
 		}

 		AST addREGEXP(String localRegexp) {
 			return new RegexpAst(localRegexp, false);
 		}

 		AST addTYPED_REGEXP(String localRegexp) {
 			return new RegexpAst(localRegexp, true);
 		}
 	}

 	private ParserException parserException(String msg) {
 		return new ParserException(
 				msg,
 				currentScriptSource.getDescription(),
 				reader.getLineNumber());
 	}
 }